openvibers/openvibe.live Commits
Live speech-to-text: audio longer than 45 s goes to Parakeet in 30 s pieces (a 300 s VOD window needed 3.1 GB in one run; a 30 s piece about 0.8 GB) (#148)
Changes
@@@ -7,7 +7,8 @@* Measured 2026-10-16 on 2 min of real stream talk on this 4-core box: 394 words in 31 s,* punctuated, where whisper base.en (the old live model) caught 327 words in 15 s and dropped* whole sentences ("they put in" → "the Putin"), and large-v3-turbo needed 121 s (1x real time,-- * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did.++ * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did. Audio longer than 45 s++ * (VOD windows) is decoded in 30 s pieces: one 300 s piece needed 3.1 GB.* - whisper.cpp, for every other language (Japanese, Korean, Chinese…) and as the fallback when* Parakeet is not installed. ASR_ENGINE=whisper forces it everywhere.*@@@ -271,7 +272,48 @@ function parseParakeet(stderr, offsetSec = 0) {return out;}--function _parakeetInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {++// Parakeet's memory and time grow faster than the audio: measured on this box, 10 s took 0.74 GB, 60 s 1.1 GB and a++// 300 s VOD window 3.1 GB in 154 s. Anything longer than CHUNK_MAX_SEC is cut into CHUNK_SEC pieces (PCM, cut exactly,++// so each piece's times only shift by its start) and decoded one after another.++const CHUNK_SEC = 30;++const CHUNK_MAX_SEC = 45;++function wavSeconds(wavPath) {++ try { return Math.max(0, (fs.statSync(wavPath).size - 44) / 32000); } catch { return 0; } // 16 kHz mono s16le++}++function _splitWav(wavPath, dir) {++ return new Promise((resolve) => {++ let ff;++ try { ff = _track(spawn('nice', ['-n', '15', 'ffmpeg', '-nostdin', '-loglevel', 'error', '-y', '-i', wavPath, '-f', 'segment', '-segment_time', String(CHUNK_SEC), '-c', 'copy', path.join(dir, 'p%04d.wav')], { stdio: 'ignore' })); }++ catch { return resolve(null); }++ ff.on('close', (code) => {++ if (code !== 0) return resolve(null);++ try { resolve(fs.readdirSync(dir).filter((n) => /^p\d{4}\.wav$/.test(n)).sort().map((n) => path.join(dir, n))); } catch { resolve(null); }++ });++ ff.on('error', () => resolve(null));++ });++}++async function _parakeetInner(wavPath, opts = {}) {++ if (!wavPath || wavSeconds(wavPath) <= CHUNK_MAX_SEC) return await _parakeetOnce(wavPath, opts);++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'openvibe-pk-'));++ try {++ const pieces = await _splitWav(wavPath, dir);++ if (!pieces || !pieces.length) return { text: '', segments: [], ok: false, error: 'could not split the audio' };++ const segments = [];++ const deadline = Date.now() + (opts.timeoutMs || 180000);++ for (let i = 0; i < pieces.length; i++) {++ const left = deadline - Date.now();++ if (left <= 0) return { text: '', segments: [], ok: false, error: 'timeout' };++ const r = await _parakeetOnce(pieces[i], { ...opts, timeoutMs: left, offsetSec: (opts.offsetSec || 0) + i * CHUNK_SEC });++ if (!r.ok) return r;++ segments.push(...r.segments);++ }++ return { text: _joinSegments(segments), segments, ok: true, engine: 'parakeet' };++ } finally {++ try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* */ }++ }++}++++function _parakeetOnce(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {return new Promise((resolve) => {if (!parakeetAvailable() || !wavPath || !fs.existsSync(wavPath)) return resolve({ text: '', segments: [], ok: false, error: 'parakeet unavailable' });const args = ['-t', String(_threads(live)), '-ps', '-np', '-m', PARAKEET_MODEL, '-f', wavPath];nnew file mode 100644@@@ -0,0 +1,39 @@++'use strict';++// Long audio goes to Parakeet in 30 s pieces (server/ai/transcribe.js): a 70 s WAV is cut into three, each decoded on++// its own, and each piece's lines shift by where the piece starts. A stand-in parakeet-cli (a script that says "word"++// one second into whatever it is given) stands in for the real one.++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const { execFileSync } = require('child_process');++++const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-pk-chunks-'));++const bin = path.join(dir, 'parakeet-cli');++const calls = path.join(dir, 'calls.log');++fs.writeFileSync(bin, `#!/bin/sh\necho "$@" >> ${calls}\necho ' [ 0] id= 1 frame= 0 dur_idx= 1 dur_val= 1 p=0.9 plog=-1 t0= 100 t1= 150 word_start=true "\u2581Word."' 1>&2\n`);++fs.chmodSync(bin, 0o755);++const model = path.join(dir, 'model.bin');++fs.writeFileSync(model, 'x');++process.env.PARAKEET_BIN = bin;++process.env.PARAKEET_MODEL = model;++const t = require('../server/ai/transcribe');++++let ffmpeg = true;++try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); } catch { ffmpeg = false; }++(async () => {++ assert.strictEqual(t.engineFor('en'), 'parakeet');++ if (!ffmpeg) { console.log('transcribe-parakeet-chunks: ffmpeg not installed here, chunking not exercised'); return; }++ const wav = path.join(dir, 'long.wav');++ execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-f', 'lavfi', '-i', 'anullsrc=r=16000:cl=mono', '-t', '70', '-c:a', 'pcm_s16le', wav]);++ const r = await t.transcribeWavDetailed(wav, { offsetSec: 100, timeoutMs: 60000 });++ assert.ok(r.ok, r.error);++ assert.deepStrictEqual(r.segments.map((s) => s.start), [101, 131, 161], 'each 30 s piece shifted by its start');++ assert.strictEqual(fs.readFileSync(calls, 'utf8').trim().split('\n').length, 3, 'three pieces, three runs');++ const short = path.join(dir, 'short.wav');++ execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-f', 'lavfi', '-i', 'anullsrc=r=16000:cl=mono', '-t', '10', '-c:a', 'pcm_s16le', short]);++ const s = await t.transcribeWavDetailed(short, { offsetSec: 0 });++ assert.deepStrictEqual(s.segments.map((x) => [x.start, x.text]), [[1, 'Word.']], 'short audio: one run, no split');++ assert.ok(!fs.readdirSync(os.tmpdir()).some((n) => n.startsWith('openvibe-pk-') && fs.existsSync(path.join(os.tmpdir(), n, 'p0000.wav'))), 'pieces are cleaned up');++ console.log('transcribe-parakeet-chunks: ok');++})().catch((e) => { console.error(e); process.exit(1); }).finally(() => fs.rmSync(dir, { recursive: true, force: true }));
@@ -7,7 +7,8 @@
@@ -7,7 +7,8 @@
* Measured 2026-10-16 on 2 min of real stream talk on this 4-core box: 394 words in 31 s,
* Measured 2026-10-16 on 2 min of real stream talk on this 4-core box: 394 words in 31 s,
* punctuated, where whisper base.en (the old live model) caught 327 words in 15 s and dropped
* punctuated, where whisper base.en (the old live model) caught 327 words in 15 s and dropped
* whole sentences ("they put in" → "the Putin"), and large-v3-turbo needed 121 s (1x real time,
* whole sentences ("they put in" → "the Putin"), and large-v3-turbo needed 121 s (1x real time,
- * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did.
+ * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did. Audio longer than 45 s
+ * (VOD windows) is decoded in 30 s pieces: one 300 s piece needed 3.1 GB.
* - whisper.cpp, for every other language (Japanese, Korean, Chinese…) and as the fallback when
* - whisper.cpp, for every other language (Japanese, Korean, Chinese…) and as the fallback when
* Parakeet is not installed. ASR_ENGINE=whisper forces it everywhere.
* Parakeet is not installed. ASR_ENGINE=whisper forces it everywhere.
*
*
@@ -271,7 +272,48 @@ function parseParakeet(stderr, offsetSec = 0) {
@@ -271,7 +272,48 @@ function parseParakeet(stderr, offsetSec = 0) {
return out;
return out;
}
}
-function _parakeetInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {
+// Parakeet's memory and time grow faster than the audio: measured on this box, 10 s took 0.74 GB, 60 s 1.1 GB and a
+// 300 s VOD window 3.1 GB in 154 s. Anything longer than CHUNK_MAX_SEC is cut into CHUNK_SEC pieces (PCM, cut exactly,
+// so each piece's times only shift by its start) and decoded one after another.
+const CHUNK_SEC = 30;
+const CHUNK_MAX_SEC = 45;
+function wavSeconds(wavPath) {
+ try { return Math.max(0, (fs.statSync(wavPath).size - 44) / 32000); } catch { return 0; } // 16 kHz mono s16le
+}
+function _splitWav(wavPath, dir) {
+ return new Promise((resolve) => {
+ let ff;
+ try { ff = _track(spawn('nice', ['-n', '15', 'ffmpeg', '-nostdin', '-loglevel', 'error', '-y', '-i', wavPath, '-f', 'segment', '-segment_time', String(CHUNK_SEC), '-c', 'copy', path.join(dir, 'p%04d.wav')], { stdio: 'ignore' })); }
+ catch { return resolve(null); }
+ ff.on('close', (code) => {
+ if (code !== 0) return resolve(null);
+ try { resolve(fs.readdirSync(dir).filter((n) => /^p\d{4}\.wav$/.test(n)).sort().map((n) => path.join(dir, n))); } catch { resolve(null); }
+ });
+ ff.on('error', () => resolve(null));
+ });
+}
+async function _parakeetInner(wavPath, opts = {}) {
+ if (!wavPath || wavSeconds(wavPath) <= CHUNK_MAX_SEC) return await _parakeetOnce(wavPath, opts);
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'openvibe-pk-'));
+ try {
+ const pieces = await _splitWav(wavPath, dir);
+ if (!pieces || !pieces.length) return { text: '', segments: [], ok: false, error: 'could not split the audio' };
+ const segments = [];
+ const deadline = Date.now() + (opts.timeoutMs || 180000);
+ for (let i = 0; i < pieces.length; i++) {
+ const left = deadline - Date.now();
+ if (left <= 0) return { text: '', segments: [], ok: false, error: 'timeout' };
+ const r = await _parakeetOnce(pieces[i], { ...opts, timeoutMs: left, offsetSec: (opts.offsetSec || 0) + i * CHUNK_SEC });
+ if (!r.ok) return r;
+ segments.push(...r.segments);
+ }
+ return { text: _joinSegments(segments), segments, ok: true, engine: 'parakeet' };
+ } finally {
+ try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* */ }
+ }
+}
+
+function _parakeetOnce(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {
return new Promise((resolve) => {
return new Promise((resolve) => {
if (!parakeetAvailable() || !wavPath || !fs.existsSync(wavPath)) return resolve({ text: '', segments: [], ok: false, error: 'parakeet unavailable' });
if (!parakeetAvailable() || !wavPath || !fs.existsSync(wavPath)) return resolve({ text: '', segments: [], ok: false, error: 'parakeet unavailable' });
const args = ['-t', String(_threads(live)), '-ps', '-np', '-m', PARAKEET_MODEL, '-f', wavPath];
const args = ['-t', String(_threads(live)), '-ps', '-np', '-m', PARAKEET_MODEL, '-f', wavPath];
new file mode 100644
new file mode 100644
@@ -0,0 +1,39 @@
@@ -0,0 +1,39 @@
+'use strict';
+// Long audio goes to Parakeet in 30 s pieces (server/ai/transcribe.js): a 70 s WAV is cut into three, each decoded on
+// its own, and each piece's lines shift by where the piece starts. A stand-in parakeet-cli (a script that says "word"
+// one second into whatever it is given) stands in for the real one.
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-pk-chunks-'));
+const bin = path.join(dir, 'parakeet-cli');
+const calls = path.join(dir, 'calls.log');
+fs.writeFileSync(bin, `#!/bin/sh\necho "$@" >> ${calls}\necho ' [ 0] id= 1 frame= 0 dur_idx= 1 dur_val= 1 p=0.9 plog=-1 t0= 100 t1= 150 word_start=true "\u2581Word."' 1>&2\n`);
+fs.chmodSync(bin, 0o755);
+const model = path.join(dir, 'model.bin');
+fs.writeFileSync(model, 'x');
+process.env.PARAKEET_BIN = bin;
+process.env.PARAKEET_MODEL = model;
+const t = require('../server/ai/transcribe');
+
+let ffmpeg = true;
+try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); } catch { ffmpeg = false; }
+(async () => {
+ assert.strictEqual(t.engineFor('en'), 'parakeet');
+ if (!ffmpeg) { console.log('transcribe-parakeet-chunks: ffmpeg not installed here, chunking not exercised'); return; }
+ const wav = path.join(dir, 'long.wav');
+ execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-f', 'lavfi', '-i', 'anullsrc=r=16000:cl=mono', '-t', '70', '-c:a', 'pcm_s16le', wav]);
+ const r = await t.transcribeWavDetailed(wav, { offsetSec: 100, timeoutMs: 60000 });
+ assert.ok(r.ok, r.error);
+ assert.deepStrictEqual(r.segments.map((s) => s.start), [101, 131, 161], 'each 30 s piece shifted by its start');
+ assert.strictEqual(fs.readFileSync(calls, 'utf8').trim().split('\n').length, 3, 'three pieces, three runs');
+ const short = path.join(dir, 'short.wav');
+ execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-f', 'lavfi', '-i', 'anullsrc=r=16000:cl=mono', '-t', '10', '-c:a', 'pcm_s16le', short]);
+ const s = await t.transcribeWavDetailed(short, { offsetSec: 0 });
+ assert.deepStrictEqual(s.segments.map((x) => [x.start, x.text]), [[1, 'Word.']], 'short audio: one run, no split');
+ assert.ok(!fs.readdirSync(os.tmpdir()).some((n) => n.startsWith('openvibe-pk-') && fs.existsSync(path.join(os.tmpdir(), n, 'p0000.wav'))), 'pieces are cleaned up');
+ console.log('transcribe-parakeet-chunks: ok');
+})().catch((e) => { console.error(e); process.exit(1); }).finally(() => fs.rmSync(dir, { recursive: true, force: true }));