openvibers/openvibe.live Commits
Timeline: cut transcription lag from ~40s to ~12s
Changes
@@@ -216,7 +216,13 @@ async function captureAudioChunk(stream, seconds = 12) {// Instead run ONE long-lived ffmpeg per stream that writes rolling WAV segments. Latency// is explicitly not a concern here, so segments are consumed behind live at low priority.// Cost is bounded by VAD: only the ~13% of audio containing a voice reaches the decoder.--const SEGMENT_SEC = Math.max(10, Math.min(120, parseInt(process.env.AI_HEAR_SEGMENT_SEC, 10) || 30));++// Segment length is the dominant term in transcription lag: audio at the START of a++// segment cannot be decoded until the whole segment has been written, so a 30s segment++// meant ~30s of latency before whisper even saw it. Shorter segments trade a little++// accuracy — a phrase spanning a boundary gets split in two — for much lower lag.++// 10s is the balance point; most boundaries land in silence anyway, since only ~13% of++// this audio is speech. Tune with AI_HEAR_SEGMENT_SEC.++const SEGMENT_SEC = Math.max(5, Math.min(120, parseInt(process.env.AI_HEAR_SEGMENT_SEC, 10) || 10));const SPOOL_ROOT = path.join(__dirname, '../../data/asr');// Hard cap on spooled audio per stream. The box has ~34GB free on a filesystem that// openvibe.media is actively growing into, so an unbounded spool is not an option.@@@ -315,12 +321,19 @@ function pendingSegments(streamId) {for (let i = 0; i < overflow; i++) {try { fs.unlinkSync(path.join(dir, ready[i])); } catch { /* */ }}-- return ready.slice(overflow).map(name => ({-- name,-- path: path.join(dir, name),-- index: parseInt(name.slice(4, 10), 10),-- offsetSec: parseInt(name.slice(4, 10), 10) * SEGMENT_SEC,-- }));++ const startedAt = (_capturing.get(streamId) || {}).startedAt || null;++ return ready.slice(overflow).map(name => {++ const index = parseInt(name.slice(4, 10), 10);++ return {++ name,++ path: path.join(dir, name),++ index,++ offsetSec: index * SEGMENT_SEC,++ // Wall-clock instant this segment's audio ENDED, so the job can report how far++ // behind live the transcript actually is.++ endedAtMs: startedAt ? startedAt + (index + 1) * SEGMENT_SEC * 1000 : null,++ };++ });}function discardSegment(streamId, name) {@@@ -27,10 +27,12 @@ const db = require('../db/database');const ai = require('./ai-analysis');const audio = require('./stream-audio');--const POLL_MS = 10000;++// Poll fast: with 10s segments a 10s poll interval would add as much lag again as the++// segment itself. The tick is cheap when there is nothing to do (a readdir).++const POLL_MS = 2000;// Process at most this many segments per tick per stream. Keeps a backlog from monopolising// the CPU on a 4-core box that is also running live x264 encoding.--const MAX_SEGMENTS_PER_TICK = 2;++const MAX_SEGMENTS_PER_TICK = 6;let _busy = false;const _known = new Set(); // stream ids we have started capture for@@@ -98,7 +100,10 @@ async function _processSegment(stream, seg) {const heard = events.filter(e => e.kind === 'sound').length;if (spoken || heard) {const preview = events.find(e => e.kind === 'speech');-- console.log(`[AI-Timeline] stream ${stream.id} +${seg.offsetSec}s: ${spoken} speech, ${heard} sound` +++ // How far behind live this transcript landed, measured from the END of the audio++ // it covers. This is the number to watch when tuning AI_HEAR_SEGMENT_SEC / POLL_MS.++ const lag = seg.endedAtMs ? `${Math.round((Date.now() - seg.endedAtMs) / 1000)}s behind live` : 'lag unknown';++ console.log(`[AI-Timeline] stream ${stream.id} +${seg.offsetSec}s (${lag}): ${spoken} speech, ${heard} sound` +(preview ? ` — ${JSON.stringify(String(preview.text).slice(0, 80))}` : ''));}return events.length;
@@ -216,7 +216,13 @@ async function captureAudioChunk(stream, seconds = 12) {
@@ -216,7 +216,13 @@ async function captureAudioChunk(stream, seconds = 12) {
// Instead run ONE long-lived ffmpeg per stream that writes rolling WAV segments. Latency
// Instead run ONE long-lived ffmpeg per stream that writes rolling WAV segments. Latency
// is explicitly not a concern here, so segments are consumed behind live at low priority.
// is explicitly not a concern here, so segments are consumed behind live at low priority.
// Cost is bounded by VAD: only the ~13% of audio containing a voice reaches the decoder.
// Cost is bounded by VAD: only the ~13% of audio containing a voice reaches the decoder.
-const SEGMENT_SEC = Math.max(10, Math.min(120, parseInt(process.env.AI_HEAR_SEGMENT_SEC, 10) || 30));
+// Segment length is the dominant term in transcription lag: audio at the START of a
+// segment cannot be decoded until the whole segment has been written, so a 30s segment
+// meant ~30s of latency before whisper even saw it. Shorter segments trade a little
+// accuracy — a phrase spanning a boundary gets split in two — for much lower lag.
+// 10s is the balance point; most boundaries land in silence anyway, since only ~13% of
+// this audio is speech. Tune with AI_HEAR_SEGMENT_SEC.
+const SEGMENT_SEC = Math.max(5, Math.min(120, parseInt(process.env.AI_HEAR_SEGMENT_SEC, 10) || 10));
const SPOOL_ROOT = path.join(__dirname, '../../data/asr');
const SPOOL_ROOT = path.join(__dirname, '../../data/asr');
// Hard cap on spooled audio per stream. The box has ~34GB free on a filesystem that
// Hard cap on spooled audio per stream. The box has ~34GB free on a filesystem that
// openvibe.media is actively growing into, so an unbounded spool is not an option.
// openvibe.media is actively growing into, so an unbounded spool is not an option.
@@ -315,12 +321,19 @@ function pendingSegments(streamId) {
@@ -315,12 +321,19 @@ function pendingSegments(streamId) {
for (let i = 0; i < overflow; i++) {
for (let i = 0; i < overflow; i++) {
try { fs.unlinkSync(path.join(dir, ready[i])); } catch { /* */ }
try { fs.unlinkSync(path.join(dir, ready[i])); } catch { /* */ }
}
}
- return ready.slice(overflow).map(name => ({
- name,
- path: path.join(dir, name),
- index: parseInt(name.slice(4, 10), 10),
- offsetSec: parseInt(name.slice(4, 10), 10) * SEGMENT_SEC,
- }));
+ const startedAt = (_capturing.get(streamId) || {}).startedAt || null;
+ return ready.slice(overflow).map(name => {
+ const index = parseInt(name.slice(4, 10), 10);
+ return {
+ name,
+ path: path.join(dir, name),
+ index,
+ offsetSec: index * SEGMENT_SEC,
+ // Wall-clock instant this segment's audio ENDED, so the job can report how far
+ // behind live the transcript actually is.
+ endedAtMs: startedAt ? startedAt + (index + 1) * SEGMENT_SEC * 1000 : null,
+ };
+ });
}
}
function discardSegment(streamId, name) {
function discardSegment(streamId, name) {
@@ -27,10 +27,12 @@ const db = require('../db/database');
@@ -27,10 +27,12 @@ const db = require('../db/database');
const ai = require('./ai-analysis');
const ai = require('./ai-analysis');
const audio = require('./stream-audio');
const audio = require('./stream-audio');
-const POLL_MS = 10000;
+// Poll fast: with 10s segments a 10s poll interval would add as much lag again as the
+// segment itself. The tick is cheap when there is nothing to do (a readdir).
+const POLL_MS = 2000;
// Process at most this many segments per tick per stream. Keeps a backlog from monopolising
// Process at most this many segments per tick per stream. Keeps a backlog from monopolising
// the CPU on a 4-core box that is also running live x264 encoding.
// the CPU on a 4-core box that is also running live x264 encoding.
-const MAX_SEGMENTS_PER_TICK = 2;
+const MAX_SEGMENTS_PER_TICK = 6;
let _busy = false;
let _busy = false;
const _known = new Set(); // stream ids we have started capture for
const _known = new Set(); // stream ids we have started capture for
@@ -98,7 +100,10 @@ async function _processSegment(stream, seg) {
@@ -98,7 +100,10 @@ async function _processSegment(stream, seg) {
const heard = events.filter(e => e.kind === 'sound').length;
const heard = events.filter(e => e.kind === 'sound').length;
if (spoken || heard) {
if (spoken || heard) {
const preview = events.find(e => e.kind === 'speech');
const preview = events.find(e => e.kind === 'speech');
- console.log(`[AI-Timeline] stream ${stream.id} +${seg.offsetSec}s: ${spoken} speech, ${heard} sound` +
+ // How far behind live this transcript landed, measured from the END of the audio
+ // it covers. This is the number to watch when tuning AI_HEAR_SEGMENT_SEC / POLL_MS.
+ const lag = seg.endedAtMs ? `${Math.round((Date.now() - seg.endedAtMs) / 1000)}s behind live` : 'lag unknown';
+ console.log(`[AI-Timeline] stream ${stream.id} +${seg.offsetSec}s (${lag}): ${spoken} speech, ${heard} sound` +
(preview ? ` — ${JSON.stringify(String(preview.text).slice(0, 80))}` : ''));
(preview ? ` — ${JSON.stringify(String(preview.text).slice(0, 80))}` : ''));
}
}
return events.length;
return events.length;