openvibers/openvibe.live Commits

Timeline: cut transcription lag from ~40s to ~12s

OP OpenVibers committed 18 Aug 2026 · 289a6eb954 · 2 files, +28 −10

Segment length was the dominant term, not decode time. Audio at the start of a
segment cannot be decoded until the whole segment is written, so 30s segments
meant ~30s of lag before whisper even saw the audio, plus up to 10s of poll
interval on top.

  segment fill   30s -> 10s
  poll interval  10s -> 2s
  per-tick cap    2  -> 6   (must keep up with 3x the segment rate)

Worst case goes ~30+10+decode to ~10+2+decode.

The tradeoff is boundary splitting: a phrase spanning a segment edge is
transcribed as two fragments. It is mild here because only ~13% of this audio
is speech, so most boundaries land in silence. AI_HEAR_SEGMENT_SEC tunes it —
raise it for cleaner sentences, lower it for less lag.

Also log actual lag per segment, measured from the end of the audio it covers,
so this is tuned from measurement rather than arithmetic.

Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Xw6Sf5zzSALw8m93XNu5JE

Changes

ddiff --git a/server/ai/stream-audio.js b/server/ai/stream-audio.js
iindex dc69e1cf..f4bb3534 100644
---- a/server/ai/stream-audio.js
++++ b/server/ai/stream-audio.js
@@@ -216,7 +216,13 @@ async function captureAudioChunk(stream, seconds = 12) {
// Instead run ONE long-lived ffmpeg per stream that writes rolling WAV segments. Latency
// is explicitly not a concern here, so segments are consumed behind live at low priority.
// Cost is bounded by VAD: only the ~13% of audio containing a voice reaches the decoder.
--const SEGMENT_SEC = Math.max(10, Math.min(120, parseInt(process.env.AI_HEAR_SEGMENT_SEC, 10) || 30));
++// Segment length is the dominant term in transcription lag: audio at the START of a
++// segment cannot be decoded until the whole segment has been written, so a 30s segment
++// meant ~30s of latency before whisper even saw it. Shorter segments trade a little
++// accuracy — a phrase spanning a boundary gets split in two — for much lower lag.
++// 10s is the balance point; most boundaries land in silence anyway, since only ~13% of
++// this audio is speech. Tune with AI_HEAR_SEGMENT_SEC.
++const SEGMENT_SEC = Math.max(5, Math.min(120, parseInt(process.env.AI_HEAR_SEGMENT_SEC, 10) || 10));
const SPOOL_ROOT = path.join(__dirname, '../../data/asr');
// Hard cap on spooled audio per stream. The box has ~34GB free on a filesystem that
// openvibe.media is actively growing into, so an unbounded spool is not an option.
@@@ -315,12 +321,19 @@ function pendingSegments(streamId) {
for (let i = 0; i < overflow; i++) {
try { fs.unlinkSync(path.join(dir, ready[i])); } catch { /* */ }
}
-- return ready.slice(overflow).map(name => ({
-- name,
-- path: path.join(dir, name),
-- index: parseInt(name.slice(4, 10), 10),
-- offsetSec: parseInt(name.slice(4, 10), 10) * SEGMENT_SEC,
-- }));
++ const startedAt = (_capturing.get(streamId) || {}).startedAt || null;
++ return ready.slice(overflow).map(name => {
++ const index = parseInt(name.slice(4, 10), 10);
++ return {
++ name,
++ path: path.join(dir, name),
++ index,
++ offsetSec: index * SEGMENT_SEC,
++ // Wall-clock instant this segment's audio ENDED, so the job can report how far
++ // behind live the transcript actually is.
++ endedAtMs: startedAt ? startedAt + (index + 1) * SEGMENT_SEC * 1000 : null,
++ };
++ });
}
function discardSegment(streamId, name) {
ddiff --git a/server/ai/timeline-job.js b/server/ai/timeline-job.js
iindex eccd8f24..d77ce465 100644
---- a/server/ai/timeline-job.js
++++ b/server/ai/timeline-job.js
@@@ -27,10 +27,12 @@ const db = require('../db/database');
const ai = require('./ai-analysis');
const audio = require('./stream-audio');
--const POLL_MS = 10000;
++// Poll fast: with 10s segments a 10s poll interval would add as much lag again as the
++// segment itself. The tick is cheap when there is nothing to do (a readdir).
++const POLL_MS = 2000;
// Process at most this many segments per tick per stream. Keeps a backlog from monopolising
// the CPU on a 4-core box that is also running live x264 encoding.
--const MAX_SEGMENTS_PER_TICK = 2;
++const MAX_SEGMENTS_PER_TICK = 6;
let _busy = false;
const _known = new Set(); // stream ids we have started capture for
@@@ -98,7 +100,10 @@ async function _processSegment(stream, seg) {
const heard = events.filter(e => e.kind === 'sound').length;
if (spoken || heard) {
const preview = events.find(e => e.kind === 'speech');
-- console.log(`[AI-Timeline] stream ${stream.id} +${seg.offsetSec}s: ${spoken} speech, ${heard} sound` +
++ // How far behind live this transcript landed, measured from the END of the audio
++ // it covers. This is the number to watch when tuning AI_HEAR_SEGMENT_SEC / POLL_MS.
++ const lag = seg.endedAtMs ? `${Math.round((Date.now() - seg.endedAtMs) / 1000)}s behind live` : 'lag unknown';
++ console.log(`[AI-Timeline] stream ${stream.id} +${seg.offsetSec}s (${lag}): ${spoken} speech, ${heard} sound` +
(preview ? ` — ${JSON.stringify(String(preview.text).slice(0, 80))}` : ''));
}
return events.length;