AI-Hear: read the timeline instead of double-capturing

OP OpenVibers committed 18 Aug 2026 · 94f734973c · 1 file, +17 −1

With continuous capture running, stream-memory-job's own 12s grab opened a
SECOND PlainRTP consumer on the same audio producer. The two collided and the
one-shot path started writing digital silence — capture level mean=-91.0dB
max=-91.0dB, where a healthy capture reads about -32dB mean / -4dB max.

When ai_timeline_enabled is on, take the window's speech straight from
stream_timeline_events instead. That removes the second consumer, removes the
duplicated whisper pass, and gives the memory more speech than a 12s sample
ever did. Falls back to the old capture when the timeline is off.

Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Xw6Sf5zzSALw8m93XNu5JE

Changes

ddiff --git a/server/ai/stream-memory-job.js b/server/ai/stream-memory-job.js
iindex 6dcb1653..cb8f8509 100644
---- a/server/ai/stream-memory-job.js
++++ b/server/ai/stream-memory-job.js
@@@ -45,7 +45,23 @@ async function _analyzeOne(stream) {
      let heard = '';
      let heardSegments = null;
      try {
--        if (ai.transcriptionEnabled && ai.transcriptionEnabled()) {
++        // When the continuous timeline is running it already transcribes ALL of the audio,
++        // so grabbing another 12s chunk here would duplicate the work and — worse — open a
++        // second PlainRTP consumer on the same producer, which is what made this path start
++        // capturing digital silence (-91dB) once continuous capture was switched on.
++        // Read the timeline instead: strictly more speech, and free.
++        const _timeline = (() => { try { return require('./timeline-job').timelineEnabled(); } catch { return false; } })();
++        if (_timeline) {
++            try {
++                const from = Math.max(0, offset - ai.captureIntervalSec());
++                const rows = db.getTimeline(stream.id, { kind: 'speech', from, to: offset, limit: 200 }) || [];
++                heard = rows.map(r => String(r.text || '').trim()).filter(Boolean).join(' ');
++                heardSegments = rows.length
++                    ? rows.map(r => ({ start: r.start_sec, end: r.end_sec, text: r.text }))
++                    : null;
++            } catch { /* fall through to empty */ }
++            console.log(`[AI-Hear] stream ${stream.id}: from timeline transcript=${JSON.stringify((heard || '').slice(0, 100))}`);
++        } else if (ai.transcriptionEnabled && ai.transcriptionEnabled()) {
              const audio = require('./stream-audio').captureAudioChunk ? await require('./stream-audio').captureAudioChunk(stream, CHUNK_SEC) : null;
              if (audio) {
                  const tx = await require('./transcribe').transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC) });

diff --git a/server/ai/stream-memory-job.js b/server/ai/stream-memory-job.js

index 6dcb1653..cb8f8509 100644

--- a/server/ai/stream-memory-job.js

+++ b/server/ai/stream-memory-job.js

@@ -45,7 +45,23 @@ async function _analyzeOne(stream) {

let heard = '';

let heardSegments = null;

try {

- if (ai.transcriptionEnabled && ai.transcriptionEnabled()) {

+ // When the continuous timeline is running it already transcribes ALL of the audio,

+ // so grabbing another 12s chunk here would duplicate the work and — worse — open a

+ // second PlainRTP consumer on the same producer, which is what made this path start

+ // capturing digital silence (-91dB) once continuous capture was switched on.

+ // Read the timeline instead: strictly more speech, and free.

+ const _timeline = (() => { try { return require('./timeline-job').timelineEnabled(); } catch { return false; } })();

+ if (_timeline) {

+ try {

+ const from = Math.max(0, offset - ai.captureIntervalSec());

+ const rows = db.getTimeline(stream.id, { kind: 'speech', from, to: offset, limit: 200 }) || [];

+ heard = rows.map(r => String(r.text || '').trim()).filter(Boolean).join(' ');

+ heardSegments = rows.length

+ ? rows.map(r => ({ start: r.start_sec, end: r.end_sec, text: r.text }))

+ : null;

+ } catch { /* fall through to empty */ }

+ console.log(`[AI-Hear] stream ${stream.id}: from timeline transcript=${JSON.stringify((heard || '').slice(0, 100))}`);

+ } else if (ai.transcriptionEnabled && ai.transcriptionEnabled()) {

const audio = require('./stream-audio').captureAudioChunk ? await require('./stream-audio').captureAudioChunk(stream, CHUNK_SEC) : null;

if (audio) {

const tx = await require('./transcribe').transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC) });