openvibers/openvibe.live Commits

Audio events: one bad segment was taking the feature down for good

OP OpenVibers committed 20 Aug 2026 · 00063e68e9 · 3 files, +137 −17

Production log: "[AI-Timeline] audio-events disabled: Invalid typed array
length: -22". The model was fine. ffmpeg had written a zero-byte WAV for a
segment with no decodable audio; the RIFF walk found no data chunk, fell back
to assuming a 44-byte header, and worked out a sample count of -22, which
new Float32Array refuses. That throw landed in the catch meant for a broken
model file, so _loadFailed was set and every remaining segment on the box was
skipped — blaming the model, which had loaded correctly.

The sample count is clamped, and loading the model is now the only failure that
can disable anything. A segment that cannot be read is skipped by name.

The WHIP cleanup test also aborted about one run in three inside
RemoveEnvironmentCleanupHook: building a full schema next to mediasoup's native
addon was more than the test needed. It stubs the single stream lookup that
sent it to the database at all.

Co-Authored-By: Claude Opus 5 <[email protected]>

Changes

ddiff --git a/server/ai/audio-events.js b/server/ai/audio-events.js
iindex 37883ea9..e66013af 100644
---- a/server/ai/audio-events.js
++++ b/server/ai/audio-events.js
@@@ -94,7 +94,10 @@ function _wavToFloat32(wavPath) {
off += 8 + size + (size % 2);
}
if (dataOff < 0) { dataOff = 44; dataLen = buf.length - 44; }
-- const n = Math.floor(Math.min(dataLen, buf.length - dataOff) / 2);
++ // A truncated or header-only file — ffmpeg writes one when a segment has no decodable
++ // audio — leaves nothing after the header, and the sample count below goes negative.
++ const n = Math.max(0, Math.floor(Math.min(dataLen, buf.length - dataOff) / 2));
++ if (n === 0) return new Float32Array(0);
const out = new Float32Array(n);
for (let i = 0; i < n; i++) out[i] = buf.readInt16LE(dataOff + i * 2) / 32768;
return out;
@@@ -109,22 +112,33 @@ function _wavToFloat32(wavPath) {
*/
async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE } = {}) {
if (!available()) return [];
-- let scores, frames, classes, labels;
++
++ // Loading the model is the only failure worth disabling the feature for. Anything to
++ // do with ONE segment must not: a single unreadable WAV used to set _loadFailed and
++ // take audio events down for the rest of the process, with a message blaming the
++ // model — which was fine all along.
++ let ort, session, labels;
++ try {
++ ort = require('onnxruntime-node');
++ session = await _session_();
++ labels = _loadLabels();
++ } catch (e) {
++ _loadFailed = true;
++ console.warn('[AI-Timeline] audio-events disabled — model failed to load:', e.message);
++ return [];
++ }
++
++ let scores, frames, classes;
try {
-- const ort = require('onnxruntime-node');
const wave = _wavToFloat32(wavPath);
if (!wave.length) return [];
-- const session = await _session_();
-- labels = _loadLabels();
const res = await session.run({ waveform: new ort.Tensor('float32', wave, [wave.length]) });
const out = res.output_0 || res[session.outputNames[0]];
if (!out) return [];
[frames, classes] = out.dims;
scores = out.data;
} catch (e) {
-- // A broken model file should disable the feature, not retry forever on every segment.
-- _loadFailed = true;
-- console.warn('[AI-Timeline] audio-events disabled:', e.message);
++ console.warn(`[AI-Timeline] audio-events skipped ${path.basename(wavPath)}:`, e.message);
return [];
}
@@@ -166,4 +180,6 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
.sort((a, b) => a.start - b.start);
}
--module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH };
++// _wavToFloat32 is exported for tests: it is the part that has to survive whatever
++// ffmpeg leaves on disk, and it cannot be reached through detect() without the model.
++module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32 };
ddiff --git a/test/audio-events-bad-segment.test.js b/test/audio-events-bad-segment.test.js
nnew file mode 100644
iindex 00000000..3b508f93
---- /dev/null
++++ b/test/audio-events-bad-segment.test.js
@@@ -0,0 +1,102 @@
++/**
++ * One unreadable segment must not take audio-event detection down for good.
++ *
++ * ffmpeg writes a zero-byte WAV when a segment has no decodable audio. The RIFF walk
++ * found no data chunk, fell back to assuming a 44-byte header, and computed a NEGATIVE
++ * sample count — so `new Float32Array(-22)` threw. That throw landed in a catch whose job
++ * was to disable the feature when the MODEL is broken, so a single bad segment set
++ * _loadFailed and every later segment on the box was skipped, blaming a model that was
++ * fine. Production ran this way: "[AI-Timeline] audio-events disabled: Invalid typed
++ * array length: -22".
++ */
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++
++const audioEvents = require('../server/ai/audio-events');
++const src = fs.readFileSync(path.join(__dirname, '../server/ai/audio-events.js'), 'utf8');
++
++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-audio-events-'));
++const write = (name, buf) => { const p = path.join(tmp, name); fs.writeFileSync(p, buf); return p; };
++
++// A RIFF/WAVE header with no data chunk at all, and one that stops mid-header.
++const headerOnly = Buffer.alloc(12);
++headerOnly.write('RIFF', 0, 'ascii');
++headerOnly.writeUInt32LE(4, 4);
++headerOnly.write('WAVE', 8, 'ascii');
++
++(async () => {
++ // ── A: the shapes ffmpeg actually produces are handled, not thrown on ──────────────
++ const bad = {
++ 'empty.wav': Buffer.alloc(0),
++ 'header-only.wav': headerOnly,
++ 'truncated.wav': headerOnly.subarray(0, 9),
++ };
++ // Straight at the parser: detect() short-circuits when the model file is absent, so
++ // going through it would assert nothing on a machine without one.
++ for (const [name, buf] of Object.entries(bad)) {
++ const samples = audioEvents._wavToFloat32(write(name, buf));
++ assert.strictEqual(samples.length, 0, `${name} must read as zero samples, not throw`);
++ }
++ console.log('OK A: empty, header-only and truncated WAVs read as zero samples');
++
++ // ── A2: a well-formed file still reads correctly ──────────────────────────────────
++ {
++ const samples = 3;
++ const wav = Buffer.alloc(44 + samples * 2);
++ wav.write('RIFF', 0, 'ascii');
++ wav.writeUInt32LE(36 + samples * 2, 4);
++ wav.write('WAVEfmt ', 8, 'ascii');
++ wav.writeUInt32LE(16, 16);
++ wav.write('data', 36, 'ascii');
++ wav.writeUInt32LE(samples * 2, 40);
++ for (const [i, v] of [0, 32767, -32768].entries()) wav.writeInt16LE(v, 44 + i * 2);
++ const out = audioEvents._wavToFloat32(write('good.wav', wav));
++ assert.strictEqual(out.length, samples, 'a real data chunk must yield its samples');
++ assert.strictEqual(out[0], 0);
++ assert.ok(out[1] > 0.99 && out[1] <= 1, 'full-scale positive maps into [0,1]');
++ assert.strictEqual(out[2], -1, 'full-scale negative maps to -1');
++ console.log('OK A2: a well-formed WAV still reads its samples correctly');
++ }
++
++ // ── A3: and detect() itself stays quiet on a bad file ─────────────────────────────
++ assert.deepStrictEqual(await audioEvents.detect(path.join(tmp, 'empty.wav'), { offsetSec: 0 }), [],
++ 'detect() must return no events for an unreadable segment');
++ console.log('OK A3: detect() returns no events for an unreadable segment');
++
++ // ── B: and the feature is still alive afterwards ──────────────────────────────────
++ // available() is the gate every later segment passes through. If a bad file flipped
++ // _loadFailed, this is false and audio events are gone until the process restarts.
++ const modelPresent = fs.existsSync(audioEvents.MODEL_PATH) && fs.existsSync(audioEvents.CLASSMAP_PATH);
++ if (modelPresent) {
++ assert.strictEqual(audioEvents.available(), true,
++ 'a bad segment must not disable detection for every segment that follows');
++ console.log('OK B: detection still available after a bad segment');
++ } else {
++ console.log('OK B: skipped — no local model to check availability against');
++ }
++
++ // ── C: only a model load failure is allowed to be permanent ───────────────────────
++ const loadFail = src.indexOf('_loadFailed = true');
++ assert.ok(loadFail > 0, '_loadFailed must still exist');
++ assert.strictEqual(src.split('_loadFailed = true').length - 1, 1,
++ 'exactly one place may disable the feature permanently');
++ const guard = src.slice(src.lastIndexOf('try {', loadFail), loadFail);
++ assert.ok(/_session_\(\)/.test(guard), 'the permanent failure must come from loading the model');
++ assert.ok(!/_wavToFloat32/.test(guard) && !/session\.run/.test(guard),
++ 'reading a segment or running inference on it must not be able to disable the feature');
++ console.log('OK C: only model loading can disable detection permanently');
++
++ // ── D: a negative sample count cannot reach the allocator ─────────────────────────
++ assert.ok(/Math\.max\(0, Math\.floor\(/.test(src),
++ 'the sample count must be clamped before it reaches new Float32Array');
++ console.log('OK D: sample count is clamped at zero');
++
++ fs.rmSync(tmp, { recursive: true, force: true });
++ console.log('✅ audio events bad segment test passed');
++})().catch((e) => {
++ fs.rmSync(tmp, { recursive: true, force: true });
++ console.error('FAIL', e.message);
++ process.exit(1);
++});
ddiff --git a/test/whip-handler-cleanup.test.js b/test/whip-handler-cleanup.test.js
iindex f3784150..5d5ffc60 100644
---- a/test/whip-handler-cleanup.test.js
++++ b/test/whip-handler-cleanup.test.js
@@@ -3,13 +3,16 @@ const fs = require('fs');
const os = require('os');
const path = require('path');
--// cleanupSession looks the stream up to decide what to tear down, so requiring the WHIP
--// handler pulls in the database. Point it at a throwaway file with a real schema BEFORE
--// that require: the test used to open whatever data/live.db happened to be in the working
--// tree, which passed on a developed checkout and threw "no such table: streams" on a
--// fresh one — a property of the machine, not of the code under test.
++// After tearing the session down, cleanupSession asks whether the stream is still live —
++// the one thing here that touches the database. It used to reach whatever data/live.db
++// sat in the working tree, which passed on a developed checkout and threw "no such table:
++// streams" on a fresh one: a property of the machine, not of the code under test.
++//
++// Stub that single lookup instead of building a schema. Answering "no such stream" ends
++// endActiveWhipStream on its first line, which is the path this test wants anyway — what
++// it asserts is that the producer, transport and session are released.
process.env.DB_PATH = path.join(os.tmpdir(), `ov-whip-cleanup-${process.pid}.db`);
--require('../server/db/database').initDb();
++require('../server/db/database').getStreamById = () => null;
const webrtcSFU = require('../server/streaming/webrtc-sfu');
const whipHandler = require('../server/streaming/whip-handler');
@@@ -48,7 +51,6 @@ assert.strictEqual(room.transports.has('whip-test-transport-1'), false, 'transpo
assert.strictEqual(producerClosed.value, true, 'producer should be closed');
assert.strictEqual(transportClosed.value, true, 'transport should be closed');
--try { fs.unlinkSync(process.env.DB_PATH); } catch { /* */ }
--for (const ext of ['-wal', '-shm']) { try { fs.unlinkSync(process.env.DB_PATH + ext); } catch { /* */ } }
++for (const ext of ['', '-wal', '-shm']) { try { fs.unlinkSync(process.env.DB_PATH + ext); } catch { /* */ } }
console.log('✅ WHIP cleanup session regression test passed');