openvibers/openvibe.live Commits
Audio events: one bad segment was taking the feature down for good
Changes
@@@ -94,7 +94,10 @@ function _wavToFloat32(wavPath) {off += 8 + size + (size % 2);}if (dataOff < 0) { dataOff = 44; dataLen = buf.length - 44; }-- const n = Math.floor(Math.min(dataLen, buf.length - dataOff) / 2);++ // A truncated or header-only file — ffmpeg writes one when a segment has no decodable++ // audio — leaves nothing after the header, and the sample count below goes negative.++ const n = Math.max(0, Math.floor(Math.min(dataLen, buf.length - dataOff) / 2));++ if (n === 0) return new Float32Array(0);const out = new Float32Array(n);for (let i = 0; i < n; i++) out[i] = buf.readInt16LE(dataOff + i * 2) / 32768;return out;@@@ -109,22 +112,33 @@ function _wavToFloat32(wavPath) {*/async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE } = {}) {if (!available()) return [];-- let scores, frames, classes, labels;++++ // Loading the model is the only failure worth disabling the feature for. Anything to++ // do with ONE segment must not: a single unreadable WAV used to set _loadFailed and++ // take audio events down for the rest of the process, with a message blaming the++ // model — which was fine all along.++ let ort, session, labels;++ try {++ ort = require('onnxruntime-node');++ session = await _session_();++ labels = _loadLabels();++ } catch (e) {++ _loadFailed = true;++ console.warn('[AI-Timeline] audio-events disabled — model failed to load:', e.message);++ return [];++ }++++ let scores, frames, classes;try {-- const ort = require('onnxruntime-node');const wave = _wavToFloat32(wavPath);if (!wave.length) return [];-- const session = await _session_();-- labels = _loadLabels();const res = await session.run({ waveform: new ort.Tensor('float32', wave, [wave.length]) });const out = res.output_0 || res[session.outputNames[0]];if (!out) return [];[frames, classes] = out.dims;scores = out.data;} catch (e) {-- // A broken model file should disable the feature, not retry forever on every segment.-- _loadFailed = true;-- console.warn('[AI-Timeline] audio-events disabled:', e.message);++ console.warn(`[AI-Timeline] audio-events skipped ${path.basename(wavPath)}:`, e.message);return [];}@@@ -166,4 +180,6 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }.sort((a, b) => a.start - b.start);}--module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH };++// _wavToFloat32 is exported for tests: it is the part that has to survive whatever++// ffmpeg leaves on disk, and it cannot be reached through detect() without the model.++module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32 };nnew file mode 100644@@@ -0,0 +1,102 @@++/**++ * One unreadable segment must not take audio-event detection down for good.++ *++ * ffmpeg writes a zero-byte WAV when a segment has no decodable audio. The RIFF walk++ * found no data chunk, fell back to assuming a 44-byte header, and computed a NEGATIVE++ * sample count — so `new Float32Array(-22)` threw. That throw landed in a catch whose job++ * was to disable the feature when the MODEL is broken, so a single bad segment set++ * _loadFailed and every later segment on the box was skipped, blaming a model that was++ * fine. Production ran this way: "[AI-Timeline] audio-events disabled: Invalid typed++ * array length: -22".++ */++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++++const audioEvents = require('../server/ai/audio-events');++const src = fs.readFileSync(path.join(__dirname, '../server/ai/audio-events.js'), 'utf8');++++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-audio-events-'));++const write = (name, buf) => { const p = path.join(tmp, name); fs.writeFileSync(p, buf); return p; };++++// A RIFF/WAVE header with no data chunk at all, and one that stops mid-header.++const headerOnly = Buffer.alloc(12);++headerOnly.write('RIFF', 0, 'ascii');++headerOnly.writeUInt32LE(4, 4);++headerOnly.write('WAVE', 8, 'ascii');++++(async () => {++ // ── A: the shapes ffmpeg actually produces are handled, not thrown on ──────────────++ const bad = {++ 'empty.wav': Buffer.alloc(0),++ 'header-only.wav': headerOnly,++ 'truncated.wav': headerOnly.subarray(0, 9),++ };++ // Straight at the parser: detect() short-circuits when the model file is absent, so++ // going through it would assert nothing on a machine without one.++ for (const [name, buf] of Object.entries(bad)) {++ const samples = audioEvents._wavToFloat32(write(name, buf));++ assert.strictEqual(samples.length, 0, `${name} must read as zero samples, not throw`);++ }++ console.log('OK A: empty, header-only and truncated WAVs read as zero samples');++++ // ── A2: a well-formed file still reads correctly ──────────────────────────────────++ {++ const samples = 3;++ const wav = Buffer.alloc(44 + samples * 2);++ wav.write('RIFF', 0, 'ascii');++ wav.writeUInt32LE(36 + samples * 2, 4);++ wav.write('WAVEfmt ', 8, 'ascii');++ wav.writeUInt32LE(16, 16);++ wav.write('data', 36, 'ascii');++ wav.writeUInt32LE(samples * 2, 40);++ for (const [i, v] of [0, 32767, -32768].entries()) wav.writeInt16LE(v, 44 + i * 2);++ const out = audioEvents._wavToFloat32(write('good.wav', wav));++ assert.strictEqual(out.length, samples, 'a real data chunk must yield its samples');++ assert.strictEqual(out[0], 0);++ assert.ok(out[1] > 0.99 && out[1] <= 1, 'full-scale positive maps into [0,1]');++ assert.strictEqual(out[2], -1, 'full-scale negative maps to -1');++ console.log('OK A2: a well-formed WAV still reads its samples correctly');++ }++++ // ── A3: and detect() itself stays quiet on a bad file ─────────────────────────────++ assert.deepStrictEqual(await audioEvents.detect(path.join(tmp, 'empty.wav'), { offsetSec: 0 }), [],++ 'detect() must return no events for an unreadable segment');++ console.log('OK A3: detect() returns no events for an unreadable segment');++++ // ── B: and the feature is still alive afterwards ──────────────────────────────────++ // available() is the gate every later segment passes through. If a bad file flipped++ // _loadFailed, this is false and audio events are gone until the process restarts.++ const modelPresent = fs.existsSync(audioEvents.MODEL_PATH) && fs.existsSync(audioEvents.CLASSMAP_PATH);++ if (modelPresent) {++ assert.strictEqual(audioEvents.available(), true,++ 'a bad segment must not disable detection for every segment that follows');++ console.log('OK B: detection still available after a bad segment');++ } else {++ console.log('OK B: skipped — no local model to check availability against');++ }++++ // ── C: only a model load failure is allowed to be permanent ───────────────────────++ const loadFail = src.indexOf('_loadFailed = true');++ assert.ok(loadFail > 0, '_loadFailed must still exist');++ assert.strictEqual(src.split('_loadFailed = true').length - 1, 1,++ 'exactly one place may disable the feature permanently');++ const guard = src.slice(src.lastIndexOf('try {', loadFail), loadFail);++ assert.ok(/_session_\(\)/.test(guard), 'the permanent failure must come from loading the model');++ assert.ok(!/_wavToFloat32/.test(guard) && !/session\.run/.test(guard),++ 'reading a segment or running inference on it must not be able to disable the feature');++ console.log('OK C: only model loading can disable detection permanently');++++ // ── D: a negative sample count cannot reach the allocator ─────────────────────────++ assert.ok(/Math\.max\(0, Math\.floor\(/.test(src),++ 'the sample count must be clamped before it reaches new Float32Array');++ console.log('OK D: sample count is clamped at zero');++++ fs.rmSync(tmp, { recursive: true, force: true });++ console.log('✅ audio events bad segment test passed');++})().catch((e) => {++ fs.rmSync(tmp, { recursive: true, force: true });++ console.error('FAIL', e.message);++ process.exit(1);++});@@@ -3,13 +3,16 @@ const fs = require('fs');const os = require('os');const path = require('path');--// cleanupSession looks the stream up to decide what to tear down, so requiring the WHIP--// handler pulls in the database. Point it at a throwaway file with a real schema BEFORE--// that require: the test used to open whatever data/live.db happened to be in the working--// tree, which passed on a developed checkout and threw "no such table: streams" on a--// fresh one — a property of the machine, not of the code under test.++// After tearing the session down, cleanupSession asks whether the stream is still live —++// the one thing here that touches the database. It used to reach whatever data/live.db++// sat in the working tree, which passed on a developed checkout and threw "no such table:++// streams" on a fresh one: a property of the machine, not of the code under test.++//++// Stub that single lookup instead of building a schema. Answering "no such stream" ends++// endActiveWhipStream on its first line, which is the path this test wants anyway — what++// it asserts is that the producer, transport and session are released.process.env.DB_PATH = path.join(os.tmpdir(), `ov-whip-cleanup-${process.pid}.db`);--require('../server/db/database').initDb();++require('../server/db/database').getStreamById = () => null;const webrtcSFU = require('../server/streaming/webrtc-sfu');const whipHandler = require('../server/streaming/whip-handler');@@@ -48,7 +51,6 @@ assert.strictEqual(room.transports.has('whip-test-transport-1'), false, 'transpoassert.strictEqual(producerClosed.value, true, 'producer should be closed');assert.strictEqual(transportClosed.value, true, 'transport should be closed');--try { fs.unlinkSync(process.env.DB_PATH); } catch { /* */ }--for (const ext of ['-wal', '-shm']) { try { fs.unlinkSync(process.env.DB_PATH + ext); } catch { /* */ } }++for (const ext of ['', '-wal', '-shm']) { try { fs.unlinkSync(process.env.DB_PATH + ext); } catch { /* */ } }console.log('✅ WHIP cleanup session regression test passed');
@@ -94,7 +94,10 @@ function _wavToFloat32(wavPath) {
@@ -94,7 +94,10 @@ function _wavToFloat32(wavPath) {
off += 8 + size + (size % 2);
off += 8 + size + (size % 2);
}
}
if (dataOff < 0) { dataOff = 44; dataLen = buf.length - 44; }
if (dataOff < 0) { dataOff = 44; dataLen = buf.length - 44; }
- const n = Math.floor(Math.min(dataLen, buf.length - dataOff) / 2);
+ // A truncated or header-only file — ffmpeg writes one when a segment has no decodable
+ // audio — leaves nothing after the header, and the sample count below goes negative.
+ const n = Math.max(0, Math.floor(Math.min(dataLen, buf.length - dataOff) / 2));
+ if (n === 0) return new Float32Array(0);
const out = new Float32Array(n);
const out = new Float32Array(n);
for (let i = 0; i < n; i++) out[i] = buf.readInt16LE(dataOff + i * 2) / 32768;
for (let i = 0; i < n; i++) out[i] = buf.readInt16LE(dataOff + i * 2) / 32768;
return out;
return out;
@@ -109,22 +112,33 @@ function _wavToFloat32(wavPath) {
@@ -109,22 +112,33 @@ function _wavToFloat32(wavPath) {
*/
*/
async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE } = {}) {
async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE } = {}) {
if (!available()) return [];
if (!available()) return [];
- let scores, frames, classes, labels;
+
+ // Loading the model is the only failure worth disabling the feature for. Anything to
+ // do with ONE segment must not: a single unreadable WAV used to set _loadFailed and
+ // take audio events down for the rest of the process, with a message blaming the
+ // model — which was fine all along.
+ let ort, session, labels;
+ try {
+ ort = require('onnxruntime-node');
+ session = await _session_();
+ labels = _loadLabels();
+ } catch (e) {
+ _loadFailed = true;
+ console.warn('[AI-Timeline] audio-events disabled — model failed to load:', e.message);
+ return [];
+ }
+
+ let scores, frames, classes;
try {
try {
- const ort = require('onnxruntime-node');
const wave = _wavToFloat32(wavPath);
const wave = _wavToFloat32(wavPath);
if (!wave.length) return [];
if (!wave.length) return [];
- const session = await _session_();
- labels = _loadLabels();
const res = await session.run({ waveform: new ort.Tensor('float32', wave, [wave.length]) });
const res = await session.run({ waveform: new ort.Tensor('float32', wave, [wave.length]) });
const out = res.output_0 || res[session.outputNames[0]];
const out = res.output_0 || res[session.outputNames[0]];
if (!out) return [];
if (!out) return [];
[frames, classes] = out.dims;
[frames, classes] = out.dims;
scores = out.data;
scores = out.data;
} catch (e) {
} catch (e) {
- // A broken model file should disable the feature, not retry forever on every segment.
- _loadFailed = true;
- console.warn('[AI-Timeline] audio-events disabled:', e.message);
+ console.warn(`[AI-Timeline] audio-events skipped ${path.basename(wavPath)}:`, e.message);
return [];
return [];
}
}
@@ -166,4 +180,6 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
@@ -166,4 +180,6 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
.sort((a, b) => a.start - b.start);
.sort((a, b) => a.start - b.start);
}
}
-module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH };
+// _wavToFloat32 is exported for tests: it is the part that has to survive whatever
+// ffmpeg leaves on disk, and it cannot be reached through detect() without the model.
+module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32 };
new file mode 100644
new file mode 100644
@@ -0,0 +1,102 @@
@@ -0,0 +1,102 @@
+/**
+ * One unreadable segment must not take audio-event detection down for good.
+ *
+ * ffmpeg writes a zero-byte WAV when a segment has no decodable audio. The RIFF walk
+ * found no data chunk, fell back to assuming a 44-byte header, and computed a NEGATIVE
+ * sample count — so `new Float32Array(-22)` threw. That throw landed in a catch whose job
+ * was to disable the feature when the MODEL is broken, so a single bad segment set
+ * _loadFailed and every later segment on the box was skipped, blaming a model that was
+ * fine. Production ran this way: "[AI-Timeline] audio-events disabled: Invalid typed
+ * array length: -22".
+ */
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const audioEvents = require('../server/ai/audio-events');
+const src = fs.readFileSync(path.join(__dirname, '../server/ai/audio-events.js'), 'utf8');
+
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-audio-events-'));
+const write = (name, buf) => { const p = path.join(tmp, name); fs.writeFileSync(p, buf); return p; };
+
+// A RIFF/WAVE header with no data chunk at all, and one that stops mid-header.
+const headerOnly = Buffer.alloc(12);
+headerOnly.write('RIFF', 0, 'ascii');
+headerOnly.writeUInt32LE(4, 4);
+headerOnly.write('WAVE', 8, 'ascii');
+
+(async () => {
+ // ── A: the shapes ffmpeg actually produces are handled, not thrown on ──────────────
+ const bad = {
+ 'empty.wav': Buffer.alloc(0),
+ 'header-only.wav': headerOnly,
+ 'truncated.wav': headerOnly.subarray(0, 9),
+ };
+ // Straight at the parser: detect() short-circuits when the model file is absent, so
+ // going through it would assert nothing on a machine without one.
+ for (const [name, buf] of Object.entries(bad)) {
+ const samples = audioEvents._wavToFloat32(write(name, buf));
+ assert.strictEqual(samples.length, 0, `${name} must read as zero samples, not throw`);
+ }
+ console.log('OK A: empty, header-only and truncated WAVs read as zero samples');
+
+ // ── A2: a well-formed file still reads correctly ──────────────────────────────────
+ {
+ const samples = 3;
+ const wav = Buffer.alloc(44 + samples * 2);
+ wav.write('RIFF', 0, 'ascii');
+ wav.writeUInt32LE(36 + samples * 2, 4);
+ wav.write('WAVEfmt ', 8, 'ascii');
+ wav.writeUInt32LE(16, 16);
+ wav.write('data', 36, 'ascii');
+ wav.writeUInt32LE(samples * 2, 40);
+ for (const [i, v] of [0, 32767, -32768].entries()) wav.writeInt16LE(v, 44 + i * 2);
+ const out = audioEvents._wavToFloat32(write('good.wav', wav));
+ assert.strictEqual(out.length, samples, 'a real data chunk must yield its samples');
+ assert.strictEqual(out[0], 0);
+ assert.ok(out[1] > 0.99 && out[1] <= 1, 'full-scale positive maps into [0,1]');
+ assert.strictEqual(out[2], -1, 'full-scale negative maps to -1');
+ console.log('OK A2: a well-formed WAV still reads its samples correctly');
+ }
+
+ // ── A3: and detect() itself stays quiet on a bad file ─────────────────────────────
+ assert.deepStrictEqual(await audioEvents.detect(path.join(tmp, 'empty.wav'), { offsetSec: 0 }), [],
+ 'detect() must return no events for an unreadable segment');
+ console.log('OK A3: detect() returns no events for an unreadable segment');
+
+ // ── B: and the feature is still alive afterwards ──────────────────────────────────
+ // available() is the gate every later segment passes through. If a bad file flipped
+ // _loadFailed, this is false and audio events are gone until the process restarts.
+ const modelPresent = fs.existsSync(audioEvents.MODEL_PATH) && fs.existsSync(audioEvents.CLASSMAP_PATH);
+ if (modelPresent) {
+ assert.strictEqual(audioEvents.available(), true,
+ 'a bad segment must not disable detection for every segment that follows');
+ console.log('OK B: detection still available after a bad segment');
+ } else {
+ console.log('OK B: skipped — no local model to check availability against');
+ }
+
+ // ── C: only a model load failure is allowed to be permanent ───────────────────────
+ const loadFail = src.indexOf('_loadFailed = true');
+ assert.ok(loadFail > 0, '_loadFailed must still exist');
+ assert.strictEqual(src.split('_loadFailed = true').length - 1, 1,
+ 'exactly one place may disable the feature permanently');
+ const guard = src.slice(src.lastIndexOf('try {', loadFail), loadFail);
+ assert.ok(/_session_\(\)/.test(guard), 'the permanent failure must come from loading the model');
+ assert.ok(!/_wavToFloat32/.test(guard) && !/session\.run/.test(guard),
+ 'reading a segment or running inference on it must not be able to disable the feature');
+ console.log('OK C: only model loading can disable detection permanently');
+
+ // ── D: a negative sample count cannot reach the allocator ─────────────────────────
+ assert.ok(/Math\.max\(0, Math\.floor\(/.test(src),
+ 'the sample count must be clamped before it reaches new Float32Array');
+ console.log('OK D: sample count is clamped at zero');
+
+ fs.rmSync(tmp, { recursive: true, force: true });
+ console.log('✅ audio events bad segment test passed');
+})().catch((e) => {
+ fs.rmSync(tmp, { recursive: true, force: true });
+ console.error('FAIL', e.message);
+ process.exit(1);
+});
@@ -3,13 +3,16 @@ const fs = require('fs');
@@ -3,13 +3,16 @@ const fs = require('fs');
const os = require('os');
const os = require('os');
const path = require('path');
const path = require('path');
-// cleanupSession looks the stream up to decide what to tear down, so requiring the WHIP
-// handler pulls in the database. Point it at a throwaway file with a real schema BEFORE
-// that require: the test used to open whatever data/live.db happened to be in the working
-// tree, which passed on a developed checkout and threw "no such table: streams" on a
-// fresh one — a property of the machine, not of the code under test.
+// After tearing the session down, cleanupSession asks whether the stream is still live —
+// the one thing here that touches the database. It used to reach whatever data/live.db
+// sat in the working tree, which passed on a developed checkout and threw "no such table:
+// streams" on a fresh one: a property of the machine, not of the code under test.
+//
+// Stub that single lookup instead of building a schema. Answering "no such stream" ends
+// endActiveWhipStream on its first line, which is the path this test wants anyway — what
+// it asserts is that the producer, transport and session are released.
process.env.DB_PATH = path.join(os.tmpdir(), `ov-whip-cleanup-${process.pid}.db`);
process.env.DB_PATH = path.join(os.tmpdir(), `ov-whip-cleanup-${process.pid}.db`);
-require('../server/db/database').initDb();
+require('../server/db/database').getStreamById = () => null;
const webrtcSFU = require('../server/streaming/webrtc-sfu');
const webrtcSFU = require('../server/streaming/webrtc-sfu');
const whipHandler = require('../server/streaming/whip-handler');
const whipHandler = require('../server/streaming/whip-handler');
@@ -48,7 +51,6 @@ assert.strictEqual(room.transports.has('whip-test-transport-1'), false, 'transpo
@@ -48,7 +51,6 @@ assert.strictEqual(room.transports.has('whip-test-transport-1'), false, 'transpo
assert.strictEqual(producerClosed.value, true, 'producer should be closed');
assert.strictEqual(producerClosed.value, true, 'producer should be closed');
assert.strictEqual(transportClosed.value, true, 'transport should be closed');
assert.strictEqual(transportClosed.value, true, 'transport should be closed');
-try { fs.unlinkSync(process.env.DB_PATH); } catch { /* */ }
-for (const ext of ['-wal', '-shm']) { try { fs.unlinkSync(process.env.DB_PATH + ext); } catch { /* */ } }
+for (const ext of ['', '-wal', '-shm']) { try { fs.unlinkSync(process.env.DB_PATH + ext); } catch { /* */ } }
console.log('✅ WHIP cleanup session regression test passed');
console.log('✅ WHIP cleanup session regression test passed');