openvibers/openvibe.live Commits

Background music is one timeline span, never a clip signal (item 151, C14) (#575)

OP OpenVibers committed 10 days ago · dcd1104c57 · 4 files, +108 −9

YAMNet's Music label was ignored, so a stream's music never reached the timeline. It is now its own event with
hysteresis (on after ~1.4 s of confident music, off after ~2.9 s without, kept from 5 s), outside the per-segment
cap. The clip picker and the moments leave it out: the stream's bed is not a highlight; laughter over it still is.

Changes

ddiff --git a/server/ai/ai-moments-job.js b/server/ai/ai-moments-job.js
iindex abdb4cf1..4b1649ad 100644
---- a/server/ai/ai-moments-job.js
++++ b/server/ai/ai-moments-job.js
@@@ -161,7 +161,8 @@ async function _momentContext(streamId, vodId) {
let sounds = [];
try {
sounds = _sample((await db.getTimeline(streamId, { kind: 'sound', limit: 400 }) || [])
-- .filter(e => Number(e.confidence || 0) >= 0.45), 30);
++ // Background music (audio-events.js spans) is the stream's bed, not a moment.
++ .filter(e => Number(e.confidence || 0) >= 0.45 && e.label !== 'Music'), 30);
} catch { /* timeline optional */ }
return _inPart({ memories, transcript, clipTimes, spikes, sounds }, vodId);
}
ddiff --git a/server/ai/audio-events.js b/server/ai/audio-events.js
iindex 553eac9a..f4802327 100644
---- a/server/ai/audio-events.js
++++ b/server/ai/audio-events.js
@@@ -142,6 +142,51 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
return [];
}
++ return eventsFromScores(scores, frames, classes, labels, { offsetSec, minConfidence });
++}
++
++// Music (item 151, C14): a stream's background music is one long span, not a flicker of frames, so it is its own
++// event with hysteresis instead of a per-frame label. It begins after MUSIC_ON_FRAMES frames at MUSIC_ON or more
++// (about 1.4 s), ends after MUSIC_OFF_FRAMES frames below MUSIC_OFF (about 2.9 s), and is kept when it lasted at least
++// MUSIC_MIN_SEC; it never counts against the per-segment cap. Timeline rows read "Music" with background: true.
++const MUSIC_ON = 0.45;
++const MUSIC_OFF = 0.25;
++const MUSIC_ON_FRAMES = 3;
++const MUSIC_OFF_FRAMES = 6;
++const MUSIC_MIN_SEC = 5;
++
++function musicSpans(scores, frames, classes, labels, offsetSec) {
++ const c = labels.indexOf('Music');
++ if (c < 0) return [];
++ const spans = [];
++ let on = null; // { start, end, peak } while music plays
++ let above = 0; // frames in a row at MUSIC_ON or more (while off)
++ let below = 0; // frames in a row under MUSIC_OFF (while on)
++ let firstAbove = 0;
++ for (let f = 0; f < frames; f++) {
++ const v = scores[f * classes + c];
++ const t = offsetSec + f * FRAME_HOP_SEC;
++ if (!on) {
++ if (v >= MUSIC_ON) { if (above === 0) firstAbove = t; above++; } else above = 0;
++ if (above >= MUSIC_ON_FRAMES) { on = { start: firstAbove, end: t + FRAME_HOP_SEC, peak: v }; below = 0; }
++ } else {
++ if (v > on.peak) on.peak = v;
++ if (v < MUSIC_OFF) {
++ below++;
++ if (below >= MUSIC_OFF_FRAMES) { spans.push(on); on = null; above = 0; }
++ } else { below = 0; on.end = t + FRAME_HOP_SEC; }
++ }
++ }
++ if (on) spans.push(on);
++ return spans.filter((m) => m.end - m.start >= MUSIC_MIN_SEC);
++}
++
++/**
++ * The events in one segment's YAMNet scores (frames × classes): per-frame winners above the threshold, merged into
++ * runs of the same label (at most MAX_EVENTS_PER_SEGMENT, the strongest), plus background music spans. Pure, so the
++ * tests can feed it scores without the model.
++ */
++function eventsFromScores(scores, frames, classes, labels, { offsetSec = 0, minConfidence = MIN_CONFIDENCE } = {}) {
// Collect per-frame winners above threshold, then merge runs of the same label.
const open = new Map(); // label -> { start, end, peak }
const done = [];
@@@ -168,18 +213,16 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
}
for (const ev of open.values()) done.push(ev);
++ const round = (e) => ({ start: Math.round(e.start * 100) / 100, end: Math.round(e.end * 100) / 100, label: e.label, confidence: Math.round(e.peak * 1000) / 1000 });
++ const music = musicSpans(scores, frames, classes, labels, offsetSec).map((m) => ({ ...round({ ...m, label: 'Music' }), background: true }));
return done
.sort((a, b) => b.peak - a.peak)
.slice(0, MAX_EVENTS_PER_SEGMENT)
-- .map(e => ({
-- start: Math.round(e.start * 100) / 100,
-- end: Math.round(e.end * 100) / 100,
-- label: e.label,
-- confidence: Math.round(e.peak * 1000) / 1000,
-- }))
++ .map(round)
++ .concat(music)
.sort((a, b) => a.start - b.start);
}
// _wavToFloat32 is exported for tests: it is the part that has to survive whatever
// ffmpeg leaves on disk, and it cannot be reached through detect() without the model.
--module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32 };
++module.exports = { available, detect, eventsFromScores, musicSpans, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32, MUSIC_MIN_SEC };
ddiff --git a/server/ai/auto-clip-job.js b/server/ai/auto-clip-job.js
iindex 8868a5e5..c9007cb8 100644
---- a/server/ai/auto-clip-job.js
++++ b/server/ai/auto-clip-job.js
@@@ -83,7 +83,8 @@ async function _confirmLiveMoment(stream) {
try {
const since = Math.max(0, ((Date.now() - new Date(stream.started_at + 'Z').getTime()) / 1000) - WINDOW_SEC);
sounds = (await db.getTimeline(stream.id, { kind: 'sound', from: since, limit: 40 }) || [])
-- .filter(e => Number(e.confidence || 0) >= 0.4).slice(-12);
++ // Background music (audio-events.js spans) is the stream's bed, not a moment: never a clip signal.
++ .filter(e => Number(e.confidence || 0) >= 0.4 && e.label !== 'Music').slice(-12);
} catch { /* timeline optional */ }
// The decision rules are OpenVibe.AI's versioned template live.clips.confirm (WS-O task 2); Live sends what it saw,
// heard and read, with times so the model can line up speech, sounds and the chat spike.
ddiff --git a/test/audio-music-spans.test.js b/test/audio-music-spans.test.js
nnew file mode 100644
iindex 00000000..ab1485a3
---- /dev/null
++++ b/test/audio-music-spans.test.js
@@@ -0,0 +1,54 @@
++/**
++ * Background music is one span with hysteresis, not a flicker of frames (server/ai/audio-events.js; item 151, C14):
++ * it starts after ~1.4 s of confident music and ends after ~2.9 s without, is kept only when it lasted 5 s, never
++ * counts against the per-segment cap, and never feeds the clip picker or the moments as a highlight.
++ *
++ * node test/run.js audio-music-spans
++ */
++'use strict';
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const ae = require('../server/ai/audio-events');
++
++const labels = ['Music', 'Laughter', 'Speech'];
++const HOP = 0.48;
++/** frames × 3 scores from a per-frame [music, laughter, speech] list. */
++function scoresOf(rows) {
++ const out = new Float32Array(rows.length * labels.length);
++ rows.forEach((r, f) => r.forEach((v, c) => { out[f * labels.length + c] = v; }));
++ return out;
++}
++const run = (rows, offsetSec = 100) => ae.eventsFromScores(scoresOf(rows), rows.length, labels.length, labels, { offsetSec, minConfidence: 0.35 });
++
++// 30 frames (14.4 s) of music with a 2-frame dip in the middle: one span, not two.
++const music = Array.from({ length: 30 }, (_, f) => [f === 14 || f === 15 ? 0.1 : 0.7, 0, 0]);
++let ev = run(music);
++let m = ev.filter((e) => e.label === 'Music');
++assert.strictEqual(m.length, 1, 'one span across a short dip');
++assert.strictEqual(m[0].background, true);
++assert.strictEqual(m[0].start, 100, 'starts where the confident run began');
++assert.ok(m[0].end - m[0].start >= 13.5, `lasts the whole run (${m[0].end - m[0].start})`);
++assert.strictEqual(m[0].confidence, 0.7);
++
++// A 2-second jingle (4 frames) is below MUSIC_MIN_SEC: no span. A flicker (alternating frames) never starts one.
++assert.deepStrictEqual(run(Array.from({ length: 4 }, () => [0.9, 0, 0])).filter((e) => e.label === 'Music'), []);
++assert.deepStrictEqual(run(Array.from({ length: 30 }, (_, f) => [f % 2 ? 0.9 : 0.1, 0, 0])).filter((e) => e.label === 'Music'), []);
++
++// Two pieces of music with a long silence between them: two spans.
++const two = [...Array.from({ length: 14 }, () => [0.8, 0, 0]), ...Array.from({ length: 10 }, () => [0.05, 0, 0]), ...Array.from({ length: 14 }, () => [0.8, 0, 0])];
++m = run(two).filter((e) => e.label === 'Music');
++assert.strictEqual(m.length, 2, 'two spans');
++
++// Laughter over music: both, and the laughter is an ordinary event (Music never a per-frame event).
++const laughOverMusic = Array.from({ length: 20 }, (_, f) => [0.6, f >= 5 && f <= 8 ? 0.8 : 0, 0]);
++ev = run(laughOverMusic);
++assert.ok(ev.some((e) => e.label === 'Laughter' && !e.background), 'laughter is a moment');
++assert.strictEqual(ev.filter((e) => e.label === 'Music').length, 1);
++assert.ok(ev.every((e, i) => i === 0 || ev[i - 1].start <= e.start), 'in time order');
++
++// Music never feeds the clip picker or the moments as a highlight.
++const src = (f) => fs.readFileSync(path.join(__dirname, '..', 'server', 'ai', f), 'utf8');
++assert.match(src('auto-clip-job.js'), /e\.label !== 'Music'/);
++assert.match(src('ai-moments-job.js'), /e\.label !== 'Music'/);
++console.log('audio music spans: all checks passed');