openvibers/openvibe.live Commits
Background music is one timeline span, never a clip signal (item 151, C14) (#575)
Changes
@@@ -161,7 +161,8 @@ async function _momentContext(streamId, vodId) {let sounds = [];try {sounds = _sample((await db.getTimeline(streamId, { kind: 'sound', limit: 400 }) || [])-- .filter(e => Number(e.confidence || 0) >= 0.45), 30);++ // Background music (audio-events.js spans) is the stream's bed, not a moment.++ .filter(e => Number(e.confidence || 0) >= 0.45 && e.label !== 'Music'), 30);} catch { /* timeline optional */ }return _inPart({ memories, transcript, clipTimes, spikes, sounds }, vodId);}@@@ -142,6 +142,51 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }return [];}++ return eventsFromScores(scores, frames, classes, labels, { offsetSec, minConfidence });++}++++// Music (item 151, C14): a stream's background music is one long span, not a flicker of frames, so it is its own++// event with hysteresis instead of a per-frame label. It begins after MUSIC_ON_FRAMES frames at MUSIC_ON or more++// (about 1.4 s), ends after MUSIC_OFF_FRAMES frames below MUSIC_OFF (about 2.9 s), and is kept when it lasted at least++// MUSIC_MIN_SEC; it never counts against the per-segment cap. Timeline rows read "Music" with background: true.++const MUSIC_ON = 0.45;++const MUSIC_OFF = 0.25;++const MUSIC_ON_FRAMES = 3;++const MUSIC_OFF_FRAMES = 6;++const MUSIC_MIN_SEC = 5;++++function musicSpans(scores, frames, classes, labels, offsetSec) {++ const c = labels.indexOf('Music');++ if (c < 0) return [];++ const spans = [];++ let on = null; // { start, end, peak } while music plays++ let above = 0; // frames in a row at MUSIC_ON or more (while off)++ let below = 0; // frames in a row under MUSIC_OFF (while on)++ let firstAbove = 0;++ for (let f = 0; f < frames; f++) {++ const v = scores[f * classes + c];++ const t = offsetSec + f * FRAME_HOP_SEC;++ if (!on) {++ if (v >= MUSIC_ON) { if (above === 0) firstAbove = t; above++; } else above = 0;++ if (above >= MUSIC_ON_FRAMES) { on = { start: firstAbove, end: t + FRAME_HOP_SEC, peak: v }; below = 0; }++ } else {++ if (v > on.peak) on.peak = v;++ if (v < MUSIC_OFF) {++ below++;++ if (below >= MUSIC_OFF_FRAMES) { spans.push(on); on = null; above = 0; }++ } else { below = 0; on.end = t + FRAME_HOP_SEC; }++ }++ }++ if (on) spans.push(on);++ return spans.filter((m) => m.end - m.start >= MUSIC_MIN_SEC);++}++++/**++ * The events in one segment's YAMNet scores (frames × classes): per-frame winners above the threshold, merged into++ * runs of the same label (at most MAX_EVENTS_PER_SEGMENT, the strongest), plus background music spans. Pure, so the++ * tests can feed it scores without the model.++ */++function eventsFromScores(scores, frames, classes, labels, { offsetSec = 0, minConfidence = MIN_CONFIDENCE } = {}) {// Collect per-frame winners above threshold, then merge runs of the same label.const open = new Map(); // label -> { start, end, peak }const done = [];@@@ -168,18 +213,16 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }}for (const ev of open.values()) done.push(ev);++ const round = (e) => ({ start: Math.round(e.start * 100) / 100, end: Math.round(e.end * 100) / 100, label: e.label, confidence: Math.round(e.peak * 1000) / 1000 });++ const music = musicSpans(scores, frames, classes, labels, offsetSec).map((m) => ({ ...round({ ...m, label: 'Music' }), background: true }));return done.sort((a, b) => b.peak - a.peak).slice(0, MAX_EVENTS_PER_SEGMENT)-- .map(e => ({-- start: Math.round(e.start * 100) / 100,-- end: Math.round(e.end * 100) / 100,-- label: e.label,-- confidence: Math.round(e.peak * 1000) / 1000,-- }))++ .map(round)++ .concat(music).sort((a, b) => a.start - b.start);}// _wavToFloat32 is exported for tests: it is the part that has to survive whatever// ffmpeg leaves on disk, and it cannot be reached through detect() without the model.--module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32 };++module.exports = { available, detect, eventsFromScores, musicSpans, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32, MUSIC_MIN_SEC };@@@ -83,7 +83,8 @@ async function _confirmLiveMoment(stream) {try {const since = Math.max(0, ((Date.now() - new Date(stream.started_at + 'Z').getTime()) / 1000) - WINDOW_SEC);sounds = (await db.getTimeline(stream.id, { kind: 'sound', from: since, limit: 40 }) || [])-- .filter(e => Number(e.confidence || 0) >= 0.4).slice(-12);++ // Background music (audio-events.js spans) is the stream's bed, not a moment: never a clip signal.++ .filter(e => Number(e.confidence || 0) >= 0.4 && e.label !== 'Music').slice(-12);} catch { /* timeline optional */ }// The decision rules are OpenVibe.AI's versioned template live.clips.confirm (WS-O task 2); Live sends what it saw,// heard and read, with times so the model can line up speech, sounds and the chat spike.nnew file mode 100644@@@ -0,0 +1,54 @@++/**++ * Background music is one span with hysteresis, not a flicker of frames (server/ai/audio-events.js; item 151, C14):++ * it starts after ~1.4 s of confident music and ends after ~2.9 s without, is kept only when it lasted 5 s, never++ * counts against the per-segment cap, and never feeds the clip picker or the moments as a highlight.++ *++ * node test/run.js audio-music-spans++ */++'use strict';++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const ae = require('../server/ai/audio-events');++++const labels = ['Music', 'Laughter', 'Speech'];++const HOP = 0.48;++/** frames × 3 scores from a per-frame [music, laughter, speech] list. */++function scoresOf(rows) {++ const out = new Float32Array(rows.length * labels.length);++ rows.forEach((r, f) => r.forEach((v, c) => { out[f * labels.length + c] = v; }));++ return out;++}++const run = (rows, offsetSec = 100) => ae.eventsFromScores(scoresOf(rows), rows.length, labels.length, labels, { offsetSec, minConfidence: 0.35 });++++// 30 frames (14.4 s) of music with a 2-frame dip in the middle: one span, not two.++const music = Array.from({ length: 30 }, (_, f) => [f === 14 || f === 15 ? 0.1 : 0.7, 0, 0]);++let ev = run(music);++let m = ev.filter((e) => e.label === 'Music');++assert.strictEqual(m.length, 1, 'one span across a short dip');++assert.strictEqual(m[0].background, true);++assert.strictEqual(m[0].start, 100, 'starts where the confident run began');++assert.ok(m[0].end - m[0].start >= 13.5, `lasts the whole run (${m[0].end - m[0].start})`);++assert.strictEqual(m[0].confidence, 0.7);++++// A 2-second jingle (4 frames) is below MUSIC_MIN_SEC: no span. A flicker (alternating frames) never starts one.++assert.deepStrictEqual(run(Array.from({ length: 4 }, () => [0.9, 0, 0])).filter((e) => e.label === 'Music'), []);++assert.deepStrictEqual(run(Array.from({ length: 30 }, (_, f) => [f % 2 ? 0.9 : 0.1, 0, 0])).filter((e) => e.label === 'Music'), []);++++// Two pieces of music with a long silence between them: two spans.++const two = [...Array.from({ length: 14 }, () => [0.8, 0, 0]), ...Array.from({ length: 10 }, () => [0.05, 0, 0]), ...Array.from({ length: 14 }, () => [0.8, 0, 0])];++m = run(two).filter((e) => e.label === 'Music');++assert.strictEqual(m.length, 2, 'two spans');++++// Laughter over music: both, and the laughter is an ordinary event (Music never a per-frame event).++const laughOverMusic = Array.from({ length: 20 }, (_, f) => [0.6, f >= 5 && f <= 8 ? 0.8 : 0, 0]);++ev = run(laughOverMusic);++assert.ok(ev.some((e) => e.label === 'Laughter' && !e.background), 'laughter is a moment');++assert.strictEqual(ev.filter((e) => e.label === 'Music').length, 1);++assert.ok(ev.every((e, i) => i === 0 || ev[i - 1].start <= e.start), 'in time order');++++// Music never feeds the clip picker or the moments as a highlight.++const src = (f) => fs.readFileSync(path.join(__dirname, '..', 'server', 'ai', f), 'utf8');++assert.match(src('auto-clip-job.js'), /e\.label !== 'Music'/);++assert.match(src('ai-moments-job.js'), /e\.label !== 'Music'/);++console.log('audio music spans: all checks passed');
@@ -161,7 +161,8 @@ async function _momentContext(streamId, vodId) {
@@ -161,7 +161,8 @@ async function _momentContext(streamId, vodId) {
let sounds = [];
let sounds = [];
try {
try {
sounds = _sample((await db.getTimeline(streamId, { kind: 'sound', limit: 400 }) || [])
sounds = _sample((await db.getTimeline(streamId, { kind: 'sound', limit: 400 }) || [])
- .filter(e => Number(e.confidence || 0) >= 0.45), 30);
+ // Background music (audio-events.js spans) is the stream's bed, not a moment.
+ .filter(e => Number(e.confidence || 0) >= 0.45 && e.label !== 'Music'), 30);
} catch { /* timeline optional */ }
} catch { /* timeline optional */ }
return _inPart({ memories, transcript, clipTimes, spikes, sounds }, vodId);
return _inPart({ memories, transcript, clipTimes, spikes, sounds }, vodId);
}
}
@@ -142,6 +142,51 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
@@ -142,6 +142,51 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
return [];
return [];
}
}
+ return eventsFromScores(scores, frames, classes, labels, { offsetSec, minConfidence });
+}
+
+// Music (item 151, C14): a stream's background music is one long span, not a flicker of frames, so it is its own
+// event with hysteresis instead of a per-frame label. It begins after MUSIC_ON_FRAMES frames at MUSIC_ON or more
+// (about 1.4 s), ends after MUSIC_OFF_FRAMES frames below MUSIC_OFF (about 2.9 s), and is kept when it lasted at least
+// MUSIC_MIN_SEC; it never counts against the per-segment cap. Timeline rows read "Music" with background: true.
+const MUSIC_ON = 0.45;
+const MUSIC_OFF = 0.25;
+const MUSIC_ON_FRAMES = 3;
+const MUSIC_OFF_FRAMES = 6;
+const MUSIC_MIN_SEC = 5;
+
+function musicSpans(scores, frames, classes, labels, offsetSec) {
+ const c = labels.indexOf('Music');
+ if (c < 0) return [];
+ const spans = [];
+ let on = null; // { start, end, peak } while music plays
+ let above = 0; // frames in a row at MUSIC_ON or more (while off)
+ let below = 0; // frames in a row under MUSIC_OFF (while on)
+ let firstAbove = 0;
+ for (let f = 0; f < frames; f++) {
+ const v = scores[f * classes + c];
+ const t = offsetSec + f * FRAME_HOP_SEC;
+ if (!on) {
+ if (v >= MUSIC_ON) { if (above === 0) firstAbove = t; above++; } else above = 0;
+ if (above >= MUSIC_ON_FRAMES) { on = { start: firstAbove, end: t + FRAME_HOP_SEC, peak: v }; below = 0; }
+ } else {
+ if (v > on.peak) on.peak = v;
+ if (v < MUSIC_OFF) {
+ below++;
+ if (below >= MUSIC_OFF_FRAMES) { spans.push(on); on = null; above = 0; }
+ } else { below = 0; on.end = t + FRAME_HOP_SEC; }
+ }
+ }
+ if (on) spans.push(on);
+ return spans.filter((m) => m.end - m.start >= MUSIC_MIN_SEC);
+}
+
+/**
+ * The events in one segment's YAMNet scores (frames × classes): per-frame winners above the threshold, merged into
+ * runs of the same label (at most MAX_EVENTS_PER_SEGMENT, the strongest), plus background music spans. Pure, so the
+ * tests can feed it scores without the model.
+ */
+function eventsFromScores(scores, frames, classes, labels, { offsetSec = 0, minConfidence = MIN_CONFIDENCE } = {}) {
// Collect per-frame winners above threshold, then merge runs of the same label.
// Collect per-frame winners above threshold, then merge runs of the same label.
const open = new Map(); // label -> { start, end, peak }
const open = new Map(); // label -> { start, end, peak }
const done = [];
const done = [];
@@ -168,18 +213,16 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
@@ -168,18 +213,16 @@ async function detect(wavPath, { offsetSec = 0, minConfidence = MIN_CONFIDENCE }
}
}
for (const ev of open.values()) done.push(ev);
for (const ev of open.values()) done.push(ev);
+ const round = (e) => ({ start: Math.round(e.start * 100) / 100, end: Math.round(e.end * 100) / 100, label: e.label, confidence: Math.round(e.peak * 1000) / 1000 });
+ const music = musicSpans(scores, frames, classes, labels, offsetSec).map((m) => ({ ...round({ ...m, label: 'Music' }), background: true }));
return done
return done
.sort((a, b) => b.peak - a.peak)
.sort((a, b) => b.peak - a.peak)
.slice(0, MAX_EVENTS_PER_SEGMENT)
.slice(0, MAX_EVENTS_PER_SEGMENT)
- .map(e => ({
- start: Math.round(e.start * 100) / 100,
- end: Math.round(e.end * 100) / 100,
- label: e.label,
- confidence: Math.round(e.peak * 1000) / 1000,
- }))
+ .map(round)
+ .concat(music)
.sort((a, b) => a.start - b.start);
.sort((a, b) => a.start - b.start);
}
}
// _wavToFloat32 is exported for tests: it is the part that has to survive whatever
// _wavToFloat32 is exported for tests: it is the part that has to survive whatever
// ffmpeg leaves on disk, and it cannot be reached through detect() without the model.
// ffmpeg leaves on disk, and it cannot be reached through detect() without the model.
-module.exports = { available, detect, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32 };
+module.exports = { available, detect, eventsFromScores, musicSpans, MODEL_PATH, CLASSMAP_PATH, _wavToFloat32, MUSIC_MIN_SEC };
@@ -83,7 +83,8 @@ async function _confirmLiveMoment(stream) {
@@ -83,7 +83,8 @@ async function _confirmLiveMoment(stream) {
try {
try {
const since = Math.max(0, ((Date.now() - new Date(stream.started_at + 'Z').getTime()) / 1000) - WINDOW_SEC);
const since = Math.max(0, ((Date.now() - new Date(stream.started_at + 'Z').getTime()) / 1000) - WINDOW_SEC);
sounds = (await db.getTimeline(stream.id, { kind: 'sound', from: since, limit: 40 }) || [])
sounds = (await db.getTimeline(stream.id, { kind: 'sound', from: since, limit: 40 }) || [])
- .filter(e => Number(e.confidence || 0) >= 0.4).slice(-12);
+ // Background music (audio-events.js spans) is the stream's bed, not a moment: never a clip signal.
+ .filter(e => Number(e.confidence || 0) >= 0.4 && e.label !== 'Music').slice(-12);
} catch { /* timeline optional */ }
} catch { /* timeline optional */ }
// The decision rules are OpenVibe.AI's versioned template live.clips.confirm (WS-O task 2); Live sends what it saw,
// The decision rules are OpenVibe.AI's versioned template live.clips.confirm (WS-O task 2); Live sends what it saw,
// heard and read, with times so the model can line up speech, sounds and the chat spike.
// heard and read, with times so the model can line up speech, sounds and the chat spike.
new file mode 100644
new file mode 100644
@@ -0,0 +1,54 @@
@@ -0,0 +1,54 @@
+/**
+ * Background music is one span with hysteresis, not a flicker of frames (server/ai/audio-events.js; item 151, C14):
+ * it starts after ~1.4 s of confident music and ends after ~2.9 s without, is kept only when it lasted 5 s, never
+ * counts against the per-segment cap, and never feeds the clip picker or the moments as a highlight.
+ *
+ * node test/run.js audio-music-spans
+ */
+'use strict';
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const ae = require('../server/ai/audio-events');
+
+const labels = ['Music', 'Laughter', 'Speech'];
+const HOP = 0.48;
+/** frames × 3 scores from a per-frame [music, laughter, speech] list. */
+function scoresOf(rows) {
+ const out = new Float32Array(rows.length * labels.length);
+ rows.forEach((r, f) => r.forEach((v, c) => { out[f * labels.length + c] = v; }));
+ return out;
+}
+const run = (rows, offsetSec = 100) => ae.eventsFromScores(scoresOf(rows), rows.length, labels.length, labels, { offsetSec, minConfidence: 0.35 });
+
+// 30 frames (14.4 s) of music with a 2-frame dip in the middle: one span, not two.
+const music = Array.from({ length: 30 }, (_, f) => [f === 14 || f === 15 ? 0.1 : 0.7, 0, 0]);
+let ev = run(music);
+let m = ev.filter((e) => e.label === 'Music');
+assert.strictEqual(m.length, 1, 'one span across a short dip');
+assert.strictEqual(m[0].background, true);
+assert.strictEqual(m[0].start, 100, 'starts where the confident run began');
+assert.ok(m[0].end - m[0].start >= 13.5, `lasts the whole run (${m[0].end - m[0].start})`);
+assert.strictEqual(m[0].confidence, 0.7);
+
+// A 2-second jingle (4 frames) is below MUSIC_MIN_SEC: no span. A flicker (alternating frames) never starts one.
+assert.deepStrictEqual(run(Array.from({ length: 4 }, () => [0.9, 0, 0])).filter((e) => e.label === 'Music'), []);
+assert.deepStrictEqual(run(Array.from({ length: 30 }, (_, f) => [f % 2 ? 0.9 : 0.1, 0, 0])).filter((e) => e.label === 'Music'), []);
+
+// Two pieces of music with a long silence between them: two spans.
+const two = [...Array.from({ length: 14 }, () => [0.8, 0, 0]), ...Array.from({ length: 10 }, () => [0.05, 0, 0]), ...Array.from({ length: 14 }, () => [0.8, 0, 0])];
+m = run(two).filter((e) => e.label === 'Music');
+assert.strictEqual(m.length, 2, 'two spans');
+
+// Laughter over music: both, and the laughter is an ordinary event (Music never a per-frame event).
+const laughOverMusic = Array.from({ length: 20 }, (_, f) => [0.6, f >= 5 && f <= 8 ? 0.8 : 0, 0]);
+ev = run(laughOverMusic);
+assert.ok(ev.some((e) => e.label === 'Laughter' && !e.background), 'laughter is a moment');
+assert.strictEqual(ev.filter((e) => e.label === 'Music').length, 1);
+assert.ok(ev.every((e, i) => i === 0 || ev[i - 1].start <= e.start), 'in time order');
+
+// Music never feeds the clip picker or the moments as a highlight.
+const src = (f) => fs.readFileSync(path.join(__dirname, '..', 'server', 'ai', f), 'utf8');
+assert.match(src('auto-clip-job.js'), /e\.label !== 'Music'/);
+assert.match(src('ai-moments-job.js'), /e\.label !== 'Music'/);
+console.log('audio music spans: all checks passed');