openvibers/openvibe.live Commits

AI hears tagged sounds only: the detector's long tail never reaches a model (#671)

OP OpenVibers committed 6 days ago · 71ba7ec6ef · 7 files, +54 −8

Changes

ddiff --git a/server/ai/ai-analysis.js b/server/ai/ai-analysis.js
iindex d158e8ad..a081706f 100644
---- a/server/ai/ai-analysis.js
++++ b/server/ai/ai-analysis.js
@@@ -82,7 +82,9 @@ async function summarizeStreamMemories(memories, streamId = null) {
if (streamId) {
try {
input.speech = (await db.getTimeline(streamId, { kind: 'speech', limit: 400 }) || []).slice(-200).map(r => ({ start_sec: Number(r.start_sec) || 0, text: String(r.text || '').slice(0, 2000) }));
-- input.sounds = (await db.getTimeline(streamId, { kind: 'sound', limit: 120 }) || []).slice(-60).map(r => ({ start_sec: Number(r.start_sec) || 0, label: String(r.label || '').slice(0, 120), confidence: Number(r.confidence || 0) }));
++ // Tagged sounds above their bar only, by tag (sound-captions.evidence): the long tail is mostly wrong.
++ input.sounds = require('./sound-captions').evidence((await db.getTimeline(streamId, { kind: 'sound', limit: 120 }) || []).slice(-60)
++ .map(r => ({ start_sec: Number(r.start_sec) || 0, label: String(r.label || '').slice(0, 120), confidence: Number(r.confidence || 0) })));
} catch { /* timeline optional */ }
}
// The same run also classifies the stream: the category is inferred from what is actually on
ddiff --git a/server/ai/ai-moments-job.js b/server/ai/ai-moments-job.js
iindex e28e8eca..ab86d75b 100644
---- a/server/ai/ai-moments-job.js
++++ b/server/ai/ai-moments-job.js
@@@ -25,6 +25,7 @@ const clipPolicy = require('./clip-policy');
const dedupe = require('./clip-dedupe');
const explicit = require('./explicit');
const clipScore = require('./clip-score');
++const soundCaptions = require('./sound-captions');
let cfg = null; try { cfg = require('../config'); } catch { /* */ }
let thumb = null; try { thumb = require('../media-proxy/live-thumbs'); } catch { /* optional */ }
const FRAMES_TMP_DIR = path.join(os.tmpdir(), 'openvibe-live-moments');
@@@ -170,11 +171,14 @@ async function _momentContext(streamId, vodId) {
// Background music (audio-events.js spans) is the stream's bed, not a moment.
.filter(e => Number(e.confidence || 0) >= 0.45 && e.label !== 'Music');
} catch { /* timeline optional */ }
-- const sounds = _sample(heard, 30);
// The stream's average chat per 30 s, empty stretches counted (C07/C08: averaging only the busy buckets made every
// spike look small on a quiet channel and every lull look big on a busy one).
let chatAvg = 0;
const spanSec = started > 0 ? Math.max(0, ((ms(span.ended_at) || Date.now()) - started) / 1000) : 0;
++ // Only sounds with a tag above its bar, said by that tag, reach the prompts below (sound-captions.evidence): the
++ // detector's long tail ("Animal", "Vehicle", "Livestock") is mostly wrong and once titled a trading stream's moment.
++ heard = soundCaptions.evidence(heard, { spanSec });
++ const sounds = _sample(heard, 30);
try {
if (spanSec >= 60) chatAvg = (Number((await chatReads.streamStats(streamId) || {}).messages) || 0) / (spanSec / 30);
} catch { /* chat optional */ }
ddiff --git a/server/ai/context/index.js b/server/ai/context/index.js
iindex 259367dc..8e01a340 100644
---- a/server/ai/context/index.js
++++ b/server/ai/context/index.js
@@@ -24,6 +24,7 @@ const crypto = require('crypto');
const db = require('../../db/database');
const chatReads = require('../../chat/chat-reads');
const insightClient = require('../../chat/insight-client');
++const soundCaptions = require('../sound-captions');
function clip(str, n) { return (str == null ? '' : String(str)).replace(/\s+/g, ' ').trim().slice(0, n); }
function clipWords(str, n) { const t = clip(str, n * 8); return t.length > n ? t.slice(0, n).replace(/\s+\S*$/, '') + '…' : t; }
@@@ -143,13 +144,15 @@ async function heardBlock(stream, settings) {
let lastEnd = null;
for (const r of rows) {
if (r.kind === 'speech') { const t = scrubSpeech(r.text); if (hasWords(t)) { speech.push(`[${mmss(r.start_sec)}] ${t}`); lastEnd = r.end_sec || r.start_sec; } }
-- else if (r.kind === 'sound' && r.label) sounds.push({ label: r.label, at: r.start_sec, confidence: r.confidence });
++ else if (r.kind === 'sound' && r.label) sounds.push({ label: r.label, start_sec: r.start_sec, confidence: r.confidence });
}
++ // Only tagged sounds above their bar, by tag (sound-captions.evidence); the detector's long tail is mostly wrong.
++ const heardSounds = soundCaptions.evidence(sounds, { spanSec: settings.hear_window_sec || 90 }).map((e) => ({ label: e.label, at: e.start_sec, confidence: e.confidence }));
let text = speech.join('\n');
if (text.length > 900) text = text.slice(-900).replace(/^[^\[]*/, '');
-- const uniqSounds = [...new Set(sounds.map(s => s.label))].slice(0, 6);
++ const uniqSounds = [...new Set(heardSounds.map(s => s.label))].slice(0, 6);
const ageMs = lastEnd != null ? Math.max(0, (off - lastEnd) * 1000) : null;
-- return { text, ageMs, count: speech.length, sounds: uniqSounds, latestSound: sounds.length ? sounds[sounds.length - 1] : null };
++ return { text, ageMs, count: speech.length, sounds: uniqSounds, latestSound: heardSounds.length ? heardSounds[heardSounds.length - 1] : null };
}
async function seenBlock(stream) {
ddiff --git a/server/ai/sound-captions.js b/server/ai/sound-captions.js
iindex e625906a..5bd0ddb9 100644
---- a/server/ai/sound-captions.js
++++ b/server/ai/sound-captions.js
@@@ -81,4 +81,31 @@ function cues(events, { spanSec = 0 } = {}) {
return out.map((c) => ({ start: Math.round(c.start * 100) / 100, end: Math.round(c.end * 100) / 100, text: c.text }));
}
--module.exports = { cues, tagOf, TAGS };
++/**
++ * The sounds an AI prompt may hear (2026-11-07: a rare misheard "Livestock" in a trading stream became the moment
++ * "Live chaos: farm noises clash"). Only a label with a tag above that tag's bar passes, a rare-only tag (explosion,
++ * gunfire) only where it is rare in the stream, and each is said by its tag ("laughter", not "Chuckle, chortle"), so
++ * the long tail never reaches a model. `spanSec` defaults to the events' own time range.
++ *
++ * evidence(events, { spanSec }) → [{ ...event, label: tag }]
++ */
++function evidence(events, { spanSec } = {}) {
++ const list = (Array.isArray(events) ? events : []).filter((e) => e && e.label && e.label !== 'Music');
++ if (!list.length) return [];
++ let span = Number(spanSec);
++ if (!Number.isFinite(span) || span <= 0) {
++ const times = list.map((e) => Number(e.start_sec) || 0);
++ span = Math.max(...times) - Math.min(...times);
++ }
++ const rare = new Set(clipScore.rareSounds(list, span).map((e) => String(e.label || '')));
++ const out = [];
++ for (const e of list) {
++ const t = tagOf(e.label);
++ if (!t || Number(e.confidence || 0) < t.min) continue;
++ if (t.rareOnly && !rare.has(String(e.label))) continue;
++ out.push({ ...e, label: t.tag });
++ }
++ return out;
++}
++
++module.exports = { cues, tagOf, evidence, TAGS };
ddiff --git a/test/rare-sounds.test.js b/test/rare-sounds.test.js
iindex 1acdd857..b5e3e21a 100644
---- a/test/rare-sounds.test.js
++++ b/test/rare-sounds.test.js
@@@ -41,7 +41,7 @@ console.warn = () => {};
assert.ok(m, 'a clip moment is found');
assert.strictEqual(m.offset, 3000, 'the laugh, not the first explosion');
assert.strictEqual(m.evidence.picked_by, 'sound');
-- assert.deepStrictEqual(m.evidence.sounds.map((e) => e.label), ['Laughter'], 'the explosions nearby are not evidence');
++ assert.deepStrictEqual(m.evidence.sounds.map((e) => e.label), ['laughter'], 'the explosions nearby are not evidence; the laugh is said by its tag');
db.getTimeline = async () => evening.filter((e) => e.label === 'Explosion');
assert.strictEqual(await moments.findBestMoment(v, { flavor: 'clip' }), null, 'a stream of nothing but its background gives no clip');
ddiff --git a/test/sound-captions.test.js b/test/sound-captions.test.js
iindex cd849a61..4b347b20 100644
---- a/test/sound-captions.test.js
++++ b/test/sound-captions.test.js
@@@ -22,4 +22,14 @@ assert.deepStrictEqual(sc.cues([ev(50, 'Explosion', 0.9, 51), ev(70, 'Gunshot, g
[{ start: 50, end: 51.5, text: '[explosion]' }], 'one rare explosion is news; gunfire below 0.85 is not captioned');
assert.deepStrictEqual(sc.cues(null), []);
assert.strictEqual(sc.tagOf('Dog').tag, 'dog barking');
++
++// What a model may hear (evidence): 2026-11-07, a rare misheard "Livestock" in a trading stream became the moment
++// "Live chaos: farm noises clash". The long tail never passes; a tag passes above its bar, by its tag; a rare-only tag
++// only where it is rare; music is the bed, not a sound.
++const heard = sc.evidence([ev(10, 'Livestock, farm animals, working animals', 0.7), ev(12, 'Animal', 0.9), ev(20, 'Chuckle, chortle', 0.8),
++ ev(30, 'Laughter', 0.5), ev(40, 'Explosion', 0.9), ev(45, 'Music', 0.9)], { spanSec: 3600 });
++assert.deepStrictEqual(heard.map((e) => [e.start_sec, e.label]), [[20, 'laughter'], [40, 'explosion']], 'only tagged sounds above their bar, by tag');
++assert.deepStrictEqual(sc.evidence(shooter, { spanSec: 3600 }), [], 'a shooter\'s constant explosions are not evidence');
++assert.deepStrictEqual(sc.evidence(null), []);
++assert.strictEqual(sc.evidence([ev(5, 'Applause', 0.8)]).length, 1, 'no span given: the events\' own range');
console.log('sound captions: all checks passed');
ddiff --git a/test/vod-moment-evidence.test.js b/test/vod-moment-evidence.test.js
iindex d9031f6d..fe651add 100644
---- a/test/vod-moment-evidence.test.js
++++ b/test/vod-moment-evidence.test.js
@@@ -155,7 +155,7 @@ console.warn = () => {};
assert.strictEqual(clip.evidence.chatters, 5);
assert.ok(clip.evidence.spike.avg > 0.9 && clip.evidence.spike.avg < 1.1, 'the average counts the quiet stretches: 240 messages over 240 buckets');
assert.ok(clip.evidence.speechWords >= 10, 'the words come from the whole transcript near the moment, not a sample');
-- assert.strictEqual(clip.evidence.sounds[0].label, 'Laughter');
++ assert.strictEqual(clip.evidence.sounds[0].label, 'laughter', 'the sound is said by its tag (sound-captions.evidence)');
// ── The AI may abstain: no weaker signal is tried in its place ──
ai.isEnabled = async () => true; ai.withinBudget = async () => true;