openvibers/openvibe.live Commits
AI hears tagged sounds only: the detector's long tail never reaches a model (#671)
Changes
@@@ -82,7 +82,9 @@ async function summarizeStreamMemories(memories, streamId = null) {if (streamId) {try {input.speech = (await db.getTimeline(streamId, { kind: 'speech', limit: 400 }) || []).slice(-200).map(r => ({ start_sec: Number(r.start_sec) || 0, text: String(r.text || '').slice(0, 2000) }));-- input.sounds = (await db.getTimeline(streamId, { kind: 'sound', limit: 120 }) || []).slice(-60).map(r => ({ start_sec: Number(r.start_sec) || 0, label: String(r.label || '').slice(0, 120), confidence: Number(r.confidence || 0) }));++ // Tagged sounds above their bar only, by tag (sound-captions.evidence): the long tail is mostly wrong.++ input.sounds = require('./sound-captions').evidence((await db.getTimeline(streamId, { kind: 'sound', limit: 120 }) || []).slice(-60)++ .map(r => ({ start_sec: Number(r.start_sec) || 0, label: String(r.label || '').slice(0, 120), confidence: Number(r.confidence || 0) })));} catch { /* timeline optional */ }}// The same run also classifies the stream: the category is inferred from what is actually on@@@ -25,6 +25,7 @@ const clipPolicy = require('./clip-policy');const dedupe = require('./clip-dedupe');const explicit = require('./explicit');const clipScore = require('./clip-score');++const soundCaptions = require('./sound-captions');let cfg = null; try { cfg = require('../config'); } catch { /* */ }let thumb = null; try { thumb = require('../media-proxy/live-thumbs'); } catch { /* optional */ }const FRAMES_TMP_DIR = path.join(os.tmpdir(), 'openvibe-live-moments');@@@ -170,11 +171,14 @@ async function _momentContext(streamId, vodId) {// Background music (audio-events.js spans) is the stream's bed, not a moment..filter(e => Number(e.confidence || 0) >= 0.45 && e.label !== 'Music');} catch { /* timeline optional */ }-- const sounds = _sample(heard, 30);// The stream's average chat per 30 s, empty stretches counted (C07/C08: averaging only the busy buckets made every// spike look small on a quiet channel and every lull look big on a busy one).let chatAvg = 0;const spanSec = started > 0 ? Math.max(0, ((ms(span.ended_at) || Date.now()) - started) / 1000) : 0;++ // Only sounds with a tag above its bar, said by that tag, reach the prompts below (sound-captions.evidence): the++ // detector's long tail ("Animal", "Vehicle", "Livestock") is mostly wrong and once titled a trading stream's moment.++ heard = soundCaptions.evidence(heard, { spanSec });++ const sounds = _sample(heard, 30);try {if (spanSec >= 60) chatAvg = (Number((await chatReads.streamStats(streamId) || {}).messages) || 0) / (spanSec / 30);} catch { /* chat optional */ }@@@ -24,6 +24,7 @@ const crypto = require('crypto');const db = require('../../db/database');const chatReads = require('../../chat/chat-reads');const insightClient = require('../../chat/insight-client');++const soundCaptions = require('../sound-captions');function clip(str, n) { return (str == null ? '' : String(str)).replace(/\s+/g, ' ').trim().slice(0, n); }function clipWords(str, n) { const t = clip(str, n * 8); return t.length > n ? t.slice(0, n).replace(/\s+\S*$/, '') + '…' : t; }@@@ -143,13 +144,15 @@ async function heardBlock(stream, settings) {let lastEnd = null;for (const r of rows) {if (r.kind === 'speech') { const t = scrubSpeech(r.text); if (hasWords(t)) { speech.push(`[${mmss(r.start_sec)}] ${t}`); lastEnd = r.end_sec || r.start_sec; } }-- else if (r.kind === 'sound' && r.label) sounds.push({ label: r.label, at: r.start_sec, confidence: r.confidence });++ else if (r.kind === 'sound' && r.label) sounds.push({ label: r.label, start_sec: r.start_sec, confidence: r.confidence });}++ // Only tagged sounds above their bar, by tag (sound-captions.evidence); the detector's long tail is mostly wrong.++ const heardSounds = soundCaptions.evidence(sounds, { spanSec: settings.hear_window_sec || 90 }).map((e) => ({ label: e.label, at: e.start_sec, confidence: e.confidence }));let text = speech.join('\n');if (text.length > 900) text = text.slice(-900).replace(/^[^\[]*/, '');-- const uniqSounds = [...new Set(sounds.map(s => s.label))].slice(0, 6);++ const uniqSounds = [...new Set(heardSounds.map(s => s.label))].slice(0, 6);const ageMs = lastEnd != null ? Math.max(0, (off - lastEnd) * 1000) : null;-- return { text, ageMs, count: speech.length, sounds: uniqSounds, latestSound: sounds.length ? sounds[sounds.length - 1] : null };++ return { text, ageMs, count: speech.length, sounds: uniqSounds, latestSound: heardSounds.length ? heardSounds[heardSounds.length - 1] : null };}async function seenBlock(stream) {@@@ -81,4 +81,31 @@ function cues(events, { spanSec = 0 } = {}) {return out.map((c) => ({ start: Math.round(c.start * 100) / 100, end: Math.round(c.end * 100) / 100, text: c.text }));}--module.exports = { cues, tagOf, TAGS };++/**++ * The sounds an AI prompt may hear (2026-11-07: a rare misheard "Livestock" in a trading stream became the moment++ * "Live chaos: farm noises clash"). Only a label with a tag above that tag's bar passes, a rare-only tag (explosion,++ * gunfire) only where it is rare in the stream, and each is said by its tag ("laughter", not "Chuckle, chortle"), so++ * the long tail never reaches a model. `spanSec` defaults to the events' own time range.++ *++ * evidence(events, { spanSec }) → [{ ...event, label: tag }]++ */++function evidence(events, { spanSec } = {}) {++ const list = (Array.isArray(events) ? events : []).filter((e) => e && e.label && e.label !== 'Music');++ if (!list.length) return [];++ let span = Number(spanSec);++ if (!Number.isFinite(span) || span <= 0) {++ const times = list.map((e) => Number(e.start_sec) || 0);++ span = Math.max(...times) - Math.min(...times);++ }++ const rare = new Set(clipScore.rareSounds(list, span).map((e) => String(e.label || '')));++ const out = [];++ for (const e of list) {++ const t = tagOf(e.label);++ if (!t || Number(e.confidence || 0) < t.min) continue;++ if (t.rareOnly && !rare.has(String(e.label))) continue;++ out.push({ ...e, label: t.tag });++ }++ return out;++}++++module.exports = { cues, tagOf, evidence, TAGS };@@@ -41,7 +41,7 @@ console.warn = () => {};assert.ok(m, 'a clip moment is found');assert.strictEqual(m.offset, 3000, 'the laugh, not the first explosion');assert.strictEqual(m.evidence.picked_by, 'sound');-- assert.deepStrictEqual(m.evidence.sounds.map((e) => e.label), ['Laughter'], 'the explosions nearby are not evidence');++ assert.deepStrictEqual(m.evidence.sounds.map((e) => e.label), ['laughter'], 'the explosions nearby are not evidence; the laugh is said by its tag');db.getTimeline = async () => evening.filter((e) => e.label === 'Explosion');assert.strictEqual(await moments.findBestMoment(v, { flavor: 'clip' }), null, 'a stream of nothing but its background gives no clip');@@@ -22,4 +22,14 @@ assert.deepStrictEqual(sc.cues([ev(50, 'Explosion', 0.9, 51), ev(70, 'Gunshot, g[{ start: 50, end: 51.5, text: '[explosion]' }], 'one rare explosion is news; gunfire below 0.85 is not captioned');assert.deepStrictEqual(sc.cues(null), []);assert.strictEqual(sc.tagOf('Dog').tag, 'dog barking');++++// What a model may hear (evidence): 2026-11-07, a rare misheard "Livestock" in a trading stream became the moment++// "Live chaos: farm noises clash". The long tail never passes; a tag passes above its bar, by its tag; a rare-only tag++// only where it is rare; music is the bed, not a sound.++const heard = sc.evidence([ev(10, 'Livestock, farm animals, working animals', 0.7), ev(12, 'Animal', 0.9), ev(20, 'Chuckle, chortle', 0.8),++ ev(30, 'Laughter', 0.5), ev(40, 'Explosion', 0.9), ev(45, 'Music', 0.9)], { spanSec: 3600 });++assert.deepStrictEqual(heard.map((e) => [e.start_sec, e.label]), [[20, 'laughter'], [40, 'explosion']], 'only tagged sounds above their bar, by tag');++assert.deepStrictEqual(sc.evidence(shooter, { spanSec: 3600 }), [], 'a shooter\'s constant explosions are not evidence');++assert.deepStrictEqual(sc.evidence(null), []);++assert.strictEqual(sc.evidence([ev(5, 'Applause', 0.8)]).length, 1, 'no span given: the events\' own range');console.log('sound captions: all checks passed');@@@ -155,7 +155,7 @@ console.warn = () => {};assert.strictEqual(clip.evidence.chatters, 5);assert.ok(clip.evidence.spike.avg > 0.9 && clip.evidence.spike.avg < 1.1, 'the average counts the quiet stretches: 240 messages over 240 buckets');assert.ok(clip.evidence.speechWords >= 10, 'the words come from the whole transcript near the moment, not a sample');-- assert.strictEqual(clip.evidence.sounds[0].label, 'Laughter');++ assert.strictEqual(clip.evidence.sounds[0].label, 'laughter', 'the sound is said by its tag (sound-captions.evidence)');// ── The AI may abstain: no weaker signal is tried in its place ──ai.isEnabled = async () => true; ai.withinBudget = async () => true;
@@ -82,7 +82,9 @@ async function summarizeStreamMemories(memories, streamId = null) {
@@ -82,7 +82,9 @@ async function summarizeStreamMemories(memories, streamId = null) {
if (streamId) {
if (streamId) {
try {
try {
input.speech = (await db.getTimeline(streamId, { kind: 'speech', limit: 400 }) || []).slice(-200).map(r => ({ start_sec: Number(r.start_sec) || 0, text: String(r.text || '').slice(0, 2000) }));
input.speech = (await db.getTimeline(streamId, { kind: 'speech', limit: 400 }) || []).slice(-200).map(r => ({ start_sec: Number(r.start_sec) || 0, text: String(r.text || '').slice(0, 2000) }));
- input.sounds = (await db.getTimeline(streamId, { kind: 'sound', limit: 120 }) || []).slice(-60).map(r => ({ start_sec: Number(r.start_sec) || 0, label: String(r.label || '').slice(0, 120), confidence: Number(r.confidence || 0) }));
+ // Tagged sounds above their bar only, by tag (sound-captions.evidence): the long tail is mostly wrong.
+ input.sounds = require('./sound-captions').evidence((await db.getTimeline(streamId, { kind: 'sound', limit: 120 }) || []).slice(-60)
+ .map(r => ({ start_sec: Number(r.start_sec) || 0, label: String(r.label || '').slice(0, 120), confidence: Number(r.confidence || 0) })));
} catch { /* timeline optional */ }
} catch { /* timeline optional */ }
}
}
// The same run also classifies the stream: the category is inferred from what is actually on
// The same run also classifies the stream: the category is inferred from what is actually on
@@ -25,6 +25,7 @@ const clipPolicy = require('./clip-policy');
@@ -25,6 +25,7 @@ const clipPolicy = require('./clip-policy');
const dedupe = require('./clip-dedupe');
const dedupe = require('./clip-dedupe');
const explicit = require('./explicit');
const explicit = require('./explicit');
const clipScore = require('./clip-score');
const clipScore = require('./clip-score');
+const soundCaptions = require('./sound-captions');
let cfg = null; try { cfg = require('../config'); } catch { /* */ }
let cfg = null; try { cfg = require('../config'); } catch { /* */ }
let thumb = null; try { thumb = require('../media-proxy/live-thumbs'); } catch { /* optional */ }
let thumb = null; try { thumb = require('../media-proxy/live-thumbs'); } catch { /* optional */ }
const FRAMES_TMP_DIR = path.join(os.tmpdir(), 'openvibe-live-moments');
const FRAMES_TMP_DIR = path.join(os.tmpdir(), 'openvibe-live-moments');
@@ -170,11 +171,14 @@ async function _momentContext(streamId, vodId) {
@@ -170,11 +171,14 @@ async function _momentContext(streamId, vodId) {
// Background music (audio-events.js spans) is the stream's bed, not a moment.
// Background music (audio-events.js spans) is the stream's bed, not a moment.
.filter(e => Number(e.confidence || 0) >= 0.45 && e.label !== 'Music');
.filter(e => Number(e.confidence || 0) >= 0.45 && e.label !== 'Music');
} catch { /* timeline optional */ }
} catch { /* timeline optional */ }
- const sounds = _sample(heard, 30);
// The stream's average chat per 30 s, empty stretches counted (C07/C08: averaging only the busy buckets made every
// The stream's average chat per 30 s, empty stretches counted (C07/C08: averaging only the busy buckets made every
// spike look small on a quiet channel and every lull look big on a busy one).
// spike look small on a quiet channel and every lull look big on a busy one).
let chatAvg = 0;
let chatAvg = 0;
const spanSec = started > 0 ? Math.max(0, ((ms(span.ended_at) || Date.now()) - started) / 1000) : 0;
const spanSec = started > 0 ? Math.max(0, ((ms(span.ended_at) || Date.now()) - started) / 1000) : 0;
+ // Only sounds with a tag above its bar, said by that tag, reach the prompts below (sound-captions.evidence): the
+ // detector's long tail ("Animal", "Vehicle", "Livestock") is mostly wrong and once titled a trading stream's moment.
+ heard = soundCaptions.evidence(heard, { spanSec });
+ const sounds = _sample(heard, 30);
try {
try {
if (spanSec >= 60) chatAvg = (Number((await chatReads.streamStats(streamId) || {}).messages) || 0) / (spanSec / 30);
if (spanSec >= 60) chatAvg = (Number((await chatReads.streamStats(streamId) || {}).messages) || 0) / (spanSec / 30);
} catch { /* chat optional */ }
} catch { /* chat optional */ }
@@ -24,6 +24,7 @@ const crypto = require('crypto');
@@ -24,6 +24,7 @@ const crypto = require('crypto');
const db = require('../../db/database');
const db = require('../../db/database');
const chatReads = require('../../chat/chat-reads');
const chatReads = require('../../chat/chat-reads');
const insightClient = require('../../chat/insight-client');
const insightClient = require('../../chat/insight-client');
+const soundCaptions = require('../sound-captions');
function clip(str, n) { return (str == null ? '' : String(str)).replace(/\s+/g, ' ').trim().slice(0, n); }
function clip(str, n) { return (str == null ? '' : String(str)).replace(/\s+/g, ' ').trim().slice(0, n); }
function clipWords(str, n) { const t = clip(str, n * 8); return t.length > n ? t.slice(0, n).replace(/\s+\S*$/, '') + '…' : t; }
function clipWords(str, n) { const t = clip(str, n * 8); return t.length > n ? t.slice(0, n).replace(/\s+\S*$/, '') + '…' : t; }
@@ -143,13 +144,15 @@ async function heardBlock(stream, settings) {
@@ -143,13 +144,15 @@ async function heardBlock(stream, settings) {
let lastEnd = null;
let lastEnd = null;
for (const r of rows) {
for (const r of rows) {
if (r.kind === 'speech') { const t = scrubSpeech(r.text); if (hasWords(t)) { speech.push(`[${mmss(r.start_sec)}] ${t}`); lastEnd = r.end_sec || r.start_sec; } }
if (r.kind === 'speech') { const t = scrubSpeech(r.text); if (hasWords(t)) { speech.push(`[${mmss(r.start_sec)}] ${t}`); lastEnd = r.end_sec || r.start_sec; } }
- else if (r.kind === 'sound' && r.label) sounds.push({ label: r.label, at: r.start_sec, confidence: r.confidence });
+ else if (r.kind === 'sound' && r.label) sounds.push({ label: r.label, start_sec: r.start_sec, confidence: r.confidence });
}
}
+ // Only tagged sounds above their bar, by tag (sound-captions.evidence); the detector's long tail is mostly wrong.
+ const heardSounds = soundCaptions.evidence(sounds, { spanSec: settings.hear_window_sec || 90 }).map((e) => ({ label: e.label, at: e.start_sec, confidence: e.confidence }));
let text = speech.join('\n');
let text = speech.join('\n');
if (text.length > 900) text = text.slice(-900).replace(/^[^\[]*/, '');
if (text.length > 900) text = text.slice(-900).replace(/^[^\[]*/, '');
- const uniqSounds = [...new Set(sounds.map(s => s.label))].slice(0, 6);
+ const uniqSounds = [...new Set(heardSounds.map(s => s.label))].slice(0, 6);
const ageMs = lastEnd != null ? Math.max(0, (off - lastEnd) * 1000) : null;
const ageMs = lastEnd != null ? Math.max(0, (off - lastEnd) * 1000) : null;
- return { text, ageMs, count: speech.length, sounds: uniqSounds, latestSound: sounds.length ? sounds[sounds.length - 1] : null };
+ return { text, ageMs, count: speech.length, sounds: uniqSounds, latestSound: heardSounds.length ? heardSounds[heardSounds.length - 1] : null };
}
}
async function seenBlock(stream) {
async function seenBlock(stream) {
@@ -81,4 +81,31 @@ function cues(events, { spanSec = 0 } = {}) {
@@ -81,4 +81,31 @@ function cues(events, { spanSec = 0 } = {}) {
return out.map((c) => ({ start: Math.round(c.start * 100) / 100, end: Math.round(c.end * 100) / 100, text: c.text }));
return out.map((c) => ({ start: Math.round(c.start * 100) / 100, end: Math.round(c.end * 100) / 100, text: c.text }));
}
}
-module.exports = { cues, tagOf, TAGS };
+/**
+ * The sounds an AI prompt may hear (2026-11-07: a rare misheard "Livestock" in a trading stream became the moment
+ * "Live chaos: farm noises clash"). Only a label with a tag above that tag's bar passes, a rare-only tag (explosion,
+ * gunfire) only where it is rare in the stream, and each is said by its tag ("laughter", not "Chuckle, chortle"), so
+ * the long tail never reaches a model. `spanSec` defaults to the events' own time range.
+ *
+ * evidence(events, { spanSec }) → [{ ...event, label: tag }]
+ */
+function evidence(events, { spanSec } = {}) {
+ const list = (Array.isArray(events) ? events : []).filter((e) => e && e.label && e.label !== 'Music');
+ if (!list.length) return [];
+ let span = Number(spanSec);
+ if (!Number.isFinite(span) || span <= 0) {
+ const times = list.map((e) => Number(e.start_sec) || 0);
+ span = Math.max(...times) - Math.min(...times);
+ }
+ const rare = new Set(clipScore.rareSounds(list, span).map((e) => String(e.label || '')));
+ const out = [];
+ for (const e of list) {
+ const t = tagOf(e.label);
+ if (!t || Number(e.confidence || 0) < t.min) continue;
+ if (t.rareOnly && !rare.has(String(e.label))) continue;
+ out.push({ ...e, label: t.tag });
+ }
+ return out;
+}
+
+module.exports = { cues, tagOf, evidence, TAGS };
@@ -41,7 +41,7 @@ console.warn = () => {};
@@ -41,7 +41,7 @@ console.warn = () => {};
assert.ok(m, 'a clip moment is found');
assert.ok(m, 'a clip moment is found');
assert.strictEqual(m.offset, 3000, 'the laugh, not the first explosion');
assert.strictEqual(m.offset, 3000, 'the laugh, not the first explosion');
assert.strictEqual(m.evidence.picked_by, 'sound');
assert.strictEqual(m.evidence.picked_by, 'sound');
- assert.deepStrictEqual(m.evidence.sounds.map((e) => e.label), ['Laughter'], 'the explosions nearby are not evidence');
+ assert.deepStrictEqual(m.evidence.sounds.map((e) => e.label), ['laughter'], 'the explosions nearby are not evidence; the laugh is said by its tag');
db.getTimeline = async () => evening.filter((e) => e.label === 'Explosion');
db.getTimeline = async () => evening.filter((e) => e.label === 'Explosion');
assert.strictEqual(await moments.findBestMoment(v, { flavor: 'clip' }), null, 'a stream of nothing but its background gives no clip');
assert.strictEqual(await moments.findBestMoment(v, { flavor: 'clip' }), null, 'a stream of nothing but its background gives no clip');
@@ -22,4 +22,14 @@ assert.deepStrictEqual(sc.cues([ev(50, 'Explosion', 0.9, 51), ev(70, 'Gunshot, g
@@ -22,4 +22,14 @@ assert.deepStrictEqual(sc.cues([ev(50, 'Explosion', 0.9, 51), ev(70, 'Gunshot, g
[{ start: 50, end: 51.5, text: '[explosion]' }], 'one rare explosion is news; gunfire below 0.85 is not captioned');
[{ start: 50, end: 51.5, text: '[explosion]' }], 'one rare explosion is news; gunfire below 0.85 is not captioned');
assert.deepStrictEqual(sc.cues(null), []);
assert.deepStrictEqual(sc.cues(null), []);
assert.strictEqual(sc.tagOf('Dog').tag, 'dog barking');
assert.strictEqual(sc.tagOf('Dog').tag, 'dog barking');
+
+// What a model may hear (evidence): 2026-11-07, a rare misheard "Livestock" in a trading stream became the moment
+// "Live chaos: farm noises clash". The long tail never passes; a tag passes above its bar, by its tag; a rare-only tag
+// only where it is rare; music is the bed, not a sound.
+const heard = sc.evidence([ev(10, 'Livestock, farm animals, working animals', 0.7), ev(12, 'Animal', 0.9), ev(20, 'Chuckle, chortle', 0.8),
+ ev(30, 'Laughter', 0.5), ev(40, 'Explosion', 0.9), ev(45, 'Music', 0.9)], { spanSec: 3600 });
+assert.deepStrictEqual(heard.map((e) => [e.start_sec, e.label]), [[20, 'laughter'], [40, 'explosion']], 'only tagged sounds above their bar, by tag');
+assert.deepStrictEqual(sc.evidence(shooter, { spanSec: 3600 }), [], 'a shooter\'s constant explosions are not evidence');
+assert.deepStrictEqual(sc.evidence(null), []);
+assert.strictEqual(sc.evidence([ev(5, 'Applause', 0.8)]).length, 1, 'no span given: the events\' own range');
console.log('sound captions: all checks passed');
console.log('sound captions: all checks passed');
@@ -155,7 +155,7 @@ console.warn = () => {};
@@ -155,7 +155,7 @@ console.warn = () => {};
assert.strictEqual(clip.evidence.chatters, 5);
assert.strictEqual(clip.evidence.chatters, 5);
assert.ok(clip.evidence.spike.avg > 0.9 && clip.evidence.spike.avg < 1.1, 'the average counts the quiet stretches: 240 messages over 240 buckets');
assert.ok(clip.evidence.spike.avg > 0.9 && clip.evidence.spike.avg < 1.1, 'the average counts the quiet stretches: 240 messages over 240 buckets');
assert.ok(clip.evidence.speechWords >= 10, 'the words come from the whole transcript near the moment, not a sample');
assert.ok(clip.evidence.speechWords >= 10, 'the words come from the whole transcript near the moment, not a sample');
- assert.strictEqual(clip.evidence.sounds[0].label, 'Laughter');
+ assert.strictEqual(clip.evidence.sounds[0].label, 'laughter', 'the sound is said by its tag (sound-captions.evidence)');
// ── The AI may abstain: no weaker signal is tried in its place ──
// ── The AI may abstain: no weaker signal is tried in its place ──
ai.isEnabled = async () => true; ai.withinBudget = async () => true;
ai.isEnabled = async () => true; ai.withinBudget = async () => true;