openvibers/openvibe.ai Commits

media.analyze on real streams (WS-O task 5, after the first production runs). Three fixes, from a 3-minute stretch of a JapaneseOldGuy VOD with speech 94% of the time but no highlights. Silence is -50 dB, real silence: a quiet stream talks around -40 LUFS, which -35 dB called silent (43%). Only black picture and silence make a window dead air, not a still picture (a talking head, a menu), and a window with speech is never dead. whisper.cpp's detected language (result.language with -l auto) is reported instead of 'auto'. Tests: a quiet talking stream over a still picture gives speech highlights and none in its silent black stretch; a fake whisper-cli reports ja for auto and keeps en.

OP OpenVibers committed 27 Sept 2026 · 2d9a16f856 · 3 files, +34 −6


Changes

ddiff --git a/server/providers/whisper.js b/server/providers/whisper.js
iindex 9bf3f96..2d9b4cb 100644
---- a/server/providers/whisper.js
++++ b/server/providers/whisper.js
@@@ -130,7 +130,9 @@ function createWhisperProvider(record, { config }) {
end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,
text: it.text || '',
})), Boolean(vm));
-- done(() => resolve({ text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: lang, model: path.basename(model) }));
++ // With -l auto, whisper.cpp names the language it detected in result.language.
++ const detected = parsed && parsed.result && /^[a-z]{2,3}$/.test(String(parsed.result.language || '')) ? parsed.result.language : null;
++ done(() => resolve({ text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: lang === 'auto' && detected ? detected : lang, model: path.basename(model) }));
});
ch.on('error', (e) => done(() => reject(new ProviderError(e.message, { code: 'provider.unavailable' }))));
});
ddiff --git a/server/workflows/media-analysis.js b/server/workflows/media-analysis.js
iindex 84c8fae..d59be08 100644
---- a/server/workflows/media-analysis.js
++++ b/server/workflows/media-analysis.js
@@@ -114,7 +114,8 @@ function union(spans) {
/** Windows (30 s, or a quarter of a short clip; every half window) scored by loudness, scene changes and speech; the best that do not overlap. */
function highlightsOf({ duration, signals, perSecond, segments }) {
const I = signals.loudness.typical_lufs;
-- const dead = union([...signals.black, ...signals.frozen, ...signals.silence]);
++ // Black picture and silence make a window dead air; a still picture (a talking head, a menu) does not.
++ const dead = union([...signals.black, ...signals.silence]);
const W = Math.max(5, Math.min(WINDOW, duration / 4));
const cands = [];
for (let s = 0; s < Math.max(duration - W / 2, 0.001); s += W / 2) {
@@@ -130,9 +131,10 @@ function highlightsOf({ duration, signals, perSecond, segments }) {
const cuts = signals.scene_changes.filter((c) => c.t >= s && c.t < e).length;
if (cuts >= 2) { score += Math.min(2, cuts * 0.5); reasons.push('scene changes'); }
const words = segments.reduce((n, g) => n + (overlap(s, e, g.start, g.end) > 0 ? g.text.split(/\s+/).filter(Boolean).length : 0), 0);
-- if (words / len >= 1.5) { score += 1; reasons.push('speech'); }
++ const talking = words / len >= 1.5;
++ if (talking) { score += 1; reasons.push('speech'); }
const still = dead.reduce((n, d) => n + overlap(s, e, d.start, d.end), 0);
-- if (still / len > 0.5 || !score) continue;
++ if ((still / len > 0.5 && !talking) || !score) continue;
const excerpt = segments.filter((g) => overlap(s, e, g.start, g.end) > 0).map((g) => g.text).join(' ').replace(/\s+/g, ' ').trim().slice(0, 300);
cands.push({ start: r2(s), end: r2(e), score: r2(score), reasons, excerpt });
}
@@@ -228,7 +230,8 @@ function createMediaAnalysis({ registry, pool, fetcher, quotas, config = {}, spa
async function signalsOf(file, { duration, streams, seconds, signal }) {
const graph = [];
if (streams.video) graph.push(`[0:v:0]scale=160:-2,scdet=threshold=${ma.sceneThreshold || 10},blackdetect=d=1:pix_th=0.10,freezedetect=n=-60dB:d=4[vo]`);
-- if (streams.audio) graph.push('[0:a:0]silencedetect=n=-35dB:d=2,ebur128[ao]');
++ // -50 dB: real silence. A quiet stream talks at -40 LUFS, which -35 dB would call silent.
++ if (streams.audio) graph.push('[0:a:0]silencedetect=n=-50dB:d=2,ebur128[ao]');
if (!graph.length) return null;
const args = ['-hide_banner', '-nostats', '-nostdin'];
if (seconds) args.push('-t', String(seconds));
ddiff --git a/test/media-analysis.test.js b/test/media-analysis.test.js
iindex 1f8e1dc..c8bb5b3 100644
---- a/test/media-analysis.test.js
++++ b/test/media-analysis.test.js
@@@ -7,7 +7,8 @@
// and the overview from the local model (media.local), which sees only the measured facts and the transcript;
// - the local model down: an extractive overview that says so; paid only with allow_paid AND a media.paid budget,
// never past it; the paid call is the run's provider;
--// - too little free disk refuses before downloading.
++// - too little free disk refuses before downloading;
++// - a quiet stream's talking is not dead air; whisper names the language it detected.
// The runs need ffprobe/ffmpeg on PATH; without them those parts are skipped (the parser part always runs).
const assert = require('assert');
const fs = require('fs');
@@@ -57,6 +58,28 @@ t.test('the parser reads every filter\'s lines; loud means well above the typica
assert.strictEqual(ma.scenesOf(many, 4010).length, 300, 'at most 300 scenes; the strongest changes are the boundaries');
assert.strictEqual(ma.speechRatio([{ start: 0, end: 10, text: 'a' }, { start: 5, end: 12, text: 'b' }], 24), 0.5, 'overlapping speech counts once');
assert.strictEqual(ma.clock(3725), '1:02:05');
++ // A quiet stream: someone talking over a still picture is a highlight, not dead air; a silent black stretch is.
++ const talk = { scene_changes: [], black: [{ start: 60, end: 90 }], frozen: [{ start: 0, end: 60 }], silence: [{ start: 60, end: 90 }], loudness: { typical_lufs: -41, integrated_lufs: -39, peaks: [] } };
++ const segs = Array.from({ length: 12 }, (_, i) => ({ start: i * 5, end: i * 5 + 5, text: 'and then we go over there and look at this one more time okay' }));
++ const hl = ma.highlightsOf({ duration: 90, signals: talk, perSecond: new Map(), segments: segs });
++ assert.ok(hl.length && hl.every((x) => x.reasons.includes('speech') && x.start < 60), JSON.stringify(hl));
++});
++
++t.test('whisper reports the language it detected, not "auto"', async () => {
++ if (!HAS_FFMPEG) return console.log(' (skipped: no ffmpeg/ffprobe)');
++ const dir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'ov-ai-whisper-'));
++ try {
++ const bin = path.join(dir, 'whisper-cli');
++ fs.writeFileSync(bin, `#!/bin/sh\nwhile [ $# -gt 0 ]; do case "$1" in -of) OUT="$2"; shift;; -l) L="$2"; shift;; esac; shift; done\n[ "$L" = auto ] && LANG_OUT=ja || LANG_OUT="$L"\nprintf '{"result":{"language":"%s"},"transcription":[{"offsets":{"from":0,"to":1500},"text":"こんにちは、みなさん"}]}' "$LANG_OUT" > "$OUT.json"\n`, { mode: 0o755 });
++ for (const f of ['base.bin', 'multi.bin']) fs.writeFileSync(path.join(dir, f), 'x');
++ const { createWhisperProvider } = require('../server/providers/whisper');
++ const w = createWhisperProvider({ key: 'whisper' }, { config: { whisper: { bin, model: path.join(dir, 'base.bin'), modelMulti: path.join(dir, 'multi.bin'), vad: false, threads: 1, beam: 1, maxConcurrent: 1 } } });
++ const clip = path.join(dir, 'clip.mp4');
++ fs.writeFileSync(clip, FIXTURE);
++ const r = await w.transcribe({ filePath: clip, language: 'auto', seconds: 3, timeoutMs: 30000 });
++ assert.deepStrictEqual([r.language, r.text], ['ja', 'こんにちは、みなさん']);
++ assert.strictEqual((await w.transcribe({ filePath: clip, language: 'en', seconds: 3, timeoutMs: 30000 })).language, 'en');
++ } finally { fs.rmSync(dir, { recursive: true, force: true }); }
});
t.test('a real run: signals, scenes, highlights with evidence, and the local model\'s overview', async () => {