openvibers/openvibe.ai Commits
media.analyze on real streams (WS-O task 5, after the first production runs). Three fixes, from a 3-minute stretch of a JapaneseOldGuy VOD with speech 94% of the time but no highlights. Silence is -50 dB, real silence: a quiet stream talks around -40 LUFS, which -35 dB called silent (43%). Only black picture and silence make a window dead air, not a still picture (a talking head, a menu), and a window with speech is never dead. whisper.cpp's detected language (result.language with -l auto) is reported instead of 'auto'. Tests: a quiet talking stream over a still picture gives speech highlights and none in its silent black stretch; a fake whisper-cli reports ja for auto and keeps en.
Changes
@@@ -130,7 +130,9 @@ function createWhisperProvider(record, { config }) {end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,text: it.text || '',})), Boolean(vm));-- done(() => resolve({ text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: lang, model: path.basename(model) }));++ // With -l auto, whisper.cpp names the language it detected in result.language.++ const detected = parsed && parsed.result && /^[a-z]{2,3}$/.test(String(parsed.result.language || '')) ? parsed.result.language : null;++ done(() => resolve({ text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: lang === 'auto' && detected ? detected : lang, model: path.basename(model) }));});ch.on('error', (e) => done(() => reject(new ProviderError(e.message, { code: 'provider.unavailable' }))));});@@@ -114,7 +114,8 @@ function union(spans) {/** Windows (30 s, or a quarter of a short clip; every half window) scored by loudness, scene changes and speech; the best that do not overlap. */function highlightsOf({ duration, signals, perSecond, segments }) {const I = signals.loudness.typical_lufs;-- const dead = union([...signals.black, ...signals.frozen, ...signals.silence]);++ // Black picture and silence make a window dead air; a still picture (a talking head, a menu) does not.++ const dead = union([...signals.black, ...signals.silence]);const W = Math.max(5, Math.min(WINDOW, duration / 4));const cands = [];for (let s = 0; s < Math.max(duration - W / 2, 0.001); s += W / 2) {@@@ -130,9 +131,10 @@ function highlightsOf({ duration, signals, perSecond, segments }) {const cuts = signals.scene_changes.filter((c) => c.t >= s && c.t < e).length;if (cuts >= 2) { score += Math.min(2, cuts * 0.5); reasons.push('scene changes'); }const words = segments.reduce((n, g) => n + (overlap(s, e, g.start, g.end) > 0 ? g.text.split(/\s+/).filter(Boolean).length : 0), 0);-- if (words / len >= 1.5) { score += 1; reasons.push('speech'); }++ const talking = words / len >= 1.5;++ if (talking) { score += 1; reasons.push('speech'); }const still = dead.reduce((n, d) => n + overlap(s, e, d.start, d.end), 0);-- if (still / len > 0.5 || !score) continue;++ if ((still / len > 0.5 && !talking) || !score) continue;const excerpt = segments.filter((g) => overlap(s, e, g.start, g.end) > 0).map((g) => g.text).join(' ').replace(/\s+/g, ' ').trim().slice(0, 300);cands.push({ start: r2(s), end: r2(e), score: r2(score), reasons, excerpt });}@@@ -228,7 +230,8 @@ function createMediaAnalysis({ registry, pool, fetcher, quotas, config = {}, spaasync function signalsOf(file, { duration, streams, seconds, signal }) {const graph = [];if (streams.video) graph.push(`[0:v:0]scale=160:-2,scdet=threshold=${ma.sceneThreshold || 10},blackdetect=d=1:pix_th=0.10,freezedetect=n=-60dB:d=4[vo]`);-- if (streams.audio) graph.push('[0:a:0]silencedetect=n=-35dB:d=2,ebur128[ao]');++ // -50 dB: real silence. A quiet stream talks at -40 LUFS, which -35 dB would call silent.++ if (streams.audio) graph.push('[0:a:0]silencedetect=n=-50dB:d=2,ebur128[ao]');if (!graph.length) return null;const args = ['-hide_banner', '-nostats', '-nostdin'];if (seconds) args.push('-t', String(seconds));@@@ -7,7 +7,8 @@// and the overview from the local model (media.local), which sees only the measured facts and the transcript;// - the local model down: an extractive overview that says so; paid only with allow_paid AND a media.paid budget,// never past it; the paid call is the run's provider;--// - too little free disk refuses before downloading.++// - too little free disk refuses before downloading;++// - a quiet stream's talking is not dead air; whisper names the language it detected.// The runs need ffprobe/ffmpeg on PATH; without them those parts are skipped (the parser part always runs).const assert = require('assert');const fs = require('fs');@@@ -57,6 +58,28 @@ t.test('the parser reads every filter\'s lines; loud means well above the typicaassert.strictEqual(ma.scenesOf(many, 4010).length, 300, 'at most 300 scenes; the strongest changes are the boundaries');assert.strictEqual(ma.speechRatio([{ start: 0, end: 10, text: 'a' }, { start: 5, end: 12, text: 'b' }], 24), 0.5, 'overlapping speech counts once');assert.strictEqual(ma.clock(3725), '1:02:05');++ // A quiet stream: someone talking over a still picture is a highlight, not dead air; a silent black stretch is.++ const talk = { scene_changes: [], black: [{ start: 60, end: 90 }], frozen: [{ start: 0, end: 60 }], silence: [{ start: 60, end: 90 }], loudness: { typical_lufs: -41, integrated_lufs: -39, peaks: [] } };++ const segs = Array.from({ length: 12 }, (_, i) => ({ start: i * 5, end: i * 5 + 5, text: 'and then we go over there and look at this one more time okay' }));++ const hl = ma.highlightsOf({ duration: 90, signals: talk, perSecond: new Map(), segments: segs });++ assert.ok(hl.length && hl.every((x) => x.reasons.includes('speech') && x.start < 60), JSON.stringify(hl));++});++++t.test('whisper reports the language it detected, not "auto"', async () => {++ if (!HAS_FFMPEG) return console.log(' (skipped: no ffmpeg/ffprobe)');++ const dir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'ov-ai-whisper-'));++ try {++ const bin = path.join(dir, 'whisper-cli');++ fs.writeFileSync(bin, `#!/bin/sh\nwhile [ $# -gt 0 ]; do case "$1" in -of) OUT="$2"; shift;; -l) L="$2"; shift;; esac; shift; done\n[ "$L" = auto ] && LANG_OUT=ja || LANG_OUT="$L"\nprintf '{"result":{"language":"%s"},"transcription":[{"offsets":{"from":0,"to":1500},"text":"こんにちは、みなさん"}]}' "$LANG_OUT" > "$OUT.json"\n`, { mode: 0o755 });++ for (const f of ['base.bin', 'multi.bin']) fs.writeFileSync(path.join(dir, f), 'x');++ const { createWhisperProvider } = require('../server/providers/whisper');++ const w = createWhisperProvider({ key: 'whisper' }, { config: { whisper: { bin, model: path.join(dir, 'base.bin'), modelMulti: path.join(dir, 'multi.bin'), vad: false, threads: 1, beam: 1, maxConcurrent: 1 } } });++ const clip = path.join(dir, 'clip.mp4');++ fs.writeFileSync(clip, FIXTURE);++ const r = await w.transcribe({ filePath: clip, language: 'auto', seconds: 3, timeoutMs: 30000 });++ assert.deepStrictEqual([r.language, r.text], ['ja', 'こんにちは、みなさん']);++ assert.strictEqual((await w.transcribe({ filePath: clip, language: 'en', seconds: 3, timeoutMs: 30000 })).language, 'en');++ } finally { fs.rmSync(dir, { recursive: true, force: true }); }});t.test('a real run: signals, scenes, highlights with evidence, and the local model\'s overview', async () => {
@@ -130,7 +130,9 @@ function createWhisperProvider(record, { config }) {
@@ -130,7 +130,9 @@ function createWhisperProvider(record, { config }) {
end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,
end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,
text: it.text || '',
text: it.text || '',
})), Boolean(vm));
})), Boolean(vm));
- done(() => resolve({ text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: lang, model: path.basename(model) }));
+ // With -l auto, whisper.cpp names the language it detected in result.language.
+ const detected = parsed && parsed.result && /^[a-z]{2,3}$/.test(String(parsed.result.language || '')) ? parsed.result.language : null;
+ done(() => resolve({ text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: lang === 'auto' && detected ? detected : lang, model: path.basename(model) }));
});
});
ch.on('error', (e) => done(() => reject(new ProviderError(e.message, { code: 'provider.unavailable' }))));
ch.on('error', (e) => done(() => reject(new ProviderError(e.message, { code: 'provider.unavailable' }))));
});
});
@@ -114,7 +114,8 @@ function union(spans) {
@@ -114,7 +114,8 @@ function union(spans) {
/** Windows (30 s, or a quarter of a short clip; every half window) scored by loudness, scene changes and speech; the best that do not overlap. */
/** Windows (30 s, or a quarter of a short clip; every half window) scored by loudness, scene changes and speech; the best that do not overlap. */
function highlightsOf({ duration, signals, perSecond, segments }) {
function highlightsOf({ duration, signals, perSecond, segments }) {
const I = signals.loudness.typical_lufs;
const I = signals.loudness.typical_lufs;
- const dead = union([...signals.black, ...signals.frozen, ...signals.silence]);
+ // Black picture and silence make a window dead air; a still picture (a talking head, a menu) does not.
+ const dead = union([...signals.black, ...signals.silence]);
const W = Math.max(5, Math.min(WINDOW, duration / 4));
const W = Math.max(5, Math.min(WINDOW, duration / 4));
const cands = [];
const cands = [];
for (let s = 0; s < Math.max(duration - W / 2, 0.001); s += W / 2) {
for (let s = 0; s < Math.max(duration - W / 2, 0.001); s += W / 2) {
@@ -130,9 +131,10 @@ function highlightsOf({ duration, signals, perSecond, segments }) {
@@ -130,9 +131,10 @@ function highlightsOf({ duration, signals, perSecond, segments }) {
const cuts = signals.scene_changes.filter((c) => c.t >= s && c.t < e).length;
const cuts = signals.scene_changes.filter((c) => c.t >= s && c.t < e).length;
if (cuts >= 2) { score += Math.min(2, cuts * 0.5); reasons.push('scene changes'); }
if (cuts >= 2) { score += Math.min(2, cuts * 0.5); reasons.push('scene changes'); }
const words = segments.reduce((n, g) => n + (overlap(s, e, g.start, g.end) > 0 ? g.text.split(/\s+/).filter(Boolean).length : 0), 0);
const words = segments.reduce((n, g) => n + (overlap(s, e, g.start, g.end) > 0 ? g.text.split(/\s+/).filter(Boolean).length : 0), 0);
- if (words / len >= 1.5) { score += 1; reasons.push('speech'); }
+ const talking = words / len >= 1.5;
+ if (talking) { score += 1; reasons.push('speech'); }
const still = dead.reduce((n, d) => n + overlap(s, e, d.start, d.end), 0);
const still = dead.reduce((n, d) => n + overlap(s, e, d.start, d.end), 0);
- if (still / len > 0.5 || !score) continue;
+ if ((still / len > 0.5 && !talking) || !score) continue;
const excerpt = segments.filter((g) => overlap(s, e, g.start, g.end) > 0).map((g) => g.text).join(' ').replace(/\s+/g, ' ').trim().slice(0, 300);
const excerpt = segments.filter((g) => overlap(s, e, g.start, g.end) > 0).map((g) => g.text).join(' ').replace(/\s+/g, ' ').trim().slice(0, 300);
cands.push({ start: r2(s), end: r2(e), score: r2(score), reasons, excerpt });
cands.push({ start: r2(s), end: r2(e), score: r2(score), reasons, excerpt });
}
}
@@ -228,7 +230,8 @@ function createMediaAnalysis({ registry, pool, fetcher, quotas, config = {}, spa
@@ -228,7 +230,8 @@ function createMediaAnalysis({ registry, pool, fetcher, quotas, config = {}, spa
async function signalsOf(file, { duration, streams, seconds, signal }) {
async function signalsOf(file, { duration, streams, seconds, signal }) {
const graph = [];
const graph = [];
if (streams.video) graph.push(`[0:v:0]scale=160:-2,scdet=threshold=${ma.sceneThreshold || 10},blackdetect=d=1:pix_th=0.10,freezedetect=n=-60dB:d=4[vo]`);
if (streams.video) graph.push(`[0:v:0]scale=160:-2,scdet=threshold=${ma.sceneThreshold || 10},blackdetect=d=1:pix_th=0.10,freezedetect=n=-60dB:d=4[vo]`);
- if (streams.audio) graph.push('[0:a:0]silencedetect=n=-35dB:d=2,ebur128[ao]');
+ // -50 dB: real silence. A quiet stream talks at -40 LUFS, which -35 dB would call silent.
+ if (streams.audio) graph.push('[0:a:0]silencedetect=n=-50dB:d=2,ebur128[ao]');
if (!graph.length) return null;
if (!graph.length) return null;
const args = ['-hide_banner', '-nostats', '-nostdin'];
const args = ['-hide_banner', '-nostats', '-nostdin'];
if (seconds) args.push('-t', String(seconds));
if (seconds) args.push('-t', String(seconds));
@@ -7,7 +7,8 @@
@@ -7,7 +7,8 @@
// and the overview from the local model (media.local), which sees only the measured facts and the transcript;
// and the overview from the local model (media.local), which sees only the measured facts and the transcript;
// - the local model down: an extractive overview that says so; paid only with allow_paid AND a media.paid budget,
// - the local model down: an extractive overview that says so; paid only with allow_paid AND a media.paid budget,
// never past it; the paid call is the run's provider;
// never past it; the paid call is the run's provider;
-// - too little free disk refuses before downloading.
+// - too little free disk refuses before downloading;
+// - a quiet stream's talking is not dead air; whisper names the language it detected.
// The runs need ffprobe/ffmpeg on PATH; without them those parts are skipped (the parser part always runs).
// The runs need ffprobe/ffmpeg on PATH; without them those parts are skipped (the parser part always runs).
const assert = require('assert');
const assert = require('assert');
const fs = require('fs');
const fs = require('fs');
@@ -57,6 +58,28 @@ t.test('the parser reads every filter\'s lines; loud means well above the typica
@@ -57,6 +58,28 @@ t.test('the parser reads every filter\'s lines; loud means well above the typica
assert.strictEqual(ma.scenesOf(many, 4010).length, 300, 'at most 300 scenes; the strongest changes are the boundaries');
assert.strictEqual(ma.scenesOf(many, 4010).length, 300, 'at most 300 scenes; the strongest changes are the boundaries');
assert.strictEqual(ma.speechRatio([{ start: 0, end: 10, text: 'a' }, { start: 5, end: 12, text: 'b' }], 24), 0.5, 'overlapping speech counts once');
assert.strictEqual(ma.speechRatio([{ start: 0, end: 10, text: 'a' }, { start: 5, end: 12, text: 'b' }], 24), 0.5, 'overlapping speech counts once');
assert.strictEqual(ma.clock(3725), '1:02:05');
assert.strictEqual(ma.clock(3725), '1:02:05');
+ // A quiet stream: someone talking over a still picture is a highlight, not dead air; a silent black stretch is.
+ const talk = { scene_changes: [], black: [{ start: 60, end: 90 }], frozen: [{ start: 0, end: 60 }], silence: [{ start: 60, end: 90 }], loudness: { typical_lufs: -41, integrated_lufs: -39, peaks: [] } };
+ const segs = Array.from({ length: 12 }, (_, i) => ({ start: i * 5, end: i * 5 + 5, text: 'and then we go over there and look at this one more time okay' }));
+ const hl = ma.highlightsOf({ duration: 90, signals: talk, perSecond: new Map(), segments: segs });
+ assert.ok(hl.length && hl.every((x) => x.reasons.includes('speech') && x.start < 60), JSON.stringify(hl));
+});
+
+t.test('whisper reports the language it detected, not "auto"', async () => {
+ if (!HAS_FFMPEG) return console.log(' (skipped: no ffmpeg/ffprobe)');
+ const dir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'ov-ai-whisper-'));
+ try {
+ const bin = path.join(dir, 'whisper-cli');
+ fs.writeFileSync(bin, `#!/bin/sh\nwhile [ $# -gt 0 ]; do case "$1" in -of) OUT="$2"; shift;; -l) L="$2"; shift;; esac; shift; done\n[ "$L" = auto ] && LANG_OUT=ja || LANG_OUT="$L"\nprintf '{"result":{"language":"%s"},"transcription":[{"offsets":{"from":0,"to":1500},"text":"こんにちは、みなさん"}]}' "$LANG_OUT" > "$OUT.json"\n`, { mode: 0o755 });
+ for (const f of ['base.bin', 'multi.bin']) fs.writeFileSync(path.join(dir, f), 'x');
+ const { createWhisperProvider } = require('../server/providers/whisper');
+ const w = createWhisperProvider({ key: 'whisper' }, { config: { whisper: { bin, model: path.join(dir, 'base.bin'), modelMulti: path.join(dir, 'multi.bin'), vad: false, threads: 1, beam: 1, maxConcurrent: 1 } } });
+ const clip = path.join(dir, 'clip.mp4');
+ fs.writeFileSync(clip, FIXTURE);
+ const r = await w.transcribe({ filePath: clip, language: 'auto', seconds: 3, timeoutMs: 30000 });
+ assert.deepStrictEqual([r.language, r.text], ['ja', 'こんにちは、みなさん']);
+ assert.strictEqual((await w.transcribe({ filePath: clip, language: 'en', seconds: 3, timeoutMs: 30000 })).language, 'en');
+ } finally { fs.rmSync(dir, { recursive: true, force: true }); }
});
});
t.test('a real run: signals, scenes, highlights with evidence, and the local model\'s overview', async () => {
t.test('a real run: signals, scenes, highlights with evidence, and the local model\'s overview', async () => {