openvibers/openvibe.live Commits

Live: a recording's transcript stops at a window AI cannot do yet and retries from there, instead of keeping a hole (#123)

OP OpenVibers committed 29 days ago · 5cc0aef37b · 4 files, +102 −5

Changes

ddiff --git a/server/ai/ai-analysis.js b/server/ai/ai-analysis.js
iindex 26a35842..7e792198 100644
---- a/server/ai/ai-analysis.js
++++ b/server/ai/ai-analysis.js
@@@ -289,7 +289,15 @@ async function generateVodTranscript(vod) {
onWindow: async (sec, segs) => await db.saveVodTranscriptProgress(vod.id, sec, segs),
});
} catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
-- if (r.text) { // got speech → store it (+segments)
++ if (r.partial) { // stopped at a window AI could not do yet
++ const n = await db.bumpVodTranscriptAttempt(vod.id);
++ if (n >= MAX_TX_ATTEMPTS && r.text) { // the last attempt: keep what came before the gap
++ try { await db.setVodTranscript(vod.id, r.text, r.segments || []); } catch { /* */ }
++ await db.setVodTranscriptStatus(vod.id, 'done', `transcribed up to ${Math.round((r.stoppedAtSec || 0) / 60)} min: ${r.error}`);
++ } else { // retry later from that window (progress is saved)
++ await db.setVodTranscriptStatus(vod.id, n >= MAX_TX_ATTEMPTS ? 'failed' : 'retry', r.error || 'transcription stopped', _txBackoffMin(n));
++ }
++ } else if (r.text) { // got speech → store it (+segments)
try { await db.setVodTranscript(vod.id, r.text, r.segments || []); } catch { /* */ }
await db.setVodTranscriptStatus(vod.id, 'done');
} else if (r.ok) { // ran clean, genuinely no speech → terminal
ddiff --git a/server/ai/media-analysis.js b/server/ai/media-analysis.js
iindex 4596df90..60606db4 100644
---- a/server/ai/media-analysis.js
++++ b/server/ai/media-analysis.js
@@@ -226,6 +226,10 @@ async function _aiWindow(src, start, len) {
return { ok: true, segments: (out.segments || []).filter(g => g && g.text).map(g => ({ start: Number(g.start) || 0, end: Number(g.end) || 0, text: String(g.text) })), error: null };
}
++// AI errors that a later attempt will not fix: anything else (its whisper down, a run interrupted, a timeout) is worth
++// waiting out rather than skipping the window.
++const PERMANENT_AI_ERRORS = new Set(['media.unreadable', 'media.not_found', 'media.too_long', 'validation.failed']);
++
async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments = null, onWindow = null } = {}) {
const transcribe = require('./transcribe');
// Recordings on Media are transcribed by OpenVibe.AI; only a local file (legacy) is still decoded here.
@@@ -264,6 +268,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
console.warn(`[AI] VOD is ${Math.round(total / 60)}min — transcribing the first ${MAX_WINDOWS * WINDOW_SEC / 60}min only`);
windows.length = MAX_WINDOWS;
}
++ let stopped = null;
for (const start of windows) {
const len = Math.min(WINDOW_SEC, total - start);
if (viaAi) {
@@@ -273,6 +278,10 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
if (w.ok && typeof onWindow === 'function') {
try { await onWindow(Math.min(total, start + len), segments); } catch { /* progress is best-effort */ }
}
++ // A window AI cannot do right now (its whisper's circuit open, the run interrupted): stop here. The caller
++ // retries later from this window (progress is saved per window), instead of storing a transcript with a
++ // hole where this window was (10-14: 46 windows of one recording failed in the same second and were skipped).
++ if (!w.ok && !PERMANENT_AI_ERRORS.has(w.error)) { stopped = { atSec: start, error: w.error || 'OpenVibe.AI did not transcribe' }; break; }
continue;
}
const wav = _tmp('wav');
@@@ -295,7 +304,10 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
// retries instead of marking the VOD permanently silent.
// Windows are contiguous now, so the transcript reads as continuous prose rather than
// three disconnected excerpts joined by an ellipsis.
-- return { text: parts.join(' ').replace(/\s+/g, ' ').trim(), segments, ok: anyOk, error: anyOk ? null : lastErr };
++ const text = parts.join(' ').replace(/\s+/g, ' ').trim();
++ // partial: stopped at a window worth retrying; the text so far is what came before it.
++ if (stopped) return { text, segments, ok: false, partial: true, stoppedAtSec: stopped.atSec, error: stopped.error };
++ return { text, segments, ok: anyOk, error: anyOk ? null : lastErr };
}
/**
ddiff --git a/test/transcript-ai.test.js b/test/transcript-ai.test.js
iindex dbb620d7..c9226b9a 100644
---- a/test/transcript-ai.test.js
++++ b/test/transcript-ai.test.js
@@@ -31,6 +31,7 @@ spawnSync('ffmpeg', ['-y', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=610
const keys = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 });
const runs = [];
let failAt = null;
++let failCode = 'provider.unavailable';
const network = http.createServer((req, res) => {
let raw = ''; req.on('data', (c) => { raw += c; });
req.on('end', () => {
@@@ -46,7 +47,7 @@ const ai = http.createServer((req, res) => {
runs.push(body);
res.setHeader('Content-Type', 'application/json');
const s = body.input.start_sec;
-- if (s === failAt) { res.statusCode = 201; return res.end(JSON.stringify({ run: { id: 'run_x', status: 'failed', error: { code: 'provider.error' } } })); }
++ if (s === failAt) { res.statusCode = 201; return res.end(JSON.stringify({ run: { id: 'run_x', status: 'failed', error: { code: failCode } } })); }
res.statusCode = 201;
res.end(JSON.stringify({ run: { id: 'run_01JAB2C3D4E5F6G7H8J9K0MNPA', status: 'succeeded', synthetic: false, workflow: { key: body.workflow }, output: { text: `words at ${s}`, language: 'en', segments: [{ start: s + 1.5, end: s + 4, text: `words at ${s}` }] }, usage: { tokens_in: 0, tokens_out: 0, cost_usd: 0 } } }));
});
@@@ -79,10 +80,20 @@ const media = http.createServer((req, res) => {
r = await ma.transcribeOnly(url, { resumeFromSec: 600, priorSegments: [{ start: 1, end: 2, text: 'kept' }] });
assert.deepStrictEqual(runs.map((x) => [x.input.start_sec, x.input.low_power]), [[600, false]]);
assert.strictEqual(r.text, 'kept words at 600');
-- // A failed window is reported; the others still count.
++ // A window AI cannot do right now (its whisper down): the pass stops there instead of leaving a hole, and says
++ // where, so the retry resumes from that window.
runs.length = 0; failAt = 300;
++ const saved = [];
++ r = await ma.transcribeOnly(url, { onWindow: (sec) => saved.push(sec) });
++ assert.deepStrictEqual(runs.map((x) => x.input.start_sec), [0, 300], 'nothing after the failed window is tried');
++ assert.deepStrictEqual([r.ok, r.partial, r.stoppedAtSec, r.error, r.text, saved], [false, true, 300, 'provider.unavailable', 'words at 0', [300]]);
++ runs.length = 0; failAt = null;
++ r = await ma.transcribeOnly(url, { resumeFromSec: 300, priorSegments: r.segments });
++ assert.deepStrictEqual([runs.map((x) => x.input.start_sec), r.ok, r.text], [[300, 600], true, 'words at 0 words at 300 words at 600'], 'the retry fills the gap');
++ // An error a retry will not fix (the media unreadable there) is reported and skipped; the others still count.
++ runs.length = 0; failAt = 300; failCode = 'media.unreadable';
r = await ma.transcribeOnly(url);
-- assert.deepStrictEqual([r.ok, r.segments.length], [true, 2]);
++ assert.deepStrictEqual([r.ok, r.partial, r.segments.length], [true, undefined, 2]);
} finally {
for (const s of [network, ai, media]) s.close();
fs.rmSync(tmp, { recursive: true, force: true });
ddiff --git a/test/transcript-partial.test.js b/test/transcript-partial.test.js
nnew file mode 100644
iindex 00000000..c63aed3a
---- /dev/null
++++ b/test/transcript-partial.test.js
@@@ -0,0 +1,66 @@
++'use strict';
++// A VOD transcript that stopped at a window OpenVibe.AI could not do yet (its whisper down) is retried later from that
++// window, never stored with a hole in it; only the last attempt keeps what came before the gap. Stubs only (no
++// database, no AI): the transcript pass's db calls and media-analysis.transcribeOnly are replaced on their modules.
++// node test/transcript-partial.test.js
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++
++process.env.NODE_ENV = 'test';
++console.warn = () => {};
++const db = require('../server/db/database');
++const ma = require('../server/ai/media-analysis');
++const tx = require('../server/ai/transcribe');
++const aiAnalysis = require('../server/ai/ai-analysis');
++
++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-live-txp-'));
++const file = path.join(tmp, 'rec.mp4');
++fs.writeFileSync(file, 'x');
++
++const calls = [];
++let attempts = 0;
++Object.assign(db, {
++ getSetting: async () => undefined,
++ getLiveStreams: async () => [],
++ getVodTranscriptProgress: async () => ({ progressSec: 0, segments: [] }),
++ saveVodTranscriptProgress: async () => {},
++ bumpVodTranscriptAttempt: async () => ++attempts,
++ setVodTranscriptStatus: async (id, status, error, delay) => { calls.push(['status', status, error || null, delay || 0]); },
++ setVodTranscript: async (id, text) => { calls.push(['transcript', text]); },
++});
++tx.available = () => true;
++const partial = { text: 'words at 0', segments: [{ start: 1, end: 2, text: 'words at 0' }], ok: false, partial: true, stoppedAtSec: 300, error: 'provider.unavailable' };
++ma.transcribeOnly = async () => partial;
++
++(async () => {
++ try {
++ const vod = { id: 42, file_path: file };
++ await aiAnalysis.generateVodTranscript(vod);
++ assert.deepStrictEqual(calls.filter((c) => c[0] !== 'status' || c[1] !== 'processing'), [['status', 'retry', 'provider.unavailable', 1]],
++ 'stopped at a window: retried later, nothing stored');
++
++ calls.length = 0;
++ attempts = 7; // the next one is the last (MAX_TX_ATTEMPTS = 8)
++ await aiAnalysis.generateVodTranscript(vod);
++ assert.deepStrictEqual(calls.filter((c) => c[1] !== 'processing'), [['transcript', 'words at 0'], ['status', 'done', 'transcribed up to 5 min: provider.unavailable', 0]],
++ 'the last attempt keeps what came before the gap and says where it stops');
++
++ calls.length = 0;
++ attempts = 7;
++ ma.transcribeOnly = async () => ({ ...partial, text: '', segments: [] });
++ await aiAnalysis.generateVodTranscript(vod);
++ assert.deepStrictEqual(calls.filter((c) => c[1] !== 'processing'), [['status', 'failed', 'provider.unavailable', 360]], 'nothing at all after the last attempt: failed');
++
++ calls.length = 0;
++ attempts = 0;
++ ma.transcribeOnly = async () => ({ text: 'all of it', segments: [{ start: 1, end: 2, text: 'all of it' }], ok: true, error: null });
++ await aiAnalysis.generateVodTranscript(vod);
++ assert.deepStrictEqual(calls.filter((c) => c[1] !== 'processing'), [['transcript', 'all of it'], ['status', 'done', null, 0]], 'a whole transcript is stored as before');
++ } finally {
++ fs.rmSync(tmp, { recursive: true, force: true });
++ }
++ console.log('transcript partial: all checks passed');
++ process.exit(0);
++})().catch((e) => { console.error(e); process.exit(1); });