openvibers/openvibe.ai Commits

AI: media whisper cannot read fails as media.unreadable and never opens whisper's circuit for everyone (#67)

OP OpenVibers committed 29 days ago · 264a407075 · 5 files, +95 −6

**One unreadable recording no longer takes transcription away from everyone.**

**Found:** while reading AI's failed runs from 2026-10-14 19:46 UTC. One recording made whisper's ffmpeg exit 8. AI counted that as whisper failing, so whisper's circuit opened, and every other recording's windows answered `provider.unavailable` until the cooldown ended (46 runs in one second).

**Why it counted:** ffmpeg reads Media through AI's loopback reader (`server/media-proxy.js`). When Media answered 404, the reader answered ffmpeg with a 502. A missing recording therefore looked the same as a down server, and it was counted toward whisper's health.

**Change:**
- **`fetcher.openStream`:** a non-200 answer from the source keeps its status on the error (`upstreamStatus`).
- **`media-proxy`:** Media's 4xx goes to ffmpeg unchanged. Anything else (Media down, a timeout) is still a 502.
- **`providers/whisper.js`:** ffmpeg's stderr is kept (the last 4 KB). When it says the input cannot be read (a 4xx, not media, no audio stream, no moov atom), the provider answers `422 media.unreadable` with ffmpeg's last line. Any other exit is still `provider.error`.
- **`providers/index.js`:** a 422 is the caller's fault, so it never opens the circuit (`callerFault`, unchanged). When a candidate could not read the media, the run fails with `media.unreadable` rather than `provider.unavailable`, so Live skips that window instead of waiting for whisper to come back (Live#123).

**Tests:** `whisper-unreadable.test.js` (new, needs ffmpeg):
- the provider answers `422 media.unreadable` for a 404 and for a file that is not media;
- three `live.media.transcribe` runs on a recording Media answers 404 for each fail with `media.unreadable`, and whisper's circuit stays closed (it opens after two failures in the test).

The tests this reaches all pass: media-proxy, ssrf, providers, provider-billing, provider-responses, media-analysis, credentials and transcribe-window.

Changes

ddiff --git a/server/fetcher.js b/server/fetcher.js
iindex da7ab55..f8eec62 100644
---- a/server/fetcher.js
++++ b/server/fetcher.js
@@@ -208,7 +208,12 @@ function createFetcher(config, { transport = null } = {}) {
target = follow(target, new URL(res.headers.location, target.url).toString());
continue;
}
-- if (![200, 206, 416].includes(res.statusCode)) { res.resume(); throw new AiError(502, 'source.unavailable', `source answered HTTP ${res.statusCode}`); }
++ if (![200, 206, 416].includes(res.statusCode)) {
++ res.resume();
++ const err = new AiError(502, 'source.unavailable', `source answered HTTP ${res.statusCode}`);
++ err.upstreamStatus = res.statusCode; // the loopback reader passes a 4xx on, so ffmpeg reads "gone", not "down"
++ throw err;
++ }
return { res };
}
throw new AiError(502, 'source.unavailable', 'too many redirects');
ddiff --git a/server/media-proxy.js b/server/media-proxy.js
iindex 3857994..973f119 100644
---- a/server/media-proxy.js
++++ b/server/media-proxy.js
@@@ -44,7 +44,10 @@ function createMediaProxy({ fetcher, log = console, clock = { now: () => Date.no
}, (err) => {
if (ac.signal.aborted) return;
log.warn && log.warn(`[media-proxy] ${err.code || 'error'}: ${String(err.detail || err.message).slice(0, 200)}`);
-- if (!res.headersSent) res.writeHead(err.status === 422 ? 403 : 502).end();
++ // Media's 4xx (the recording gone, not allowed) is passed on as it was: ffmpeg then says the media cannot be
++ // read, which is the caller's media and not whisper's health. Everything else (Media down) stays a 502.
++ const gone = err.upstreamStatus >= 400 && err.upstreamStatus < 500 ? err.upstreamStatus : null;
++ if (!res.headersSent) res.writeHead(gone || (err.status === 422 ? 403 : 502)).end();
else res.destroy();
});
});
ddiff --git a/server/providers/index.js b/server/providers/index.js
iindex d137b2e..c0f8f61 100644
---- a/server/providers/index.js
++++ b/server/providers/index.js
@@@ -399,6 +399,7 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
if (req.image) features.push('vision');
if (req.json) features.push('json');
const tried = [];
++ let unreadable = null; // a candidate could not read the caller's media (whisper's ffmpeg): no other will either
const { order: list, explain } = await candidates(route, features, operation, ctx);
for (let i = 0; i < list.length; i++) {
const c = list[i];
@@@ -466,11 +467,18 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
// A request the provider rejected as malformed is the caller's fault, not a sign the
// provider is unhealthy; a 429 is capacity, not health either, and moves on without
// opening the circuit. Neither must take a shared provider away from other callers.
++ if (lastErr && lastErr.code === 'media.unreadable') unreadable = lastErr;
if (!callerFault(lastErr) && !rateLimited(lastErr)) await recordFailure(c.provider, lastErr);
tried.push({ provider: c.provider, error: String(lastErr && lastErr.message || lastErr).slice(0, 200) });
log.warn(`[ai] ${operation} via ${c.provider}/${model || '-'} failed: ${lastErr && lastErr.message}`);
}
explain.selected = null;
++ // The media itself could not be read: no provider will do better, and the caller should hear why (and not
++ // retry as if a provider were down).
++ if (unreadable) {
++ explain.reasons = [...(explain.reasons || []), 'the media could not be read'];
++ throw new AiError(422, 'media.unreadable', String(unreadable.message || 'the media could not be read').slice(0, 300), { tried, explain });
++ }
explain.reasons = [...(explain.reasons || []), 'no provider on this route could answer'];
throw new AiError(503, 'provider.unavailable', 'no provider on this route could answer', { tried, explain });
}
ddiff --git a/server/providers/whisper.js b/server/providers/whisper.js
iindex 8648fed..7f3e408 100644
---- a/server/providers/whisper.js
++++ b/server/providers/whisper.js
@@@ -19,6 +19,10 @@ const path = require('path');
const { spawn } = require('child_process');
const { ProviderError } = require('./common');
++// ffmpeg's own words for an input it cannot read: the media's fault (gone, not media, no audio), not the decoder's.
++const UNREADABLE = /Server returned 4\d\d|HTTP error 4\d\d|Invalid data found when processing input|No such file or directory|does not contain any stream|Output file does not contain any stream|Stream map .* matches no streams|moov atom not found|could not find codec parameters|End of file/i;
++const lastLine = (t) => String(t || '').trim().split('\n').filter(Boolean).pop().slice(0, 200) || '';
++
const HALLUCINATIONS = new Set([
'thank you', 'thank you.', 'thanks for watching', 'thanks for watching!', 'thanks for watching.', 'please subscribe', 'subscribe',
'like and subscribe', 'thank you for watching', 'thank you very much', 'thank you so much', 'silence', 'music', 'applause',
@@@ -73,8 +77,8 @@ function createWhisperProvider(record, { config }) {
const release = (lane) => { const L = lanes[lane]; const n = L.waiters.shift(); if (n) n(); else L.running = Math.max(0, L.running - 1); };
const active = new Set();
-- function spawnTracked(cmd, args) {
-- const ch = spawn(cmd, args, { stdio: 'ignore' });
++ function spawnTracked(cmd, args, { stderr = false } = {}) {
++ const ch = spawn(cmd, args, { stdio: ['ignore', 'ignore', stderr ? 'pipe' : 'ignore'] });
active.add(ch);
const drop = () => active.delete(ch);
ch.on('close', drop); ch.on('error', drop);
@@@ -92,14 +96,22 @@ function createWhisperProvider(record, { config }) {
if (normalize) args.push('-af', 'loudnorm=I=-16:TP=-1.5:LRA=11');
args.push('-f', 'wav', wav);
let ff;
-- try { ff = spawnTracked('ffmpeg', args); } catch (e) { return reject(new ProviderError(`ffmpeg: ${e.message}`, { code: 'provider.unavailable' })); }
++ try { ff = spawnTracked('ffmpeg', args, { stderr: true }); } catch (e) { return reject(new ProviderError(`ffmpeg: ${e.message}`, { code: 'provider.unavailable' })); }
++ let errTail = '';
++ if (ff.stderr) ff.stderr.on('data', (c) => { errTail = (errTail + c).slice(-4000); });
const kill = () => { try { ff.kill('SIGKILL'); } catch { /* */ } };
const timer = setTimeout(kill, timeoutMs);
if (signal) signal.addEventListener('abort', kill, { once: true });
ff.on('close', (code) => {
clearTimeout(timer);
if (signal && signal.aborted) { try { fs.unlinkSync(wav); } catch { /* */ } return reject(new ProviderError('cancelled', { code: 'provider.cancelled' })); }
-- if (code !== 0) { try { fs.unlinkSync(wav); } catch { /* */ } return reject(new ProviderError(`ffmpeg exited ${code}`, { code: 'provider.error' })); }
++ if (code !== 0) {
++ try { fs.unlinkSync(wav); } catch { /* */ }
++ // The input itself could not be read (a 4xx from Media, not media, no audio): the caller's media, not
++ // whisper's health, so it is a 422 media.unreadable and does not open the circuit for everyone.
++ if (UNREADABLE.test(errTail)) return reject(new ProviderError(`ffmpeg could not read the media (exit ${code}): ${lastLine(errTail)}`, { status: 422, code: 'media.unreadable' }));
++ return reject(new ProviderError(`ffmpeg exited ${code}`, { code: 'provider.error' }));
++ }
resolve(wav);
});
ff.on('error', (e) => { clearTimeout(timer); reject(new ProviderError(`ffmpeg: ${e.message}`, { code: 'provider.unavailable' })); });
ddiff --git a/test/whisper-unreadable.test.js b/test/whisper-unreadable.test.js
nnew file mode 100644
iindex 0000000..df6cca1
---- /dev/null
++++ b/test/whisper-unreadable.test.js
@@@ -0,0 +1,61 @@
++'use strict';
++// Media whisper cannot read (Media answering 404 for a recording, a file that is not media) is the caller's media, not
++// whisper's ill health: the provider answers a 422 media.unreadable, which the router passes on as the run's error and
++// never counts toward opening whisper's circuit (2026-10-14: one unreadable recording opened it, and every other
++// recording's windows answered provider.unavailable). Needs ffmpeg; skipped without it.
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const http = require('http');
++const { spawnSync } = require('child_process');
++const { createWhisperProvider } = require('../server/providers/whisper');
++
++if (spawnSync('ffmpeg', ['-version']).status !== 0) {
++ console.log('whisper unreadable media: skipped (no ffmpeg)');
++ process.exit(0);
++}
++
++(async () => {
++ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-ai-unreadable-'));
++ // whisper.cpp itself is never reached: the input fails first. A placeholder binary and model make it "installed".
++ const bin = path.join(tmp, 'whisper-cli');
++ const model = path.join(tmp, 'model.bin');
++ fs.writeFileSync(bin, '#!/bin/sh\nexit 1\n', { mode: 0o755 });
++ fs.writeFileSync(model, 'x');
++ const notMedia = path.join(tmp, 'notes.txt');
++ fs.writeFileSync(notMedia, 'this is not audio\n'.repeat(50));
++ const server = http.createServer((req, res) => { res.writeHead(404, { 'Content-Type': 'text/plain' }); res.end('gone'); });
++ await new Promise((r) => server.listen(0, '127.0.0.1', r));
++ const whisper = createWhisperProvider({ key: 'whisper' }, { config: { whisper: { bin, model, threads: 2, beam: 1, maxConcurrent: 1, vad: false } } });
++ try {
++ for (const [what, src] of [['a 404 from Media', `http://127.0.0.1:${server.address().port}/v/1`], ['a file that is not media', notMedia]]) {
++ await assert.rejects(whisper.transcribe({ filePath: src, seconds: 5, timeoutMs: 20000 }), (e) => {
++ assert.deepStrictEqual([e.code, e.status], ['media.unreadable', 422], `${what}: ${e.code} ${e.message}`);
++ assert.match(e.message, /could not read the media/);
++ return true;
++ });
++ }
++ console.log('✓ unreadable input: a 422 media.unreadable from the provider (a 404, and not media)');
++
++ // Through the app: live.media.transcribe on a recording Media answers 404 for. Each run fails with
++ // media.unreadable, and whisper's circuit stays closed (two failures would open it).
++ const { boot, request, token, ALL } = require('./helpers');
++ const h = await boot({ env: { OV_MEDIA_INTERNAL_URL: `http://127.0.0.1:${server.address().port}`, WHISPER_BIN: bin, WHISPER_MODEL: model, WHISPER_VAD: '0', AI_STUB_FALLBACK: 'false', AI_BREAKER_FAILURES: '2', AI_BREAKER_COOLDOWN_MS: '600000' } });
++ try {
++ const tok = token('live', ALL);
++ for (let i = 0; i < 3; i++) {
++ const r = await request(h.base, 'POST', '/api/v1/runs?wait=20000', { tok, body: { workflow: 'live.media.transcribe', input: { media_url: `http://127.0.0.1:${server.address().port}/v/${i}`, language: 'en', start_sec: 0, seconds: 30 } } });
++ const run = r.body.run || {};
++ assert.deepStrictEqual([run.status, run.error && run.error.code], ['failed', 'media.unreadable'], JSON.stringify(run.error));
++ }
++ assert.strictEqual((await h.pool.health('whisper')).state, 'closed', 'unreadable media never opens whisper\'s circuit');
++ } finally { await h.stop(); }
++ console.log('✓ through the app: each run fails media.unreadable and whisper\'s circuit stays closed');
++ } finally {
++ server.close();
++ fs.rmSync(tmp, { recursive: true, force: true });
++ }
++ console.log('whisper unreadable media: all checks passed');
++ process.exit(0);
++})().catch((e) => { console.error(e); process.exit(1); });