openvibers/openvibe.media Commits
A thumbnail job gives up on a recording it cannot draw from: recorded size, three tries a week (#289)
Changes
@@@ -65,6 +65,18 @@ async function run(job) {if (recording && ['zero_byte', 'missing_file'].includes(recording.health_status)) {throw new JobError('media_unavailable', `The recording is ${recording.health_status === 'zero_byte' ? 'empty (0 bytes)' : 'missing'}`, { permanent: true, status: 404 });}++ // The size the row recorded, wherever the file lives now: VOD 1803 (526 bytes) is on R2, where the local size check++ // below never runs, and its job failed as retryable on every page that showed it (109 times on 2026-11-02).++ const recordedSize = Number((recording || row).file_size);++ if (Number.isFinite(recordedSize) && recordedSize > 0 && recordedSize < TOO_SMALL_FOR_A_FRAME) {++ throw new JobError('media_unavailable', `The media file is too small to hold a picture (${recordedSize} bytes)`, { permanent: true, status: 404 });++ }++ // A recording ffmpeg could draw no frame from three times this week will not give one on the fourth try (VOD 6030,++ // 907 KB on B2 with a clean health scan: 47 failed jobs in two days). It is answered as unavailable (404), which the++ // callers keep for a long while, instead of a 500 that every page asks about again.++ if (await failedBefore(job.app_id, kind, id) >= GIVE_UP_AFTER) {++ throw new JobError('media_unavailable', `No frame could be drawn from this ${kind} (${GIVE_UP_AFTER} tries this week)`, { permanent: true, status: 404 });++ }let source = null;if (kind === 'clip' && row.file_path && require('fs').existsSync(row.file_path)) source = { kind: 'file', value: row.file_path };else source = await require('../vod/vod-storage').resolveMediaSource(virtual ? recording : row);@@@ -101,8 +113,21 @@ async function run(job) {// Under 1 KB is a container header at most, with no frame in it: no picture to take.const TOO_SMALL_FOR_A_FRAME = 1024;++// Failed draws of one recording in the last week after which the job stops trying.++const GIVE_UP_AFTER = 3;++const WEEK_MS = 7 * 24 * 60 * 60 * 1000;++++/** This recording's thumbnail jobs that failed to draw a frame (generate_failed) in the last week. */++async function failedBefore(appId, kind, id) {++ const since = new Date(Date.now() - WEEK_MS).toISOString().replace('T', ' ').slice(0, 19);++ const r = await db.get(`SELECT COUNT(*) AS n FROM media_jobs WHERE job_type = 'thumbnail.regenerate' AND status = 'failed'++ AND error_code = 'generate_failed' AND app_id = ? AND params = ? AND created_at > ?`, [appId, JSON.stringify({ kind, id }), since]);++ return Number((r && r.n) || 0);++}module.exports = {spec: { lane: 'light', maxAttempts: 2, timeoutMs: 2 * 60 * 1000, needsObject: false, backoffS: () => 20, validate, run },targetOf,++ failedBefore,++ GIVE_UP_AFTER,};@@@ -102,6 +102,31 @@ const express = require('express');} else console.log('damaged-recording case skipped (ffmpeg not found)');assert.ok(/empty \(0 bytes\)/.test((await queue.list('live', { type: 'thumbnail.regenerate' })).jobs.find(j => JSON.parse(j.params).id === marked).error));assert.strictEqual((await db.all("SELECT COUNT(*) AS n FROM media_jobs WHERE error_code = 'generate_failed'"))[0].n, 0, 'none of them is a generate failure any more');++ // VOD 1803 for real (2026-11-02): 526 bytes, but on R2, so no local file to measure. The size the row recorded is++ // enough: media_unavailable (404) before any download or ffmpeg run.++ const remoteTiny = Number((await db.run(`INSERT INTO vods (app_id, title, file_path, file_size, is_public, visibility, health_status, storage_provider, storage_key)++ VALUES ('live', 't', ?, 526, 1, 'public', 'ok', 'r2', 'vods/tiny-remote.webm') RETURNING id`, [path.join(V, 'offloaded-away.webm')])).lastInsertRowid);++ const rt = await post(`/api/v1/live/thumbnails/vod/${remoteTiny}`);++ assert.strictEqual(rt.status, 404, JSON.stringify(rt.body));++ assert.match((await queue.get(rt.body.job_id)).error, /too small to hold a picture \(526 bytes\)/);++ // VOD 6030 (2026-11-02: 907 KB on B2, health ok, 47 failed jobs in two days): after three failed draws this week++ // the job stops trying and answers unavailable (404), which its callers keep for a long while.++ fs.writeFileSync(path.join(V, 'stubborn.webm'), Buffer.alloc(4096, 7));++ const stubborn = Number((await db.run(`INSERT INTO vods (app_id, title, file_path, file_size, is_public, visibility, health_status)++ VALUES ('live', 't', ?, 4096, 1, 'public', 'ok') RETURNING id`, [path.join(V, 'stubborn.webm')])).lastInsertRowid);++ const { failedBefore, GIVE_UP_AFTER } = require('../server/jobs/thumbnail');++ for (let i = 0; i < GIVE_UP_AFTER; i++) {++ await db.run(`INSERT INTO media_jobs (id, app_id, job_type, status, params, error_code, error, attempts, max_attempts)++ VALUES (?, 'live', 'thumbnail.regenerate', 'failed', ?, 'generate_failed', 'Failed to generate thumbnail', 1, 1)`, [`mjob_test_stubborn_${i}`, JSON.stringify({ kind: 'vod', id: stubborn })]);++ }++ assert.strictEqual(await failedBefore('live', 'vod', stubborn), GIVE_UP_AFTER);++ assert.strictEqual(await failedBefore('live', 'vod', remoteTiny), 0, 'counted per recording');++ const t1 = Date.now();++ const gaveUp = await post(`/api/v1/live/thumbnails/vod/${stubborn}`);++ assert.strictEqual(gaveUp.status, 404, JSON.stringify(gaveUp.body));++ const gaveUpJob = await queue.get(gaveUp.body.job_id);++ assert.deepStrictEqual([gaveUpJob.error_code, /No frame could be drawn from this vod \(3 tries this week\)/.test(gaveUpJob.error)], ['media_unavailable', true]);++ assert.ok(Date.now() - t1 < 5000, 'refused before any ffmpeg run');server.close();await db.close();fs.rmSync(tmp, { recursive: true, force: true });
@@ -65,6 +65,18 @@ async function run(job) {
@@ -65,6 +65,18 @@ async function run(job) {
if (recording && ['zero_byte', 'missing_file'].includes(recording.health_status)) {
if (recording && ['zero_byte', 'missing_file'].includes(recording.health_status)) {
throw new JobError('media_unavailable', `The recording is ${recording.health_status === 'zero_byte' ? 'empty (0 bytes)' : 'missing'}`, { permanent: true, status: 404 });
throw new JobError('media_unavailable', `The recording is ${recording.health_status === 'zero_byte' ? 'empty (0 bytes)' : 'missing'}`, { permanent: true, status: 404 });
}
}
+ // The size the row recorded, wherever the file lives now: VOD 1803 (526 bytes) is on R2, where the local size check
+ // below never runs, and its job failed as retryable on every page that showed it (109 times on 2026-11-02).
+ const recordedSize = Number((recording || row).file_size);
+ if (Number.isFinite(recordedSize) && recordedSize > 0 && recordedSize < TOO_SMALL_FOR_A_FRAME) {
+ throw new JobError('media_unavailable', `The media file is too small to hold a picture (${recordedSize} bytes)`, { permanent: true, status: 404 });
+ }
+ // A recording ffmpeg could draw no frame from three times this week will not give one on the fourth try (VOD 6030,
+ // 907 KB on B2 with a clean health scan: 47 failed jobs in two days). It is answered as unavailable (404), which the
+ // callers keep for a long while, instead of a 500 that every page asks about again.
+ if (await failedBefore(job.app_id, kind, id) >= GIVE_UP_AFTER) {
+ throw new JobError('media_unavailable', `No frame could be drawn from this ${kind} (${GIVE_UP_AFTER} tries this week)`, { permanent: true, status: 404 });
+ }
let source = null;
let source = null;
if (kind === 'clip' && row.file_path && require('fs').existsSync(row.file_path)) source = { kind: 'file', value: row.file_path };
if (kind === 'clip' && row.file_path && require('fs').existsSync(row.file_path)) source = { kind: 'file', value: row.file_path };
else source = await require('../vod/vod-storage').resolveMediaSource(virtual ? recording : row);
else source = await require('../vod/vod-storage').resolveMediaSource(virtual ? recording : row);
@@ -101,8 +113,21 @@ async function run(job) {
@@ -101,8 +113,21 @@ async function run(job) {
// Under 1 KB is a container header at most, with no frame in it: no picture to take.
// Under 1 KB is a container header at most, with no frame in it: no picture to take.
const TOO_SMALL_FOR_A_FRAME = 1024;
const TOO_SMALL_FOR_A_FRAME = 1024;
+// Failed draws of one recording in the last week after which the job stops trying.
+const GIVE_UP_AFTER = 3;
+const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
+
+/** This recording's thumbnail jobs that failed to draw a frame (generate_failed) in the last week. */
+async function failedBefore(appId, kind, id) {
+ const since = new Date(Date.now() - WEEK_MS).toISOString().replace('T', ' ').slice(0, 19);
+ const r = await db.get(`SELECT COUNT(*) AS n FROM media_jobs WHERE job_type = 'thumbnail.regenerate' AND status = 'failed'
+ AND error_code = 'generate_failed' AND app_id = ? AND params = ? AND created_at > ?`, [appId, JSON.stringify({ kind, id }), since]);
+ return Number((r && r.n) || 0);
+}
module.exports = {
module.exports = {
spec: { lane: 'light', maxAttempts: 2, timeoutMs: 2 * 60 * 1000, needsObject: false, backoffS: () => 20, validate, run },
spec: { lane: 'light', maxAttempts: 2, timeoutMs: 2 * 60 * 1000, needsObject: false, backoffS: () => 20, validate, run },
targetOf,
targetOf,
+ failedBefore,
+ GIVE_UP_AFTER,
};
};
@@ -102,6 +102,31 @@ const express = require('express');
@@ -102,6 +102,31 @@ const express = require('express');
} else console.log('damaged-recording case skipped (ffmpeg not found)');
} else console.log('damaged-recording case skipped (ffmpeg not found)');
assert.ok(/empty \(0 bytes\)/.test((await queue.list('live', { type: 'thumbnail.regenerate' })).jobs.find(j => JSON.parse(j.params).id === marked).error));
assert.ok(/empty \(0 bytes\)/.test((await queue.list('live', { type: 'thumbnail.regenerate' })).jobs.find(j => JSON.parse(j.params).id === marked).error));
assert.strictEqual((await db.all("SELECT COUNT(*) AS n FROM media_jobs WHERE error_code = 'generate_failed'"))[0].n, 0, 'none of them is a generate failure any more');
assert.strictEqual((await db.all("SELECT COUNT(*) AS n FROM media_jobs WHERE error_code = 'generate_failed'"))[0].n, 0, 'none of them is a generate failure any more');
+ // VOD 1803 for real (2026-11-02): 526 bytes, but on R2, so no local file to measure. The size the row recorded is
+ // enough: media_unavailable (404) before any download or ffmpeg run.
+ const remoteTiny = Number((await db.run(`INSERT INTO vods (app_id, title, file_path, file_size, is_public, visibility, health_status, storage_provider, storage_key)
+ VALUES ('live', 't', ?, 526, 1, 'public', 'ok', 'r2', 'vods/tiny-remote.webm') RETURNING id`, [path.join(V, 'offloaded-away.webm')])).lastInsertRowid);
+ const rt = await post(`/api/v1/live/thumbnails/vod/${remoteTiny}`);
+ assert.strictEqual(rt.status, 404, JSON.stringify(rt.body));
+ assert.match((await queue.get(rt.body.job_id)).error, /too small to hold a picture \(526 bytes\)/);
+ // VOD 6030 (2026-11-02: 907 KB on B2, health ok, 47 failed jobs in two days): after three failed draws this week
+ // the job stops trying and answers unavailable (404), which its callers keep for a long while.
+ fs.writeFileSync(path.join(V, 'stubborn.webm'), Buffer.alloc(4096, 7));
+ const stubborn = Number((await db.run(`INSERT INTO vods (app_id, title, file_path, file_size, is_public, visibility, health_status)
+ VALUES ('live', 't', ?, 4096, 1, 'public', 'ok') RETURNING id`, [path.join(V, 'stubborn.webm')])).lastInsertRowid);
+ const { failedBefore, GIVE_UP_AFTER } = require('../server/jobs/thumbnail');
+ for (let i = 0; i < GIVE_UP_AFTER; i++) {
+ await db.run(`INSERT INTO media_jobs (id, app_id, job_type, status, params, error_code, error, attempts, max_attempts)
+ VALUES (?, 'live', 'thumbnail.regenerate', 'failed', ?, 'generate_failed', 'Failed to generate thumbnail', 1, 1)`, [`mjob_test_stubborn_${i}`, JSON.stringify({ kind: 'vod', id: stubborn })]);
+ }
+ assert.strictEqual(await failedBefore('live', 'vod', stubborn), GIVE_UP_AFTER);
+ assert.strictEqual(await failedBefore('live', 'vod', remoteTiny), 0, 'counted per recording');
+ const t1 = Date.now();
+ const gaveUp = await post(`/api/v1/live/thumbnails/vod/${stubborn}`);
+ assert.strictEqual(gaveUp.status, 404, JSON.stringify(gaveUp.body));
+ const gaveUpJob = await queue.get(gaveUp.body.job_id);
+ assert.deepStrictEqual([gaveUpJob.error_code, /No frame could be drawn from this vod \(3 tries this week\)/.test(gaveUpJob.error)], ['media_unavailable', true]);
+ assert.ok(Date.now() - t1 < 5000, 'refused before any ffmpeg run');
server.close();
server.close();
await db.close();
await db.close();
fs.rmSync(tmp, { recursive: true, force: true });
fs.rmSync(tmp, { recursive: true, force: true });