openvibers/openvibe.live Commits
A thumbnail Media cannot make is asked for less and less: each failure doubles the wait, 5 minutes up to a day (#555)
Changes
@@@ -70,6 +70,16 @@ function rememberNoPicture(key, ms = NO_PICTURE_MS) {if (noPicture.size >= 5000) noPicture.delete(noPicture.keys().next().value);noPicture.set(key, Date.now() + ms);}++// Media's job failing again and again for the same recording (a file ffmpeg cannot draw a frame from) is not going to++// change in five minutes: each failure doubles the wait, from 5 minutes up to a day, and a picture resets it (VOD 1803++// failed 32 times in three hours on 2026-11-02, one Media job each).++const failures = new Map(); // `${kind}:${id}` → failures in a row++function failedAgain(key) {++ const n = Math.min((failures.get(key) || 0) + 1, 12);++ if (failures.size >= 5000 && !failures.has(key)) failures.delete(failures.keys().next().value);++ failures.set(key, n);++ return Math.min(MEDIA_BUSY_MS * 2 ** (n - 1), NO_VIDEO_MS);++}async function generateFor(kind, id, req, res) {try {@@@ -96,11 +106,11 @@ async function generateFor(kind, id, req, res) {if (!(err && err.name === 'MediaApiError' && (err.status === 404 || err.status === 429 || failed))) throw err;if (failed) console.warn(`[Thumbnails] Media's thumbnail job failed for ${key}: ${err.status}${err.body && err.body.job_id ? ` (${err.body.job_id})` : ''}`);const noVideo = err.status === 404 && err.body && err.body.code === 'media.no_video';-- rememberNoPicture(key, noVideo ? NO_VIDEO_MS : err.status === 404 ? NO_PICTURE_MS : MEDIA_BUSY_MS);++ rememberNoPicture(key, noVideo ? NO_VIDEO_MS : err.status === 404 ? NO_PICTURE_MS : failed ? failedAgain(key) : MEDIA_BUSY_MS);return res.json({ thumbnail_url: null });}const url = media.publicUrl(out?.url || meta.thumbnail_url);-- if (url) return res.json({ thumbnail_url: url });++ if (url) { failures.delete(key); return res.json({ thumbnail_url: url }); }// Media is still working on it (202 with its job): no picture yet, nothing failed.if (out && out.job) return res.status(202).json({ thumbnail_url: null });// Media answered with neither a picture nor a job (a recording it cannot draw a frame from yet): no picture,@@@ -190,3 +200,4 @@ router.get('/:filename', async (req, res) => {});module.exports = router;++module.exports._failedAgain = failedAgain; // for test/thumbnail-backoff.test.js@@@ -78,6 +78,29 @@ console.warn = () => {};const live = await post('/api/thumbnails/generate/vod/5833');assert.deepStrictEqual([live.status, live.json, jobs], [200, { thumbnail_url: null }, 0], 'a recording VOD asks Media for nothing');++ // The same recording's job failing again and again (VOD 1803, 32 times in three hours on 2026-11-02): each failure++ // doubles the wait, from 5 minutes up to a day; a picture resets it.++ const backoff = require('../server/media-proxy/thumbnails')._failedAgain;++ const waits = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map(() => backoff('vod:9999') / 60000);++ assert.deepStrictEqual(waits, [5, 10, 20, 40, 80, 160, 320, 640, 1280, 1440], 'minutes: doubling, a day at most');++ media.getVod = async (id) => ({ id: Number(id), user_id: 1, visibility: 'public', thumbnail_url: null });++ let fails = 0;++ media.generateThumbnail = async () => {++ fails++;++ const err = new Error('Failed to generate thumbnail');++ Object.assign(err, { name: 'MediaApiError', status: 500, body: { error: 'Failed to generate thumbnail' } });++ throw err;++ };++ const t0 = realNow();++ Date.now = () => t0;++ await post('/api/thumbnails/generate/vod/1803');++ Date.now = () => t0 + 6 * 60 * 1000; // 6 minutes: past the first 5-minute wait++ await post('/api/thumbnails/generate/vod/1803');++ Date.now = () => t0 + 12 * 60 * 1000; // 12 minutes: within the second, 10-minute wait++ await post('/api/thumbnails/generate/vod/1803');++ assert.strictEqual(fails, 2, 'the second failure waits twice as long');++ Date.now = realNow;++server.close();console.log('thumbnail no-picture: all checks passed');process.exit(0);
@@ -70,6 +70,16 @@ function rememberNoPicture(key, ms = NO_PICTURE_MS) {
@@ -70,6 +70,16 @@ function rememberNoPicture(key, ms = NO_PICTURE_MS) {
if (noPicture.size >= 5000) noPicture.delete(noPicture.keys().next().value);
if (noPicture.size >= 5000) noPicture.delete(noPicture.keys().next().value);
noPicture.set(key, Date.now() + ms);
noPicture.set(key, Date.now() + ms);
}
}
+// Media's job failing again and again for the same recording (a file ffmpeg cannot draw a frame from) is not going to
+// change in five minutes: each failure doubles the wait, from 5 minutes up to a day, and a picture resets it (VOD 1803
+// failed 32 times in three hours on 2026-11-02, one Media job each).
+const failures = new Map(); // `${kind}:${id}` → failures in a row
+function failedAgain(key) {
+ const n = Math.min((failures.get(key) || 0) + 1, 12);
+ if (failures.size >= 5000 && !failures.has(key)) failures.delete(failures.keys().next().value);
+ failures.set(key, n);
+ return Math.min(MEDIA_BUSY_MS * 2 ** (n - 1), NO_VIDEO_MS);
+}
async function generateFor(kind, id, req, res) {
async function generateFor(kind, id, req, res) {
try {
try {
@@ -96,11 +106,11 @@ async function generateFor(kind, id, req, res) {
@@ -96,11 +106,11 @@ async function generateFor(kind, id, req, res) {
if (!(err && err.name === 'MediaApiError' && (err.status === 404 || err.status === 429 || failed))) throw err;
if (!(err && err.name === 'MediaApiError' && (err.status === 404 || err.status === 429 || failed))) throw err;
if (failed) console.warn(`[Thumbnails] Media's thumbnail job failed for ${key}: ${err.status}${err.body && err.body.job_id ? ` (${err.body.job_id})` : ''}`);
if (failed) console.warn(`[Thumbnails] Media's thumbnail job failed for ${key}: ${err.status}${err.body && err.body.job_id ? ` (${err.body.job_id})` : ''}`);
const noVideo = err.status === 404 && err.body && err.body.code === 'media.no_video';
const noVideo = err.status === 404 && err.body && err.body.code === 'media.no_video';
- rememberNoPicture(key, noVideo ? NO_VIDEO_MS : err.status === 404 ? NO_PICTURE_MS : MEDIA_BUSY_MS);
+ rememberNoPicture(key, noVideo ? NO_VIDEO_MS : err.status === 404 ? NO_PICTURE_MS : failed ? failedAgain(key) : MEDIA_BUSY_MS);
return res.json({ thumbnail_url: null });
return res.json({ thumbnail_url: null });
}
}
const url = media.publicUrl(out?.url || meta.thumbnail_url);
const url = media.publicUrl(out?.url || meta.thumbnail_url);
- if (url) return res.json({ thumbnail_url: url });
+ if (url) { failures.delete(key); return res.json({ thumbnail_url: url }); }
// Media is still working on it (202 with its job): no picture yet, nothing failed.
// Media is still working on it (202 with its job): no picture yet, nothing failed.
if (out && out.job) return res.status(202).json({ thumbnail_url: null });
if (out && out.job) return res.status(202).json({ thumbnail_url: null });
// Media answered with neither a picture nor a job (a recording it cannot draw a frame from yet): no picture,
// Media answered with neither a picture nor a job (a recording it cannot draw a frame from yet): no picture,
@@ -190,3 +200,4 @@ router.get('/:filename', async (req, res) => {
@@ -190,3 +200,4 @@ router.get('/:filename', async (req, res) => {
});
});
module.exports = router;
module.exports = router;
+module.exports._failedAgain = failedAgain; // for test/thumbnail-backoff.test.js
@@ -78,6 +78,29 @@ console.warn = () => {};
@@ -78,6 +78,29 @@ console.warn = () => {};
const live = await post('/api/thumbnails/generate/vod/5833');
const live = await post('/api/thumbnails/generate/vod/5833');
assert.deepStrictEqual([live.status, live.json, jobs], [200, { thumbnail_url: null }, 0], 'a recording VOD asks Media for nothing');
assert.deepStrictEqual([live.status, live.json, jobs], [200, { thumbnail_url: null }, 0], 'a recording VOD asks Media for nothing');
+ // The same recording's job failing again and again (VOD 1803, 32 times in three hours on 2026-11-02): each failure
+ // doubles the wait, from 5 minutes up to a day; a picture resets it.
+ const backoff = require('../server/media-proxy/thumbnails')._failedAgain;
+ const waits = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map(() => backoff('vod:9999') / 60000);
+ assert.deepStrictEqual(waits, [5, 10, 20, 40, 80, 160, 320, 640, 1280, 1440], 'minutes: doubling, a day at most');
+ media.getVod = async (id) => ({ id: Number(id), user_id: 1, visibility: 'public', thumbnail_url: null });
+ let fails = 0;
+ media.generateThumbnail = async () => {
+ fails++;
+ const err = new Error('Failed to generate thumbnail');
+ Object.assign(err, { name: 'MediaApiError', status: 500, body: { error: 'Failed to generate thumbnail' } });
+ throw err;
+ };
+ const t0 = realNow();
+ Date.now = () => t0;
+ await post('/api/thumbnails/generate/vod/1803');
+ Date.now = () => t0 + 6 * 60 * 1000; // 6 minutes: past the first 5-minute wait
+ await post('/api/thumbnails/generate/vod/1803');
+ Date.now = () => t0 + 12 * 60 * 1000; // 12 minutes: within the second, 10-minute wait
+ await post('/api/thumbnails/generate/vod/1803');
+ assert.strictEqual(fails, 2, 'the second failure waits twice as long');
+ Date.now = realNow;
+
server.close();
server.close();
console.log('thumbnail no-picture: all checks passed');
console.log('thumbnail no-picture: all checks passed');
process.exit(0);
process.exit(0);