openvibers/openvibe.live Commits

The AI backfill drops the state of a VOD or clip Media no longer has (a 404), so items deleted on openvibe.video stop blocking the transcript queue (49 clips waited) (#347)

OP OpenVibers committed 20 days ago · 2c7a276ef8 · 3 files, +81 −4

Changes

ddiff --git a/server/ai/backfill-job.js b/server/ai/backfill-job.js
iindex 800a9eea..5949347e 100644
---- a/server/ai/backfill-job.js
++++ b/server/ai/backfill-job.js
@@@ -14,20 +14,31 @@ const db = require('../db/database');
const ai = require('./ai-analysis');
const media = require('../media-client');
const remote = require('./ai-media-analyze');
++const purge = require('../media-proxy/purge');
let _timer = null;
let _busy = false;
let _loadWarnedAt = 0;
++// Media answered 404: the item was deleted there without passing through Live (on openvibe.video, or by Media's own
++// tools), so Live's rows about it never went. They go now, as after a delete through Live (purge.afterDelete). The
++// queues are newest first and take a few candidates a minute, so eight such rows on top kept every older one waiting
++// (2026-10-24: 49 clips and every older VOD).
++async function _gone(kind, id, err) {
++ if (!err || err.status !== 404) return;
++ await purge.afterDelete(kind, id);
++ console.log(`[AI backfill] ${kind} ${id} no longer exists on Media: its AI state is dropped`);
++}
++
// Resolve Media metadata for a state row ({id} = Media id). Returns a vod/clip-shaped
--// object the ai-analysis functions understand, or null (→ retry later).
++// object the ai-analysis functions understand, or null (→ retry later; a 404 drops the row).
async function _vodMeta(row) {
try {
const v = await media.getVod(row.id);
if (!v) return null;
if (v.status && String(v.status) === 'recording') return null; // still growing
return { ...v, id: row.id, duration_seconds: v.duration_seconds ?? v.duration ?? 0 };
-- } catch { return null; }
++ } catch (e) { await _gone('vod', row.id, e).catch(() => {}); return null; }
}
async function _clipMeta(row) {
try {
@@@ -39,7 +50,7 @@ async function _clipMeta(row) {
return null;
}
return { ...c, id: row.id, duration_seconds: c.duration_seconds ?? c.duration ?? 0 };
-- } catch { return null; }
++ } catch (e) { await _gone('clip', row.id, e).catch(() => {}); return null; }
}
async function tick() {
ddiff --git a/server/media-proxy/purge.js b/server/media-proxy/purge.js
iindex 8097d988..d2fe3d18 100644
---- a/server/media-proxy/purge.js
++++ b/server/media-proxy/purge.js
@@@ -3,7 +3,8 @@
* OpenVibe.Live — what Live does once OpenVibe.Media has deleted a VOD or clip for it.
*
* Every delete path calls afterDelete() after Media confirmed the delete: the owner's /api/vods and
-- * /api/clips routes (one, bulk, older-than) and the admin storage page. It
++ * /api/clips routes (one, bulk, older-than) and the admin storage page; and the AI backfill when Media
++ * answers 404 for an item deleted elsewhere (openvibe.video, Media's own tools). It
* - hides the item's comment thread on OpenVibe.Community (comments-client.js);
* - drops Live's own rows about it: the AI state (vod_ai_state / clip_ai_state), which the AI
* backfill (server/ai/backfill-job.js) takes newest-first and would otherwise keep offering for an
ddiff --git a/test/backfill-deleted-media.test.js b/test/backfill-deleted-media.test.js
nnew file mode 100644
iindex 00000000..fdd7f607
---- /dev/null
++++ b/test/backfill-deleted-media.test.js
@@@ -0,0 +1,65 @@
++'use strict';
++// The AI backfill drops the state of a VOD or clip OpenVibe.Media no longer has (deleted on openvibe.video or by Media's
++// own tools, so Live's purge never ran), instead of offering it again every minute: newest first, eight such rows kept
++// every older clip waiting (2026-10-24: 49 clips). A Media that is down or slow (a 5xx, no answer) drops nothing.
++const assert = require('assert');
++const os = require('os');
++const db = require('../server/db/database');
++const media = require('../server/media-client');
++const local = require('../server/ai/ai-analysis');
++const comments = require('../server/comments-client');
++const search = require('../server/events/search-media-documents');
++const backfill = require('../server/ai/backfill-job');
++
++(async () => {
++ await db.initDb();
++ delete process.env.LIVE_MEDIA_ANALYSIS; // the local path (production's), where the transcript queue runs
++ os.loadavg = () => [0, 0, 0]; // headroom, whatever the machine running the test is doing
++ const gone = new Set([951, 952, 953, 954, 955, 956, 957, 958, 959, 811]);
++ let mediaDown = false;
++ const lookup = (kind) => async (id) => {
++ if (mediaDown && id === 940) throw new media.MediaApiError('Media API 503', 503, null);
++ if (gone.has(Number(id))) throw new media.MediaApiError(`${kind} not found`, 404, { error: `${kind} not found` });
++ return { id, status: 'ready', user_id: 7, duration_seconds: 30 };
++ };
++ media.getClip = lookup('Clip');
++ media.getVod = lookup('VOD');
++ const hidden = [];
++ comments.hideThreadOf = (kind, id) => { hidden.push(`${kind}:${id}`); };
++ search.touchLater = () => {};
++ const transcribed = [];
++ local.transcriptionEnabled = async () => true;
++ local.generateClipTranscript = async (clip) => {
++ transcribed.push(`clip:${clip.id}`);
++ await db.setClipTranscriptStatus(clip.id, 'done');
++ await db.run("UPDATE clip_ai_state SET ai_transcript_json = '[]' WHERE clip_id = ?", [clip.id]);
++ };
++ local.generateVodTranscript = async (vod) => {
++ transcribed.push(`vod:${vod.id}`);
++ await db.setVodTranscriptStatus(vod.id, 'done');
++ await db.run("UPDATE vod_ai_state SET ai_transcript_json = '[]' WHERE vod_id = ?", [vod.id]);
++ };
++ local.isEnabled = async () => false;
++
++ await db.run("INSERT INTO clip_ai_state (clip_id, transcript_status) VALUES (940, 'pending')");
++ for (const id of [951, 952, 953, 954, 955, 956, 957, 958, 959]) await db.run("INSERT INTO clip_ai_state (clip_id, transcript_status) VALUES (?, 'pending')", [id]);
++ await db.run("INSERT INTO vod_ai_state (vod_id, transcript_status) VALUES (810, 'pending')");
++ await db.run("INSERT INTO vod_ai_state (vod_id, transcript_status) VALUES (811, 'pending')");
++
++ // A Media that cannot answer keeps the row: only a 404 means the item is gone.
++ mediaDown = true;
++ await backfill.tick();
++ assert.ok(await db.getClipAiState(940), 'a clip Media could not look up stays queued');
++ assert.ok(!transcribed.includes('clip:940'));
++ mediaDown = false;
++
++ await backfill.tick();
++ await backfill.tick();
++ for (const id of [951, 952, 953, 954, 955, 956, 957, 958, 959]) assert.ok(!(await db.getClipAiState(id)), `deleted clip ${id}'s state is dropped`);
++ assert.ok(!(await db.getVodAiState(811)), 'a deleted VOD\'s state is dropped');
++ assert.ok(transcribed.includes('clip:940'), 'the clip behind the deleted ones is transcribed');
++ assert.ok(transcribed.includes('vod:810'), 'the VOD behind the deleted one is transcribed');
++ assert.ok(hidden.includes('clip:951') && hidden.includes('vod:811'), 'the deleted items\' comment threads are hidden, as after a delete through Live');
++ console.log('backfill-deleted-media: all checks passed');
++ process.exit(0);
++})().catch((e) => { console.error(e); process.exit(1); });