openvibers/openvibe.live Commits
The AI backfill drops the state of a VOD or clip Media no longer has (a 404), so items deleted on openvibe.video stop blocking the transcript queue (49 clips waited) (#347)
Changes
@@@ -14,20 +14,31 @@ const db = require('../db/database');const ai = require('./ai-analysis');const media = require('../media-client');const remote = require('./ai-media-analyze');++const purge = require('../media-proxy/purge');let _timer = null;let _busy = false;let _loadWarnedAt = 0;++// Media answered 404: the item was deleted there without passing through Live (on openvibe.video, or by Media's own++// tools), so Live's rows about it never went. They go now, as after a delete through Live (purge.afterDelete). The++// queues are newest first and take a few candidates a minute, so eight such rows on top kept every older one waiting++// (2026-10-24: 49 clips and every older VOD).++async function _gone(kind, id, err) {++ if (!err || err.status !== 404) return;++ await purge.afterDelete(kind, id);++ console.log(`[AI backfill] ${kind} ${id} no longer exists on Media: its AI state is dropped`);++}++// Resolve Media metadata for a state row ({id} = Media id). Returns a vod/clip-shaped--// object the ai-analysis functions understand, or null (→ retry later).++// object the ai-analysis functions understand, or null (→ retry later; a 404 drops the row).async function _vodMeta(row) {try {const v = await media.getVod(row.id);if (!v) return null;if (v.status && String(v.status) === 'recording') return null; // still growingreturn { ...v, id: row.id, duration_seconds: v.duration_seconds ?? v.duration ?? 0 };-- } catch { return null; }++ } catch (e) { await _gone('vod', row.id, e).catch(() => {}); return null; }}async function _clipMeta(row) {try {@@@ -39,7 +50,7 @@ async function _clipMeta(row) {return null;}return { ...c, id: row.id, duration_seconds: c.duration_seconds ?? c.duration ?? 0 };-- } catch { return null; }++ } catch (e) { await _gone('clip', row.id, e).catch(() => {}); return null; }}async function tick() {@@@ -3,7 +3,8 @@* OpenVibe.Live — what Live does once OpenVibe.Media has deleted a VOD or clip for it.** Every delete path calls afterDelete() after Media confirmed the delete: the owner's /api/vods and-- * /api/clips routes (one, bulk, older-than) and the admin storage page. It++ * /api/clips routes (one, bulk, older-than) and the admin storage page; and the AI backfill when Media++ * answers 404 for an item deleted elsewhere (openvibe.video, Media's own tools). It* - hides the item's comment thread on OpenVibe.Community (comments-client.js);* - drops Live's own rows about it: the AI state (vod_ai_state / clip_ai_state), which the AI* backfill (server/ai/backfill-job.js) takes newest-first and would otherwise keep offering for annnew file mode 100644@@@ -0,0 +1,65 @@++'use strict';++// The AI backfill drops the state of a VOD or clip OpenVibe.Media no longer has (deleted on openvibe.video or by Media's++// own tools, so Live's purge never ran), instead of offering it again every minute: newest first, eight such rows kept++// every older clip waiting (2026-10-24: 49 clips). A Media that is down or slow (a 5xx, no answer) drops nothing.++const assert = require('assert');++const os = require('os');++const db = require('../server/db/database');++const media = require('../server/media-client');++const local = require('../server/ai/ai-analysis');++const comments = require('../server/comments-client');++const search = require('../server/events/search-media-documents');++const backfill = require('../server/ai/backfill-job');++++(async () => {++ await db.initDb();++ delete process.env.LIVE_MEDIA_ANALYSIS; // the local path (production's), where the transcript queue runs++ os.loadavg = () => [0, 0, 0]; // headroom, whatever the machine running the test is doing++ const gone = new Set([951, 952, 953, 954, 955, 956, 957, 958, 959, 811]);++ let mediaDown = false;++ const lookup = (kind) => async (id) => {++ if (mediaDown && id === 940) throw new media.MediaApiError('Media API 503', 503, null);++ if (gone.has(Number(id))) throw new media.MediaApiError(`${kind} not found`, 404, { error: `${kind} not found` });++ return { id, status: 'ready', user_id: 7, duration_seconds: 30 };++ };++ media.getClip = lookup('Clip');++ media.getVod = lookup('VOD');++ const hidden = [];++ comments.hideThreadOf = (kind, id) => { hidden.push(`${kind}:${id}`); };++ search.touchLater = () => {};++ const transcribed = [];++ local.transcriptionEnabled = async () => true;++ local.generateClipTranscript = async (clip) => {++ transcribed.push(`clip:${clip.id}`);++ await db.setClipTranscriptStatus(clip.id, 'done');++ await db.run("UPDATE clip_ai_state SET ai_transcript_json = '[]' WHERE clip_id = ?", [clip.id]);++ };++ local.generateVodTranscript = async (vod) => {++ transcribed.push(`vod:${vod.id}`);++ await db.setVodTranscriptStatus(vod.id, 'done');++ await db.run("UPDATE vod_ai_state SET ai_transcript_json = '[]' WHERE vod_id = ?", [vod.id]);++ };++ local.isEnabled = async () => false;++++ await db.run("INSERT INTO clip_ai_state (clip_id, transcript_status) VALUES (940, 'pending')");++ for (const id of [951, 952, 953, 954, 955, 956, 957, 958, 959]) await db.run("INSERT INTO clip_ai_state (clip_id, transcript_status) VALUES (?, 'pending')", [id]);++ await db.run("INSERT INTO vod_ai_state (vod_id, transcript_status) VALUES (810, 'pending')");++ await db.run("INSERT INTO vod_ai_state (vod_id, transcript_status) VALUES (811, 'pending')");++++ // A Media that cannot answer keeps the row: only a 404 means the item is gone.++ mediaDown = true;++ await backfill.tick();++ assert.ok(await db.getClipAiState(940), 'a clip Media could not look up stays queued');++ assert.ok(!transcribed.includes('clip:940'));++ mediaDown = false;++++ await backfill.tick();++ await backfill.tick();++ for (const id of [951, 952, 953, 954, 955, 956, 957, 958, 959]) assert.ok(!(await db.getClipAiState(id)), `deleted clip ${id}'s state is dropped`);++ assert.ok(!(await db.getVodAiState(811)), 'a deleted VOD\'s state is dropped');++ assert.ok(transcribed.includes('clip:940'), 'the clip behind the deleted ones is transcribed');++ assert.ok(transcribed.includes('vod:810'), 'the VOD behind the deleted one is transcribed');++ assert.ok(hidden.includes('clip:951') && hidden.includes('vod:811'), 'the deleted items\' comment threads are hidden, as after a delete through Live');++ console.log('backfill-deleted-media: all checks passed');++ process.exit(0);++})().catch((e) => { console.error(e); process.exit(1); });
@@ -14,20 +14,31 @@ const db = require('../db/database');
@@ -14,20 +14,31 @@ const db = require('../db/database');
const ai = require('./ai-analysis');
const ai = require('./ai-analysis');
const media = require('../media-client');
const media = require('../media-client');
const remote = require('./ai-media-analyze');
const remote = require('./ai-media-analyze');
+const purge = require('../media-proxy/purge');
let _timer = null;
let _timer = null;
let _busy = false;
let _busy = false;
let _loadWarnedAt = 0;
let _loadWarnedAt = 0;
+// Media answered 404: the item was deleted there without passing through Live (on openvibe.video, or by Media's own
+// tools), so Live's rows about it never went. They go now, as after a delete through Live (purge.afterDelete). The
+// queues are newest first and take a few candidates a minute, so eight such rows on top kept every older one waiting
+// (2026-10-24: 49 clips and every older VOD).
+async function _gone(kind, id, err) {
+ if (!err || err.status !== 404) return;
+ await purge.afterDelete(kind, id);
+ console.log(`[AI backfill] ${kind} ${id} no longer exists on Media: its AI state is dropped`);
+}
+
// Resolve Media metadata for a state row ({id} = Media id). Returns a vod/clip-shaped
// Resolve Media metadata for a state row ({id} = Media id). Returns a vod/clip-shaped
-// object the ai-analysis functions understand, or null (→ retry later).
+// object the ai-analysis functions understand, or null (→ retry later; a 404 drops the row).
async function _vodMeta(row) {
async function _vodMeta(row) {
try {
try {
const v = await media.getVod(row.id);
const v = await media.getVod(row.id);
if (!v) return null;
if (!v) return null;
if (v.status && String(v.status) === 'recording') return null; // still growing
if (v.status && String(v.status) === 'recording') return null; // still growing
return { ...v, id: row.id, duration_seconds: v.duration_seconds ?? v.duration ?? 0 };
return { ...v, id: row.id, duration_seconds: v.duration_seconds ?? v.duration ?? 0 };
- } catch { return null; }
+ } catch (e) { await _gone('vod', row.id, e).catch(() => {}); return null; }
}
}
async function _clipMeta(row) {
async function _clipMeta(row) {
try {
try {
@@ -39,7 +50,7 @@ async function _clipMeta(row) {
@@ -39,7 +50,7 @@ async function _clipMeta(row) {
return null;
return null;
}
}
return { ...c, id: row.id, duration_seconds: c.duration_seconds ?? c.duration ?? 0 };
return { ...c, id: row.id, duration_seconds: c.duration_seconds ?? c.duration ?? 0 };
- } catch { return null; }
+ } catch (e) { await _gone('clip', row.id, e).catch(() => {}); return null; }
}
}
async function tick() {
async function tick() {
@@ -3,7 +3,8 @@
@@ -3,7 +3,8 @@
* OpenVibe.Live — what Live does once OpenVibe.Media has deleted a VOD or clip for it.
* OpenVibe.Live — what Live does once OpenVibe.Media has deleted a VOD or clip for it.
*
*
* Every delete path calls afterDelete() after Media confirmed the delete: the owner's /api/vods and
* Every delete path calls afterDelete() after Media confirmed the delete: the owner's /api/vods and
- * /api/clips routes (one, bulk, older-than) and the admin storage page. It
+ * /api/clips routes (one, bulk, older-than) and the admin storage page; and the AI backfill when Media
+ * answers 404 for an item deleted elsewhere (openvibe.video, Media's own tools). It
* - hides the item's comment thread on OpenVibe.Community (comments-client.js);
* - hides the item's comment thread on OpenVibe.Community (comments-client.js);
* - drops Live's own rows about it: the AI state (vod_ai_state / clip_ai_state), which the AI
* - drops Live's own rows about it: the AI state (vod_ai_state / clip_ai_state), which the AI
* backfill (server/ai/backfill-job.js) takes newest-first and would otherwise keep offering for an
* backfill (server/ai/backfill-job.js) takes newest-first and would otherwise keep offering for an
new file mode 100644
new file mode 100644
@@ -0,0 +1,65 @@
@@ -0,0 +1,65 @@
+'use strict';
+// The AI backfill drops the state of a VOD or clip OpenVibe.Media no longer has (deleted on openvibe.video or by Media's
+// own tools, so Live's purge never ran), instead of offering it again every minute: newest first, eight such rows kept
+// every older clip waiting (2026-10-24: 49 clips). A Media that is down or slow (a 5xx, no answer) drops nothing.
+const assert = require('assert');
+const os = require('os');
+const db = require('../server/db/database');
+const media = require('../server/media-client');
+const local = require('../server/ai/ai-analysis');
+const comments = require('../server/comments-client');
+const search = require('../server/events/search-media-documents');
+const backfill = require('../server/ai/backfill-job');
+
+(async () => {
+ await db.initDb();
+ delete process.env.LIVE_MEDIA_ANALYSIS; // the local path (production's), where the transcript queue runs
+ os.loadavg = () => [0, 0, 0]; // headroom, whatever the machine running the test is doing
+ const gone = new Set([951, 952, 953, 954, 955, 956, 957, 958, 959, 811]);
+ let mediaDown = false;
+ const lookup = (kind) => async (id) => {
+ if (mediaDown && id === 940) throw new media.MediaApiError('Media API 503', 503, null);
+ if (gone.has(Number(id))) throw new media.MediaApiError(`${kind} not found`, 404, { error: `${kind} not found` });
+ return { id, status: 'ready', user_id: 7, duration_seconds: 30 };
+ };
+ media.getClip = lookup('Clip');
+ media.getVod = lookup('VOD');
+ const hidden = [];
+ comments.hideThreadOf = (kind, id) => { hidden.push(`${kind}:${id}`); };
+ search.touchLater = () => {};
+ const transcribed = [];
+ local.transcriptionEnabled = async () => true;
+ local.generateClipTranscript = async (clip) => {
+ transcribed.push(`clip:${clip.id}`);
+ await db.setClipTranscriptStatus(clip.id, 'done');
+ await db.run("UPDATE clip_ai_state SET ai_transcript_json = '[]' WHERE clip_id = ?", [clip.id]);
+ };
+ local.generateVodTranscript = async (vod) => {
+ transcribed.push(`vod:${vod.id}`);
+ await db.setVodTranscriptStatus(vod.id, 'done');
+ await db.run("UPDATE vod_ai_state SET ai_transcript_json = '[]' WHERE vod_id = ?", [vod.id]);
+ };
+ local.isEnabled = async () => false;
+
+ await db.run("INSERT INTO clip_ai_state (clip_id, transcript_status) VALUES (940, 'pending')");
+ for (const id of [951, 952, 953, 954, 955, 956, 957, 958, 959]) await db.run("INSERT INTO clip_ai_state (clip_id, transcript_status) VALUES (?, 'pending')", [id]);
+ await db.run("INSERT INTO vod_ai_state (vod_id, transcript_status) VALUES (810, 'pending')");
+ await db.run("INSERT INTO vod_ai_state (vod_id, transcript_status) VALUES (811, 'pending')");
+
+ // A Media that cannot answer keeps the row: only a 404 means the item is gone.
+ mediaDown = true;
+ await backfill.tick();
+ assert.ok(await db.getClipAiState(940), 'a clip Media could not look up stays queued');
+ assert.ok(!transcribed.includes('clip:940'));
+ mediaDown = false;
+
+ await backfill.tick();
+ await backfill.tick();
+ for (const id of [951, 952, 953, 954, 955, 956, 957, 958, 959]) assert.ok(!(await db.getClipAiState(id)), `deleted clip ${id}'s state is dropped`);
+ assert.ok(!(await db.getVodAiState(811)), 'a deleted VOD\'s state is dropped');
+ assert.ok(transcribed.includes('clip:940'), 'the clip behind the deleted ones is transcribed');
+ assert.ok(transcribed.includes('vod:810'), 'the VOD behind the deleted one is transcribed');
+ assert.ok(hidden.includes('clip:951') && hidden.includes('vod:811'), 'the deleted items\' comment threads are hidden, as after a delete through Live');
+ console.log('backfill-deleted-media: all checks passed');
+ process.exit(0);
+})().catch((e) => { console.error(e); process.exit(1); });