openvibers/openvibe.media Commits
Recordings nothing uses are cleaned up daily; a failed remote delete is retried (#307)
Changes
nnew file mode 100644@@@ -0,0 +1,47 @@++/**++ * OpenVibe.Media — job type storage.recordings.cleanup (light lane): deletes recording files nothing uses++ * (server/vod/recording-cleanup.js has the rules: files of deleted VODs, and duplicates whose VOD's own copy is there++ * with the same size; a grace period, holds and every other prefix are respected). Service-wide, like++ * storage.orphans.scan: it runs under queue.SYSTEM_APP, which no tenant reaches.++ *++ * params { dry_run?: boolean } result { deleted, bytes, kept, failed, checked, summary }++ *++ * Queued once a day (MEDIA_RECORDING_CLEANUP_DAYS, 1; 0 = never) by the worker, which does not start in a drill.++ */++'use strict';++++const queue = require('./queue');++const cleanup = require('../vod/recording-cleanup');++++const { JobError } = queue;++const TYPE = 'storage.recordings.cleanup';++++function validate({ appId, obj, params }) {++ if (appId !== queue.SYSTEM_APP) throw new JobError('media.job.forbidden', `${TYPE} covers the whole service's storage: it runs on its schedule`, { status: 403, permanent: true });++ if (obj) throw new JobError('media.job.invalid', `${TYPE} covers the whole storage: leave object_id out`, { permanent: true });++ return { dry_run: !!(params && params.dry_run) };++}++++async function run(job) {++ const r = await cleanup.run({ dryRun: !!(job.params && job.params.dry_run) });++ const summary = cleanup.summarize(r);++ console.log(`[Cleanup] recordings: ${summary}`);++ return { deleted: r.deleted, bytes: r.bytes, kept: r.kept, failed: r.failed, checked: r.checked, summary };++}++++/** Queue one clean-up every MEDIA_RECORDING_CLEANUP_DAYS (default 1; 0 = never). Returns how many were queued. */++async function schedule(nowMs = Date.now()) {++ const raw = parseInt(process.env.MEDIA_RECORDING_CLEANUP_DAYS || '', 10);++ const days = Number.isFinite(raw) ? raw : 1;++ if (!(days > 0)) return 0;++ const period = Math.floor(nowMs / (days * 86400 * 1000));++ try {++ const r = await queue.enqueue({ appId: queue.SYSTEM_APP, type: TYPE, params: {}, idempotencyKey: `${TYPE}:${days}d:${period}`, createdBy: 'system:schedule' });++ return r.created ? 1 : 0;++ } catch (err) {++ console.warn(`[Jobs] could not schedule ${TYPE}: ${err.message}`);++ return 0;++ }++}++++module.exports = { TYPE, spec: { lane: 'light', maxAttempts: 2, timeoutMs: 2 * 60 * 60 * 1000, needsObject: false, validate, run }, schedule };@@@ -36,5 +36,6 @@ queue.register('vod.duration.reconcile', require('./duration-reconcile').spec);queue.register('object.hash', require('./content-hash').spec);queue.register('storage.orphans.scan', require('./storage-orphans').spec);queue.register('storage.move.cleanup', require('./move-cleanup').spec);++queue.register('storage.recordings.cleanup', require('./recording-cleanup').spec);module.exports = { names: () => queue.typeNames() };@@@ -169,6 +169,7 @@ async function tick() {try { await require('./content-hash').schedule(now); } catch (err) { console.warn('[Jobs] content hash schedule:', err.message); }try { await require('./storage-orphans').schedule(now); } catch (err) { console.warn('[Jobs] storage orphan scan schedule:', err.message); }try { await require('./move-cleanup').schedule(now); } catch (err) { console.warn('[Jobs] move cleanup schedule:', err.message); }++ try { await require('./recording-cleanup').schedule(now); } catch (err) { console.warn('[Jobs] recording clean-up schedule:', err.message); }}try { await require('./vod-finalize').sweepOrphans({ now }); } catch (err) { console.warn('[Jobs] orphan sweep:', err.message); }if (now - lastSweepAt > 3600 * 1000) {nnew file mode 100644@@@ -0,0 +1,137 @@++'use strict';++/**++ * OpenVibe.Media — automatic clean-up of recording files nothing uses (the owner, 2026-11-03: "all this should be taken++ * care of automatically"). The storage orphan report (orphans-report.js) only lists; this deletes the cases that are++ * certain, and keeps everything else:++ *++ * row_gone a recording file (the local VOD directory; B2 and R2 under vods/ and live.vods/) whose VOD id, read from++ * its name, has no vods row any more. The VOD was deleted and a copy stayed behind: a remote delete that++ * failed, or an older layout's copy the delete never knew about (2026-11-03: 31 such files, 18 GB, the++ * newest from August). Deleted unless an object for that VOD is under a retention hold.++ * duplicate a B2/R2 key that no row or location names, for a VOD that still exists, when that VOD's own copy (its++ * key on the same provider, else on the other, else its local file) is there with the same size. Local++ * files are never treated as duplicates (a recording in progress writes beside its row).++ *++ * Both wait out a grace period (7 days) from the file's last change. Kept: a name with no VOD id, a copy whose size++ * differs from the VOD's, anything held, and every other prefix of the bucket (vods-orphans/ is parked for review; the++ * backups share the bucket and are never listed here).++ *++ * run({ dryRun, graceDays, now }) → { checked, deleted: { row_gone, duplicate }, bytes, kept: {…}, failed, items }++ *++ * Job storage.recordings.cleanup (server/jobs/recording-cleanup.js) runs it daily.++ */++const fs = require('fs');++const path = require('path');++const db = require('../db/database');++const config = require('../config');++++const PREFIXES = ['vods/', 'live.vods/'];++const GRACE_DAYS = 7;++++const storage = () => require('./vod-storage');++// The remote providers vod-storage knows (B2 and R2 today); placement decides their order elsewhere.++const providers = () => storage().REMOTE_PROVIDERS;++++/** The VOD id a recording file's name carries: vod-<app>-<id>-<ms>, vod-<id>-<ms>, media-<app>-vod-<id>. */++function vodIdOf(key) {++ const name = path.basename(String(key || ''));++ let m = /^vod-(?:[a-z][a-z0-9._]*-)?(\d+)-\d{10,}(?:\.|$)/i.exec(name);++ if (!m) m = /^media-[a-z0-9._]+-vod-(\d+)\./i.exec(name);++ return m ? Number(m[1]) : null;++}++++async function heldVod(id) {++ const rows = await db.all('SELECT id FROM media_objects WHERE legacy_ref LIKE ?', [`legacy:%:vod:${id}`]);++ const model = require('../objects/model');++ for (const r of rows) if (await model.isHeld(r.id)) return true;++ const row = await db.get('SELECT * FROM vods WHERE id = ?', [id]);++ return !!(row && await model.isHeldRow(row));++}++++/** The size of a VOD's own copy: its key on `provider`, then on the other provider, then its local file; null if none. */++async function ownCopySize(vod, provider) {++ const key = storage().keyForVod(vod);++ for (const p of [provider, ...providers().filter((x) => x !== provider)]) {++ if (!storage().providerConfigured(p)) continue;++ const h = await storage().headObject(p, key).catch(() => null);++ if (h) return Number(h.size);++ }++ try { return fs.statSync(storage().localPathForVod(vod)).size; } catch { return null; }++}++++async function run({ dryRun = false, graceDays = GRACE_DAYS, now = Date.now() } = {}) {++ const cutoff = now - Math.max(0, graceDays) * 86400 * 1000;++ const vods = new Map((await db.all('SELECT * FROM vods')).map((v) => [Number(v.id), v]));++ const named = new Set();++ for (const v of vods.values()) {++ if (v.storage_key) named.add(v.storage_key);++ if (v.file_path) named.add('vods/' + path.basename(v.file_path));++ }++ for (const r of await db.all("SELECT key FROM media_locations WHERE provider IN ('b2', 'r2')")) named.add(r.key);++ const localNamed = new Set();++ for (const v of vods.values()) for (const f of [v.file_path, v.master_file_path]) if (f) localNamed.add(path.basename(f));++++ const out = { dry_run: !!dryRun, checked: 0, deleted: { row_gone: 0, duplicate: 0 }, bytes: 0, failed: 0,++ kept: { young: 0, no_vod_id: 0, held: 0, size_differs: 0, no_own_copy: 0 }, items: [] };++ const heldCache = new Map();++ const isHeld = async (id) => { if (!heldCache.has(id)) heldCache.set(id, await heldVod(id)); return heldCache.get(id); };++ const remove = async (where, key, size, reason) => {++ out.items.push({ where, key, size, reason });++ if (dryRun) { out.deleted[reason]++; out.bytes += size; return; }++ let ok;++ if (where === 'local') { try { fs.unlinkSync(key); ok = true; } catch (err) { ok = err.code === 'ENOENT'; } }++ else ok = await storage().deleteObject(where, key);++ if (ok) { out.deleted[reason]++; out.bytes += size; } else out.failed++;++ };++++ // Remote: only the recording prefixes; every other key in the bucket is out of scope.++ for (const provider of providers()) {++ if (!storage().providerConfigured(provider)) continue;++ for (const prefix of PREFIXES) {++ let list = [];++ try { list = await storage().listObjects(provider, prefix); } catch (err) { console.warn(`[Cleanup] list ${provider}:${prefix}: ${err.message}`); continue; }++ for (const o of list) {++ out.checked++;++ if (named.has(o.key)) continue;++ const id = vodIdOf(o.key);++ if (id == null) { out.kept.no_vod_id++; continue; }++ if (o.last_modified && Date.parse(o.last_modified) > cutoff) { out.kept.young++; continue; }++ if (await isHeld(id)) { out.kept.held++; continue; }++ const vod = vods.get(id);++ if (!vod) { await remove(provider, o.key, o.size, 'row_gone'); continue; }++ const own = await ownCopySize(vod, provider);++ if (own == null) { out.kept.no_own_copy++; continue; }++ if (own !== o.size) { out.kept.size_differs++; continue; }++ await remove(provider, o.key, o.size, 'duplicate');++ }++ }++ }++++ // Local: the VOD directory's own files whose VOD is gone (never a duplicate: a recording in progress writes here).++ const dir = path.resolve(config.vod.path);++ let names = [];++ try { names = fs.readdirSync(dir, { withFileTypes: true }).filter((e) => e.isFile()).map((e) => e.name); } catch { /* no directory */ }++ for (const name of names) {++ out.checked++;++ if (localNamed.has(name)) continue;++ const id = vodIdOf(name);++ if (id == null) { out.kept.no_vod_id++; continue; }++ if (vods.has(id)) continue;++ const file = path.join(dir, name);++ let st;++ try { st = fs.statSync(file); } catch { continue; }++ if (st.mtimeMs > cutoff) { out.kept.young++; continue; }++ if (await isHeld(id)) { out.kept.held++; continue; }++ await remove('local', file, st.size, 'row_gone');++ }++ return out;++}++++/** One line for the log. */++function summarize(r) {++ const gb = (r.bytes / 1e9).toFixed(2);++ const kept = Object.entries(r.kept).filter(([, n]) => n).map(([k, n]) => `${n} ${k.replace(/_/g, ' ')}`).join(', ') || 'nothing';++ return `${r.dry_run ? 'would delete' : 'deleted'} ${r.deleted.row_gone} file(s) of deleted VODs and ${r.deleted.duplicate} duplicate(s), ${gb} GB; kept ${kept}${r.failed ? `; ${r.failed} delete(s) failed` : ''} (${r.checked} checked)`;++}++++module.exports = { run, summarize, vodIdOf, PREFIXES, GRACE_DAYS };@@@ -826,10 +826,26 @@ async function deleteVodObjects(vod) {await cleanupSidecar(local);const key = keyForVod(vod);for (const p of REMOTE_PROVIDERS) {-- if (providerConfigured(p)) await deleteObject(p, key);++ if (providerConfigured(p) && !await deleteObject(p, key)) retryDeleteLater(p, key);}}++// A remote delete that failed is tried again a few times in the background (the owner, 2026-11-03: "retry … up to a++// certain amount"): after 30 s, 2, 10 and 30 minutes. If all fail, the daily recording clean-up++// (recording-cleanup.js) still removes the copy its deleted VOD left. Timers are unref'd and nothing is persisted:++// a restart drops the retries, and the clean-up is the backstop.++let DELETE_RETRY_MS = [30_000, 120_000, 600_000, 1_800_000];++function retryDeleteLater(provider, key) {++ (async () => {++ for (const ms of DELETE_RETRY_MS) {++ await new Promise((r) => { const t = setTimeout(r, ms); if (t.unref) t.unref(); });++ if (await deleteObject(provider, key)) { console.log(`[VodStorage] ${provider}:${key} deleted on a retry`); return; }++ }++ console.warn(`[VodStorage] ${provider}:${key} still not deleted after ${DELETE_RETRY_MS.length + 1} tries; the daily recording clean-up will remove it`);++ })().catch(() => {});++}++function _setDeleteRetryDelays(ms) { DELETE_RETRY_MS = ms; }++// ── Disk usage ───────────────────────────────────────────────/**@@@ -1560,6 +1576,7 @@ module.exports = {r2Policy,recordTierDecision,deleteVodObjects,++ _setDeleteRetryDelays,deleteObject,presignGet,runSweep,nnew file mode 100644@@@ -0,0 +1,147 @@++'use strict';++// The recording clean-up (server/vod/recording-cleanup.js; job storage.recordings.cleanup) and the retried remote++// delete (vod-storage.js deleteVodObjects). Against a fake S3 spoken to by the real AWS SDK:++// - a file of a deleted VOD goes (B2, R2 and local), unless it is under the grace period or its VOD is held;++// - a duplicate of a living VOD goes only when the VOD's own copy is there with the same size;++// - a name with no VOD id, a different-size copy and every other prefix (backups, vods-orphans/) stay;++// - a dry run deletes nothing; the job is service-wide only and queued daily;++// - a failed remote delete is tried again until it succeeds.++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const http = require('http');++++(async () => {++ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-media-cleanup-'));++ const dir = (n) => { const d = path.join(tmp, n); fs.mkdirSync(d, { recursive: true }); return d; };++ const OLD = '2026-09-21T00:00:00.000Z';++ const buckets = { 'b2-bucket': new Map(), 'r2-bucket': new Map() };++ const put = (bucket, key, size, modified = OLD) => buckets[bucket].set(key, { size, modified });++ const deletes = [];++ let failDeletes = 0;++ const xml = (v) => String(v).replace(/&/g, '&').replace(/</g, '<');++ const s3 = http.createServer((req, res) => {++ req.resume();++ req.on('end', () => {++ const u = new URL(req.url, 'http://x');++ const [, bucket, ...rest] = decodeURIComponent(u.pathname).split('/');++ const key = rest.join('/');++ const store = buckets[bucket];++ if (!store) { res.writeHead(404); return res.end(); }++ if (req.method === 'DELETE') {++ if (failDeletes > 0) { failDeletes--; res.writeHead(500, { 'Content-Type': 'application/xml' }); return res.end('<Error><Code>InternalError</Code></Error>'); }++ deletes.push(`${bucket}:${key}`); store.delete(key); res.writeHead(204); return res.end();++ }++ if (!key && u.searchParams.get('list-type') === '2') {++ const prefix = u.searchParams.get('prefix') || '';++ const keys = [...store.keys()].filter((k) => k.startsWith(prefix)).sort();++ res.writeHead(200, { 'Content-Type': 'application/xml' });++ return res.end(`<?xml version="1.0" encoding="UTF-8"?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>${bucket}</Name><Prefix>${xml(prefix)}</Prefix><KeyCount>${keys.length}</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>${++ keys.map((k) => `<Contents><Key>${xml(k)}</Key><LastModified>${store.get(k).modified}</LastModified><ETag>"e"</ETag><Size>${store.get(k).size}</Size><StorageClass>STANDARD</StorageClass></Contents>`).join('')}</ListBucketResult>`);++ }++ const obj = store.get(key);++ if (!obj) { res.writeHead(404); return res.end(); }++ res.writeHead(200, { 'Content-Length': obj.size, ETag: '"e"' });++ res.end(req.method === 'GET' ? Buffer.alloc(obj.size) : undefined);++ });++ });++ await new Promise((r) => s3.listen(0, '127.0.0.1', r));++ const endpoint = `http://127.0.0.1:${s3.address().port}`;++ const env = {++ MEDIA_DATA_DIR: path.join(tmp, 'data'), VOD_PATH: dir('vods'), CLIPS_PATH: dir('clips'), FILES_PATH: dir('files'),++ THUMBNAILS_PATH: dir('thumbnails'), PASTES_PATH: dir('pastes'), OBJECTS_PATH: dir('objects'), ASSETS_PATH: dir('assets'),++ MEDIA_B2_ENDPOINT: endpoint, MEDIA_B2_BUCKET: 'b2-bucket', MEDIA_B2_KEY_ID: 'k', MEDIA_B2_APP_KEY: 's', MEDIA_B2_REGION: 'us-west-004',++ MEDIA_R2_ENDPOINT: endpoint, MEDIA_R2_BUCKET: 'r2-bucket', MEDIA_R2_ACCESS_KEY_ID: 'k', MEDIA_R2_SECRET_ACCESS_KEY: 's',++ AWS_REQUEST_CHECKSUM_CALCULATION: 'WHEN_REQUIRED', AWS_RESPONSE_CHECKSUM_VALIDATION: 'WHEN_REQUIRED',++ MEDIA_PUBLIC_URL: 'https://media.test',++ };++ Object.assign(process.env, env);++ delete process.env.MEDIA_RECORDING_CLEANUP_DAYS;++ const db = require('../server/db/database');++ const model = require('../server/objects/model');++ const queue = require('../server/jobs/queue');++ require('../server/jobs/types');++ const cleanup = require('../server/vod/recording-cleanup');++ const job = require('../server/jobs/recording-cleanup');++ const storage = require('../server/vod/vod-storage');++ await db.upsertApp({ app_id: 'live', api_key: 'live-key-cleanup' });++ const raw = db.getDb();++ const ins = raw.prepare(`INSERT INTO vods (id, app_id, title, file_path, file_size, is_public, visibility, duration_seconds, storage_provider, storage_key) OVERRIDING SYSTEM VALUE++ VALUES (?, 'live', ?, ?, ?, 1, 'public', 60, ?, ?) RETURNING id`);++ const local = (name, size, ageDays) => { const p = path.join(env.VOD_PATH, name); fs.writeFileSync(p, Buffer.alloc(size)); if (ageDays) { const t = (Date.now() - ageDays * 86400000) / 1000; fs.utimesSync(p, t, t); } return p; };++++ // ── Fixtures ──++ // VOD 10 lives in B2 at its own key; two older copies of it sit under live.vods/ (one the same size, one not).++ await ins.run(10, 'kept', '/old/vod-live-10-1758000000010.webm', 40, 'b2', 'vods/vod-live-10-1758000000010.webm');++ put('b2-bucket', 'vods/vod-live-10-1758000000010.webm', 40);++ put('b2-bucket', 'live.vods/objects/2026/05/01/media-live-vod-10.webm', 40); // duplicate: goes++ put('b2-bucket', 'live.vods/objects/2026/05/02/media-live-vod-10.webm', 41); // another size: stays++ // VOD 11 lives locally only; an old B2 copy of another size stays.++ await ins.run(11, 'local', local('vod-live-11-1758000000011.webm', 30), 30, 'local', null);++ put('b2-bucket', 'vods/vod-live-11-1758000000099.webm', 31);++ // Deleted VODs 612 and 613 left copies; 613 is held. 614's copy is new (inside the grace period).++ put('b2-bucket', 'vods/vod-612-1758000000612.webm', 50);++ put('r2-bucket', 'vods/vod-612-1758000000612.webm', 50);++ put('b2-bucket', 'live.vods/objects/2026/05/03/media-hobostreamer-vod-612.webm', 70);++ put('b2-bucket', 'vods/vod-613-1758000000613.webm', 60);++ put('b2-bucket', 'vods/vod-614-1758000000614.webm', 60, new Date().toISOString());++ const heldObj = await model.createObject({ app_id: 'live', kind: 'vod', lifecycle_status: 'deleted', visibility: 'private', size_bytes: 60 });++ await raw.prepare("UPDATE media_objects SET legacy_ref = 'legacy:live:vod:613' WHERE id = ?").run(heldObj);++ await model.placeHold({ object_id: heldObj, kind: 'evidence', reason: 'test', created_by: 'test' });++ // Out of scope: a name with no VOD id, the parked prefix, the backups.++ put('b2-bucket', 'vods/random-upload.webm', 10);++ put('b2-bucket', 'vods-orphans/vod-700-1758000000700.webm', 10);++ put('b2-bucket', 'openvibe-pgbackrest/archive/0001.zst', 10);++ // Local: a deleted VOD's old file goes, a fresh one stays, a living VOD's file stays.++ const goneLocal = local('vod-live-612-1758000000612.webm', 20, 10);++ const freshLocal = local('vod-live-615-1758000000615.webm', 20, 0);++++ // ── A dry run lists and deletes nothing ──++ const dry = await cleanup.run({ dryRun: true });++ assert.deepStrictEqual(dry.deleted, { row_gone: 4, duplicate: 1 }, JSON.stringify(dry.items));++ assert.strictEqual(deletes.length, 0, 'a dry run deletes nothing');++ assert.ok(fs.existsSync(goneLocal));++++ // ── The real run ──++ const r = await cleanup.run();++ assert.deepStrictEqual(r.deleted, { row_gone: 4, duplicate: 1 });++ assert.strictEqual(r.bytes, 50 + 50 + 70 + 20 + 40);++ assert.deepStrictEqual(deletes.sort(), [++ 'b2-bucket:live.vods/objects/2026/05/01/media-live-vod-10.webm',++ 'b2-bucket:live.vods/objects/2026/05/03/media-hobostreamer-vod-612.webm',++ 'b2-bucket:vods/vod-612-1758000000612.webm',++ 'r2-bucket:vods/vod-612-1758000000612.webm',++ ].sort());++ assert.ok(!fs.existsSync(goneLocal) && fs.existsSync(freshLocal), 'the old local file of a deleted VOD goes; a fresh one stays');++ for (const k of ['vods/vod-live-10-1758000000010.webm', 'live.vods/objects/2026/05/02/media-live-vod-10.webm', 'vods/vod-live-11-1758000000099.webm',++ 'vods/vod-613-1758000000613.webm', 'vods/vod-614-1758000000614.webm', 'vods/random-upload.webm', 'vods-orphans/vod-700-1758000000700.webm', 'openvibe-pgbackrest/archive/0001.zst']) {++ assert.ok(buckets['b2-bucket'].has(k), `${k} stays`);++ }++ assert.ok(r.kept.held >= 1 && r.kept.young >= 2 && r.kept.size_differs >= 2 && r.kept.no_vod_id >= 1, JSON.stringify(r.kept));++ assert.match(cleanup.summarize(r), /^deleted 4 file\(s\) of deleted VODs and 1 duplicate\(s\)/);++ assert.deepStrictEqual([cleanup.vodIdOf('vod-live-2-1758000000002.webm'), cleanup.vodIdOf('vod-1621-1786231355470.webm'), cleanup.vodIdOf('media-hobostreamer-vod-612.webm'), cleanup.vodIdOf('vod-1453-1785217692199.master.mkv'), cleanup.vodIdOf('random.webm')],++ [2, 1621, 612, 1453, null]);++++ // ── The job: service-wide only, queued once a day ──++ assert.throws(() => job.spec.validate({ appId: 'live', params: {} }), /whole service/);++ assert.deepStrictEqual(job.spec.validate({ appId: queue.SYSTEM_APP, params: { dry_run: true } }), { dry_run: true });++ const day = Date.UTC(2026, 10, 3, 12);++ assert.strictEqual(await job.schedule(day), 1);++ assert.strictEqual(await job.schedule(day + 3600 * 1000), 0, 'once a day');++ process.env.MEDIA_RECORDING_CLEANUP_DAYS = '0';++ assert.strictEqual(await job.schedule(day + 3 * 86400 * 1000), 0, '0 turns it off');++++ // ── A failed remote delete is tried again ──++ storage._setDeleteRetryDelays([20, 20, 20]);++ put('b2-bucket', 'vods/vod-live-20-1758000000020.webm', 9);++ deletes.length = 0;++ failDeletes = 2;++ await storage.deleteVodObjects({ id: 20, file_path: '/old/vod-live-20-1758000000020.webm', storage_key: 'vods/vod-live-20-1758000000020.webm' });++ for (let i = 0; i < 50 && buckets['b2-bucket'].has('vods/vod-live-20-1758000000020.webm'); i++) await new Promise((res) => setTimeout(res, 20));++ assert.ok(!buckets['b2-bucket'].has('vods/vod-live-20-1758000000020.webm'), 'the B2 copy went on a retry');++++ console.log('recording clean-up: all checks passed');++ s3.close();++ process.exit(0);++})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,47 @@
@@ -0,0 +1,47 @@
+/**
+ * OpenVibe.Media — job type storage.recordings.cleanup (light lane): deletes recording files nothing uses
+ * (server/vod/recording-cleanup.js has the rules: files of deleted VODs, and duplicates whose VOD's own copy is there
+ * with the same size; a grace period, holds and every other prefix are respected). Service-wide, like
+ * storage.orphans.scan: it runs under queue.SYSTEM_APP, which no tenant reaches.
+ *
+ * params { dry_run?: boolean } result { deleted, bytes, kept, failed, checked, summary }
+ *
+ * Queued once a day (MEDIA_RECORDING_CLEANUP_DAYS, 1; 0 = never) by the worker, which does not start in a drill.
+ */
+'use strict';
+
+const queue = require('./queue');
+const cleanup = require('../vod/recording-cleanup');
+
+const { JobError } = queue;
+const TYPE = 'storage.recordings.cleanup';
+
+function validate({ appId, obj, params }) {
+ if (appId !== queue.SYSTEM_APP) throw new JobError('media.job.forbidden', `${TYPE} covers the whole service's storage: it runs on its schedule`, { status: 403, permanent: true });
+ if (obj) throw new JobError('media.job.invalid', `${TYPE} covers the whole storage: leave object_id out`, { permanent: true });
+ return { dry_run: !!(params && params.dry_run) };
+}
+
+async function run(job) {
+ const r = await cleanup.run({ dryRun: !!(job.params && job.params.dry_run) });
+ const summary = cleanup.summarize(r);
+ console.log(`[Cleanup] recordings: ${summary}`);
+ return { deleted: r.deleted, bytes: r.bytes, kept: r.kept, failed: r.failed, checked: r.checked, summary };
+}
+
+/** Queue one clean-up every MEDIA_RECORDING_CLEANUP_DAYS (default 1; 0 = never). Returns how many were queued. */
+async function schedule(nowMs = Date.now()) {
+ const raw = parseInt(process.env.MEDIA_RECORDING_CLEANUP_DAYS || '', 10);
+ const days = Number.isFinite(raw) ? raw : 1;
+ if (!(days > 0)) return 0;
+ const period = Math.floor(nowMs / (days * 86400 * 1000));
+ try {
+ const r = await queue.enqueue({ appId: queue.SYSTEM_APP, type: TYPE, params: {}, idempotencyKey: `${TYPE}:${days}d:${period}`, createdBy: 'system:schedule' });
+ return r.created ? 1 : 0;
+ } catch (err) {
+ console.warn(`[Jobs] could not schedule ${TYPE}: ${err.message}`);
+ return 0;
+ }
+}
+
+module.exports = { TYPE, spec: { lane: 'light', maxAttempts: 2, timeoutMs: 2 * 60 * 60 * 1000, needsObject: false, validate, run }, schedule };
@@ -36,5 +36,6 @@ queue.register('vod.duration.reconcile', require('./duration-reconcile').spec);
@@ -36,5 +36,6 @@ queue.register('vod.duration.reconcile', require('./duration-reconcile').spec);
queue.register('object.hash', require('./content-hash').spec);
queue.register('object.hash', require('./content-hash').spec);
queue.register('storage.orphans.scan', require('./storage-orphans').spec);
queue.register('storage.orphans.scan', require('./storage-orphans').spec);
queue.register('storage.move.cleanup', require('./move-cleanup').spec);
queue.register('storage.move.cleanup', require('./move-cleanup').spec);
+queue.register('storage.recordings.cleanup', require('./recording-cleanup').spec);
module.exports = { names: () => queue.typeNames() };
module.exports = { names: () => queue.typeNames() };
@@ -169,6 +169,7 @@ async function tick() {
@@ -169,6 +169,7 @@ async function tick() {
try { await require('./content-hash').schedule(now); } catch (err) { console.warn('[Jobs] content hash schedule:', err.message); }
try { await require('./content-hash').schedule(now); } catch (err) { console.warn('[Jobs] content hash schedule:', err.message); }
try { await require('./storage-orphans').schedule(now); } catch (err) { console.warn('[Jobs] storage orphan scan schedule:', err.message); }
try { await require('./storage-orphans').schedule(now); } catch (err) { console.warn('[Jobs] storage orphan scan schedule:', err.message); }
try { await require('./move-cleanup').schedule(now); } catch (err) { console.warn('[Jobs] move cleanup schedule:', err.message); }
try { await require('./move-cleanup').schedule(now); } catch (err) { console.warn('[Jobs] move cleanup schedule:', err.message); }
+ try { await require('./recording-cleanup').schedule(now); } catch (err) { console.warn('[Jobs] recording clean-up schedule:', err.message); }
}
}
try { await require('./vod-finalize').sweepOrphans({ now }); } catch (err) { console.warn('[Jobs] orphan sweep:', err.message); }
try { await require('./vod-finalize').sweepOrphans({ now }); } catch (err) { console.warn('[Jobs] orphan sweep:', err.message); }
if (now - lastSweepAt > 3600 * 1000) {
if (now - lastSweepAt > 3600 * 1000) {
new file mode 100644
new file mode 100644
@@ -0,0 +1,137 @@
@@ -0,0 +1,137 @@
+'use strict';
+/**
+ * OpenVibe.Media — automatic clean-up of recording files nothing uses (the owner, 2026-11-03: "all this should be taken
+ * care of automatically"). The storage orphan report (orphans-report.js) only lists; this deletes the cases that are
+ * certain, and keeps everything else:
+ *
+ * row_gone a recording file (the local VOD directory; B2 and R2 under vods/ and live.vods/) whose VOD id, read from
+ * its name, has no vods row any more. The VOD was deleted and a copy stayed behind: a remote delete that
+ * failed, or an older layout's copy the delete never knew about (2026-11-03: 31 such files, 18 GB, the
+ * newest from August). Deleted unless an object for that VOD is under a retention hold.
+ * duplicate a B2/R2 key that no row or location names, for a VOD that still exists, when that VOD's own copy (its
+ * key on the same provider, else on the other, else its local file) is there with the same size. Local
+ * files are never treated as duplicates (a recording in progress writes beside its row).
+ *
+ * Both wait out a grace period (7 days) from the file's last change. Kept: a name with no VOD id, a copy whose size
+ * differs from the VOD's, anything held, and every other prefix of the bucket (vods-orphans/ is parked for review; the
+ * backups share the bucket and are never listed here).
+ *
+ * run({ dryRun, graceDays, now }) → { checked, deleted: { row_gone, duplicate }, bytes, kept: {…}, failed, items }
+ *
+ * Job storage.recordings.cleanup (server/jobs/recording-cleanup.js) runs it daily.
+ */
+const fs = require('fs');
+const path = require('path');
+const db = require('../db/database');
+const config = require('../config');
+
+const PREFIXES = ['vods/', 'live.vods/'];
+const GRACE_DAYS = 7;
+
+const storage = () => require('./vod-storage');
+// The remote providers vod-storage knows (B2 and R2 today); placement decides their order elsewhere.
+const providers = () => storage().REMOTE_PROVIDERS;
+
+/** The VOD id a recording file's name carries: vod-<app>-<id>-<ms>, vod-<id>-<ms>, media-<app>-vod-<id>. */
+function vodIdOf(key) {
+ const name = path.basename(String(key || ''));
+ let m = /^vod-(?:[a-z][a-z0-9._]*-)?(\d+)-\d{10,}(?:\.|$)/i.exec(name);
+ if (!m) m = /^media-[a-z0-9._]+-vod-(\d+)\./i.exec(name);
+ return m ? Number(m[1]) : null;
+}
+
+async function heldVod(id) {
+ const rows = await db.all('SELECT id FROM media_objects WHERE legacy_ref LIKE ?', [`legacy:%:vod:${id}`]);
+ const model = require('../objects/model');
+ for (const r of rows) if (await model.isHeld(r.id)) return true;
+ const row = await db.get('SELECT * FROM vods WHERE id = ?', [id]);
+ return !!(row && await model.isHeldRow(row));
+}
+
+/** The size of a VOD's own copy: its key on `provider`, then on the other provider, then its local file; null if none. */
+async function ownCopySize(vod, provider) {
+ const key = storage().keyForVod(vod);
+ for (const p of [provider, ...providers().filter((x) => x !== provider)]) {
+ if (!storage().providerConfigured(p)) continue;
+ const h = await storage().headObject(p, key).catch(() => null);
+ if (h) return Number(h.size);
+ }
+ try { return fs.statSync(storage().localPathForVod(vod)).size; } catch { return null; }
+}
+
+async function run({ dryRun = false, graceDays = GRACE_DAYS, now = Date.now() } = {}) {
+ const cutoff = now - Math.max(0, graceDays) * 86400 * 1000;
+ const vods = new Map((await db.all('SELECT * FROM vods')).map((v) => [Number(v.id), v]));
+ const named = new Set();
+ for (const v of vods.values()) {
+ if (v.storage_key) named.add(v.storage_key);
+ if (v.file_path) named.add('vods/' + path.basename(v.file_path));
+ }
+ for (const r of await db.all("SELECT key FROM media_locations WHERE provider IN ('b2', 'r2')")) named.add(r.key);
+ const localNamed = new Set();
+ for (const v of vods.values()) for (const f of [v.file_path, v.master_file_path]) if (f) localNamed.add(path.basename(f));
+
+ const out = { dry_run: !!dryRun, checked: 0, deleted: { row_gone: 0, duplicate: 0 }, bytes: 0, failed: 0,
+ kept: { young: 0, no_vod_id: 0, held: 0, size_differs: 0, no_own_copy: 0 }, items: [] };
+ const heldCache = new Map();
+ const isHeld = async (id) => { if (!heldCache.has(id)) heldCache.set(id, await heldVod(id)); return heldCache.get(id); };
+ const remove = async (where, key, size, reason) => {
+ out.items.push({ where, key, size, reason });
+ if (dryRun) { out.deleted[reason]++; out.bytes += size; return; }
+ let ok;
+ if (where === 'local') { try { fs.unlinkSync(key); ok = true; } catch (err) { ok = err.code === 'ENOENT'; } }
+ else ok = await storage().deleteObject(where, key);
+ if (ok) { out.deleted[reason]++; out.bytes += size; } else out.failed++;
+ };
+
+ // Remote: only the recording prefixes; every other key in the bucket is out of scope.
+ for (const provider of providers()) {
+ if (!storage().providerConfigured(provider)) continue;
+ for (const prefix of PREFIXES) {
+ let list = [];
+ try { list = await storage().listObjects(provider, prefix); } catch (err) { console.warn(`[Cleanup] list ${provider}:${prefix}: ${err.message}`); continue; }
+ for (const o of list) {
+ out.checked++;
+ if (named.has(o.key)) continue;
+ const id = vodIdOf(o.key);
+ if (id == null) { out.kept.no_vod_id++; continue; }
+ if (o.last_modified && Date.parse(o.last_modified) > cutoff) { out.kept.young++; continue; }
+ if (await isHeld(id)) { out.kept.held++; continue; }
+ const vod = vods.get(id);
+ if (!vod) { await remove(provider, o.key, o.size, 'row_gone'); continue; }
+ const own = await ownCopySize(vod, provider);
+ if (own == null) { out.kept.no_own_copy++; continue; }
+ if (own !== o.size) { out.kept.size_differs++; continue; }
+ await remove(provider, o.key, o.size, 'duplicate');
+ }
+ }
+ }
+
+ // Local: the VOD directory's own files whose VOD is gone (never a duplicate: a recording in progress writes here).
+ const dir = path.resolve(config.vod.path);
+ let names = [];
+ try { names = fs.readdirSync(dir, { withFileTypes: true }).filter((e) => e.isFile()).map((e) => e.name); } catch { /* no directory */ }
+ for (const name of names) {
+ out.checked++;
+ if (localNamed.has(name)) continue;
+ const id = vodIdOf(name);
+ if (id == null) { out.kept.no_vod_id++; continue; }
+ if (vods.has(id)) continue;
+ const file = path.join(dir, name);
+ let st;
+ try { st = fs.statSync(file); } catch { continue; }
+ if (st.mtimeMs > cutoff) { out.kept.young++; continue; }
+ if (await isHeld(id)) { out.kept.held++; continue; }
+ await remove('local', file, st.size, 'row_gone');
+ }
+ return out;
+}
+
+/** One line for the log. */
+function summarize(r) {
+ const gb = (r.bytes / 1e9).toFixed(2);
+ const kept = Object.entries(r.kept).filter(([, n]) => n).map(([k, n]) => `${n} ${k.replace(/_/g, ' ')}`).join(', ') || 'nothing';
+ return `${r.dry_run ? 'would delete' : 'deleted'} ${r.deleted.row_gone} file(s) of deleted VODs and ${r.deleted.duplicate} duplicate(s), ${gb} GB; kept ${kept}${r.failed ? `; ${r.failed} delete(s) failed` : ''} (${r.checked} checked)`;
+}
+
+module.exports = { run, summarize, vodIdOf, PREFIXES, GRACE_DAYS };
@@ -826,10 +826,26 @@ async function deleteVodObjects(vod) {
@@ -826,10 +826,26 @@ async function deleteVodObjects(vod) {
await cleanupSidecar(local);
await cleanupSidecar(local);
const key = keyForVod(vod);
const key = keyForVod(vod);
for (const p of REMOTE_PROVIDERS) {
for (const p of REMOTE_PROVIDERS) {
- if (providerConfigured(p)) await deleteObject(p, key);
+ if (providerConfigured(p) && !await deleteObject(p, key)) retryDeleteLater(p, key);
}
}
}
}
+// A remote delete that failed is tried again a few times in the background (the owner, 2026-11-03: "retry … up to a
+// certain amount"): after 30 s, 2, 10 and 30 minutes. If all fail, the daily recording clean-up
+// (recording-cleanup.js) still removes the copy its deleted VOD left. Timers are unref'd and nothing is persisted:
+// a restart drops the retries, and the clean-up is the backstop.
+let DELETE_RETRY_MS = [30_000, 120_000, 600_000, 1_800_000];
+function retryDeleteLater(provider, key) {
+ (async () => {
+ for (const ms of DELETE_RETRY_MS) {
+ await new Promise((r) => { const t = setTimeout(r, ms); if (t.unref) t.unref(); });
+ if (await deleteObject(provider, key)) { console.log(`[VodStorage] ${provider}:${key} deleted on a retry`); return; }
+ }
+ console.warn(`[VodStorage] ${provider}:${key} still not deleted after ${DELETE_RETRY_MS.length + 1} tries; the daily recording clean-up will remove it`);
+ })().catch(() => {});
+}
+function _setDeleteRetryDelays(ms) { DELETE_RETRY_MS = ms; }
+
// ── Disk usage ───────────────────────────────────────────────
// ── Disk usage ───────────────────────────────────────────────
/**
/**
@@ -1560,6 +1576,7 @@ module.exports = {
@@ -1560,6 +1576,7 @@ module.exports = {
r2Policy,
r2Policy,
recordTierDecision,
recordTierDecision,
deleteVodObjects,
deleteVodObjects,
+ _setDeleteRetryDelays,
deleteObject,
deleteObject,
presignGet,
presignGet,
runSweep,
runSweep,
new file mode 100644
new file mode 100644
@@ -0,0 +1,147 @@
@@ -0,0 +1,147 @@
+'use strict';
+// The recording clean-up (server/vod/recording-cleanup.js; job storage.recordings.cleanup) and the retried remote
+// delete (vod-storage.js deleteVodObjects). Against a fake S3 spoken to by the real AWS SDK:
+// - a file of a deleted VOD goes (B2, R2 and local), unless it is under the grace period or its VOD is held;
+// - a duplicate of a living VOD goes only when the VOD's own copy is there with the same size;
+// - a name with no VOD id, a different-size copy and every other prefix (backups, vods-orphans/) stay;
+// - a dry run deletes nothing; the job is service-wide only and queued daily;
+// - a failed remote delete is tried again until it succeeds.
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const http = require('http');
+
+(async () => {
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-media-cleanup-'));
+ const dir = (n) => { const d = path.join(tmp, n); fs.mkdirSync(d, { recursive: true }); return d; };
+ const OLD = '2026-09-21T00:00:00.000Z';
+ const buckets = { 'b2-bucket': new Map(), 'r2-bucket': new Map() };
+ const put = (bucket, key, size, modified = OLD) => buckets[bucket].set(key, { size, modified });
+ const deletes = [];
+ let failDeletes = 0;
+ const xml = (v) => String(v).replace(/&/g, '&').replace(/</g, '<');
+ const s3 = http.createServer((req, res) => {
+ req.resume();
+ req.on('end', () => {
+ const u = new URL(req.url, 'http://x');
+ const [, bucket, ...rest] = decodeURIComponent(u.pathname).split('/');
+ const key = rest.join('/');
+ const store = buckets[bucket];
+ if (!store) { res.writeHead(404); return res.end(); }
+ if (req.method === 'DELETE') {
+ if (failDeletes > 0) { failDeletes--; res.writeHead(500, { 'Content-Type': 'application/xml' }); return res.end('<Error><Code>InternalError</Code></Error>'); }
+ deletes.push(`${bucket}:${key}`); store.delete(key); res.writeHead(204); return res.end();
+ }
+ if (!key && u.searchParams.get('list-type') === '2') {
+ const prefix = u.searchParams.get('prefix') || '';
+ const keys = [...store.keys()].filter((k) => k.startsWith(prefix)).sort();
+ res.writeHead(200, { 'Content-Type': 'application/xml' });
+ return res.end(`<?xml version="1.0" encoding="UTF-8"?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>${bucket}</Name><Prefix>${xml(prefix)}</Prefix><KeyCount>${keys.length}</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>${
+ keys.map((k) => `<Contents><Key>${xml(k)}</Key><LastModified>${store.get(k).modified}</LastModified><ETag>"e"</ETag><Size>${store.get(k).size}</Size><StorageClass>STANDARD</StorageClass></Contents>`).join('')}</ListBucketResult>`);
+ }
+ const obj = store.get(key);
+ if (!obj) { res.writeHead(404); return res.end(); }
+ res.writeHead(200, { 'Content-Length': obj.size, ETag: '"e"' });
+ res.end(req.method === 'GET' ? Buffer.alloc(obj.size) : undefined);
+ });
+ });
+ await new Promise((r) => s3.listen(0, '127.0.0.1', r));
+ const endpoint = `http://127.0.0.1:${s3.address().port}`;
+ const env = {
+ MEDIA_DATA_DIR: path.join(tmp, 'data'), VOD_PATH: dir('vods'), CLIPS_PATH: dir('clips'), FILES_PATH: dir('files'),
+ THUMBNAILS_PATH: dir('thumbnails'), PASTES_PATH: dir('pastes'), OBJECTS_PATH: dir('objects'), ASSETS_PATH: dir('assets'),
+ MEDIA_B2_ENDPOINT: endpoint, MEDIA_B2_BUCKET: 'b2-bucket', MEDIA_B2_KEY_ID: 'k', MEDIA_B2_APP_KEY: 's', MEDIA_B2_REGION: 'us-west-004',
+ MEDIA_R2_ENDPOINT: endpoint, MEDIA_R2_BUCKET: 'r2-bucket', MEDIA_R2_ACCESS_KEY_ID: 'k', MEDIA_R2_SECRET_ACCESS_KEY: 's',
+ AWS_REQUEST_CHECKSUM_CALCULATION: 'WHEN_REQUIRED', AWS_RESPONSE_CHECKSUM_VALIDATION: 'WHEN_REQUIRED',
+ MEDIA_PUBLIC_URL: 'https://media.test',
+ };
+ Object.assign(process.env, env);
+ delete process.env.MEDIA_RECORDING_CLEANUP_DAYS;
+ const db = require('../server/db/database');
+ const model = require('../server/objects/model');
+ const queue = require('../server/jobs/queue');
+ require('../server/jobs/types');
+ const cleanup = require('../server/vod/recording-cleanup');
+ const job = require('../server/jobs/recording-cleanup');
+ const storage = require('../server/vod/vod-storage');
+ await db.upsertApp({ app_id: 'live', api_key: 'live-key-cleanup' });
+ const raw = db.getDb();
+ const ins = raw.prepare(`INSERT INTO vods (id, app_id, title, file_path, file_size, is_public, visibility, duration_seconds, storage_provider, storage_key) OVERRIDING SYSTEM VALUE
+ VALUES (?, 'live', ?, ?, ?, 1, 'public', 60, ?, ?) RETURNING id`);
+ const local = (name, size, ageDays) => { const p = path.join(env.VOD_PATH, name); fs.writeFileSync(p, Buffer.alloc(size)); if (ageDays) { const t = (Date.now() - ageDays * 86400000) / 1000; fs.utimesSync(p, t, t); } return p; };
+
+ // ── Fixtures ──
+ // VOD 10 lives in B2 at its own key; two older copies of it sit under live.vods/ (one the same size, one not).
+ await ins.run(10, 'kept', '/old/vod-live-10-1758000000010.webm', 40, 'b2', 'vods/vod-live-10-1758000000010.webm');
+ put('b2-bucket', 'vods/vod-live-10-1758000000010.webm', 40);
+ put('b2-bucket', 'live.vods/objects/2026/05/01/media-live-vod-10.webm', 40); // duplicate: goes
+ put('b2-bucket', 'live.vods/objects/2026/05/02/media-live-vod-10.webm', 41); // another size: stays
+ // VOD 11 lives locally only; an old B2 copy of another size stays.
+ await ins.run(11, 'local', local('vod-live-11-1758000000011.webm', 30), 30, 'local', null);
+ put('b2-bucket', 'vods/vod-live-11-1758000000099.webm', 31);
+ // Deleted VODs 612 and 613 left copies; 613 is held. 614's copy is new (inside the grace period).
+ put('b2-bucket', 'vods/vod-612-1758000000612.webm', 50);
+ put('r2-bucket', 'vods/vod-612-1758000000612.webm', 50);
+ put('b2-bucket', 'live.vods/objects/2026/05/03/media-hobostreamer-vod-612.webm', 70);
+ put('b2-bucket', 'vods/vod-613-1758000000613.webm', 60);
+ put('b2-bucket', 'vods/vod-614-1758000000614.webm', 60, new Date().toISOString());
+ const heldObj = await model.createObject({ app_id: 'live', kind: 'vod', lifecycle_status: 'deleted', visibility: 'private', size_bytes: 60 });
+ await raw.prepare("UPDATE media_objects SET legacy_ref = 'legacy:live:vod:613' WHERE id = ?").run(heldObj);
+ await model.placeHold({ object_id: heldObj, kind: 'evidence', reason: 'test', created_by: 'test' });
+ // Out of scope: a name with no VOD id, the parked prefix, the backups.
+ put('b2-bucket', 'vods/random-upload.webm', 10);
+ put('b2-bucket', 'vods-orphans/vod-700-1758000000700.webm', 10);
+ put('b2-bucket', 'openvibe-pgbackrest/archive/0001.zst', 10);
+ // Local: a deleted VOD's old file goes, a fresh one stays, a living VOD's file stays.
+ const goneLocal = local('vod-live-612-1758000000612.webm', 20, 10);
+ const freshLocal = local('vod-live-615-1758000000615.webm', 20, 0);
+
+ // ── A dry run lists and deletes nothing ──
+ const dry = await cleanup.run({ dryRun: true });
+ assert.deepStrictEqual(dry.deleted, { row_gone: 4, duplicate: 1 }, JSON.stringify(dry.items));
+ assert.strictEqual(deletes.length, 0, 'a dry run deletes nothing');
+ assert.ok(fs.existsSync(goneLocal));
+
+ // ── The real run ──
+ const r = await cleanup.run();
+ assert.deepStrictEqual(r.deleted, { row_gone: 4, duplicate: 1 });
+ assert.strictEqual(r.bytes, 50 + 50 + 70 + 20 + 40);
+ assert.deepStrictEqual(deletes.sort(), [
+ 'b2-bucket:live.vods/objects/2026/05/01/media-live-vod-10.webm',
+ 'b2-bucket:live.vods/objects/2026/05/03/media-hobostreamer-vod-612.webm',
+ 'b2-bucket:vods/vod-612-1758000000612.webm',
+ 'r2-bucket:vods/vod-612-1758000000612.webm',
+ ].sort());
+ assert.ok(!fs.existsSync(goneLocal) && fs.existsSync(freshLocal), 'the old local file of a deleted VOD goes; a fresh one stays');
+ for (const k of ['vods/vod-live-10-1758000000010.webm', 'live.vods/objects/2026/05/02/media-live-vod-10.webm', 'vods/vod-live-11-1758000000099.webm',
+ 'vods/vod-613-1758000000613.webm', 'vods/vod-614-1758000000614.webm', 'vods/random-upload.webm', 'vods-orphans/vod-700-1758000000700.webm', 'openvibe-pgbackrest/archive/0001.zst']) {
+ assert.ok(buckets['b2-bucket'].has(k), `${k} stays`);
+ }
+ assert.ok(r.kept.held >= 1 && r.kept.young >= 2 && r.kept.size_differs >= 2 && r.kept.no_vod_id >= 1, JSON.stringify(r.kept));
+ assert.match(cleanup.summarize(r), /^deleted 4 file\(s\) of deleted VODs and 1 duplicate\(s\)/);
+ assert.deepStrictEqual([cleanup.vodIdOf('vod-live-2-1758000000002.webm'), cleanup.vodIdOf('vod-1621-1786231355470.webm'), cleanup.vodIdOf('media-hobostreamer-vod-612.webm'), cleanup.vodIdOf('vod-1453-1785217692199.master.mkv'), cleanup.vodIdOf('random.webm')],
+ [2, 1621, 612, 1453, null]);
+
+ // ── The job: service-wide only, queued once a day ──
+ assert.throws(() => job.spec.validate({ appId: 'live', params: {} }), /whole service/);
+ assert.deepStrictEqual(job.spec.validate({ appId: queue.SYSTEM_APP, params: { dry_run: true } }), { dry_run: true });
+ const day = Date.UTC(2026, 10, 3, 12);
+ assert.strictEqual(await job.schedule(day), 1);
+ assert.strictEqual(await job.schedule(day + 3600 * 1000), 0, 'once a day');
+ process.env.MEDIA_RECORDING_CLEANUP_DAYS = '0';
+ assert.strictEqual(await job.schedule(day + 3 * 86400 * 1000), 0, '0 turns it off');
+
+ // ── A failed remote delete is tried again ──
+ storage._setDeleteRetryDelays([20, 20, 20]);
+ put('b2-bucket', 'vods/vod-live-20-1758000000020.webm', 9);
+ deletes.length = 0;
+ failDeletes = 2;
+ await storage.deleteVodObjects({ id: 20, file_path: '/old/vod-live-20-1758000000020.webm', storage_key: 'vods/vod-live-20-1758000000020.webm' });
+ for (let i = 0; i < 50 && buckets['b2-bucket'].has('vods/vod-live-20-1758000000020.webm'); i++) await new Promise((res) => setTimeout(res, 20));
+ assert.ok(!buckets['b2-bucket'].has('vods/vod-live-20-1758000000020.webm'), 'the B2 copy went on a retry');
+
+ console.log('recording clean-up: all checks passed');
+ s3.close();
+ process.exit(0);
+})().catch((e) => { console.error(e); process.exit(1); });