openvibers/openvibe.media Commits

Recordings nothing uses are cleaned up daily; a failed remote delete is retried (#307)

OP OpenVibers committed 10 days ago · 2d00bc54d5 · 6 files, +351 −1

The orphan scan only reported, so leftovers stayed: 31 files of deleted VODs (18 GB in B2, newest from August) and
81 May copies of VODs that still exist (~18.7 GB). storage.recordings.cleanup (daily) deletes files of deleted VODs
and same-size duplicates of living ones, after a 7-day grace, skipping holds; everything else is kept. A VOD's
failed B2/R2 delete is tried again after 30 s, 2, 10 and 30 minutes.

Changes

ddiff --git a/server/jobs/recording-cleanup.js b/server/jobs/recording-cleanup.js
nnew file mode 100644
iindex 0000000..62878de
---- /dev/null
++++ b/server/jobs/recording-cleanup.js
@@@ -0,0 +1,47 @@
++/**
++ * OpenVibe.Media — job type storage.recordings.cleanup (light lane): deletes recording files nothing uses
++ * (server/vod/recording-cleanup.js has the rules: files of deleted VODs, and duplicates whose VOD's own copy is there
++ * with the same size; a grace period, holds and every other prefix are respected). Service-wide, like
++ * storage.orphans.scan: it runs under queue.SYSTEM_APP, which no tenant reaches.
++ *
++ * params { dry_run?: boolean } result { deleted, bytes, kept, failed, checked, summary }
++ *
++ * Queued once a day (MEDIA_RECORDING_CLEANUP_DAYS, 1; 0 = never) by the worker, which does not start in a drill.
++ */
++'use strict';
++
++const queue = require('./queue');
++const cleanup = require('../vod/recording-cleanup');
++
++const { JobError } = queue;
++const TYPE = 'storage.recordings.cleanup';
++
++function validate({ appId, obj, params }) {
++ if (appId !== queue.SYSTEM_APP) throw new JobError('media.job.forbidden', `${TYPE} covers the whole service's storage: it runs on its schedule`, { status: 403, permanent: true });
++ if (obj) throw new JobError('media.job.invalid', `${TYPE} covers the whole storage: leave object_id out`, { permanent: true });
++ return { dry_run: !!(params && params.dry_run) };
++}
++
++async function run(job) {
++ const r = await cleanup.run({ dryRun: !!(job.params && job.params.dry_run) });
++ const summary = cleanup.summarize(r);
++ console.log(`[Cleanup] recordings: ${summary}`);
++ return { deleted: r.deleted, bytes: r.bytes, kept: r.kept, failed: r.failed, checked: r.checked, summary };
++}
++
++/** Queue one clean-up every MEDIA_RECORDING_CLEANUP_DAYS (default 1; 0 = never). Returns how many were queued. */
++async function schedule(nowMs = Date.now()) {
++ const raw = parseInt(process.env.MEDIA_RECORDING_CLEANUP_DAYS || '', 10);
++ const days = Number.isFinite(raw) ? raw : 1;
++ if (!(days > 0)) return 0;
++ const period = Math.floor(nowMs / (days * 86400 * 1000));
++ try {
++ const r = await queue.enqueue({ appId: queue.SYSTEM_APP, type: TYPE, params: {}, idempotencyKey: `${TYPE}:${days}d:${period}`, createdBy: 'system:schedule' });
++ return r.created ? 1 : 0;
++ } catch (err) {
++ console.warn(`[Jobs] could not schedule ${TYPE}: ${err.message}`);
++ return 0;
++ }
++}
++
++module.exports = { TYPE, spec: { lane: 'light', maxAttempts: 2, timeoutMs: 2 * 60 * 60 * 1000, needsObject: false, validate, run }, schedule };
ddiff --git a/server/jobs/types.js b/server/jobs/types.js
iindex faae5b7..ebda20b 100644
---- a/server/jobs/types.js
++++ b/server/jobs/types.js
@@@ -36,5 +36,6 @@ queue.register('vod.duration.reconcile', require('./duration-reconcile').spec);
queue.register('object.hash', require('./content-hash').spec);
queue.register('storage.orphans.scan', require('./storage-orphans').spec);
queue.register('storage.move.cleanup', require('./move-cleanup').spec);
++queue.register('storage.recordings.cleanup', require('./recording-cleanup').spec);
module.exports = { names: () => queue.typeNames() };
ddiff --git a/server/jobs/worker.js b/server/jobs/worker.js
iindex 8e9d9ae..edecc20 100644
---- a/server/jobs/worker.js
++++ b/server/jobs/worker.js
@@@ -169,6 +169,7 @@ async function tick() {
try { await require('./content-hash').schedule(now); } catch (err) { console.warn('[Jobs] content hash schedule:', err.message); }
try { await require('./storage-orphans').schedule(now); } catch (err) { console.warn('[Jobs] storage orphan scan schedule:', err.message); }
try { await require('./move-cleanup').schedule(now); } catch (err) { console.warn('[Jobs] move cleanup schedule:', err.message); }
++ try { await require('./recording-cleanup').schedule(now); } catch (err) { console.warn('[Jobs] recording clean-up schedule:', err.message); }
}
try { await require('./vod-finalize').sweepOrphans({ now }); } catch (err) { console.warn('[Jobs] orphan sweep:', err.message); }
if (now - lastSweepAt > 3600 * 1000) {
ddiff --git a/server/vod/recording-cleanup.js b/server/vod/recording-cleanup.js
nnew file mode 100644
iindex 0000000..44d2404
---- /dev/null
++++ b/server/vod/recording-cleanup.js
@@@ -0,0 +1,137 @@
++'use strict';
++/**
++ * OpenVibe.Media — automatic clean-up of recording files nothing uses (the owner, 2026-11-03: "all this should be taken
++ * care of automatically"). The storage orphan report (orphans-report.js) only lists; this deletes the cases that are
++ * certain, and keeps everything else:
++ *
++ * row_gone a recording file (the local VOD directory; B2 and R2 under vods/ and live.vods/) whose VOD id, read from
++ * its name, has no vods row any more. The VOD was deleted and a copy stayed behind: a remote delete that
++ * failed, or an older layout's copy the delete never knew about (2026-11-03: 31 such files, 18 GB, the
++ * newest from August). Deleted unless an object for that VOD is under a retention hold.
++ * duplicate a B2/R2 key that no row or location names, for a VOD that still exists, when that VOD's own copy (its
++ * key on the same provider, else on the other, else its local file) is there with the same size. Local
++ * files are never treated as duplicates (a recording in progress writes beside its row).
++ *
++ * Both wait out a grace period (7 days) from the file's last change. Kept: a name with no VOD id, a copy whose size
++ * differs from the VOD's, anything held, and every other prefix of the bucket (vods-orphans/ is parked for review; the
++ * backups share the bucket and are never listed here).
++ *
++ * run({ dryRun, graceDays, now }) → { checked, deleted: { row_gone, duplicate }, bytes, kept: {…}, failed, items }
++ *
++ * Job storage.recordings.cleanup (server/jobs/recording-cleanup.js) runs it daily.
++ */
++const fs = require('fs');
++const path = require('path');
++const db = require('../db/database');
++const config = require('../config');
++
++const PREFIXES = ['vods/', 'live.vods/'];
++const GRACE_DAYS = 7;
++
++const storage = () => require('./vod-storage');
++// The remote providers vod-storage knows (B2 and R2 today); placement decides their order elsewhere.
++const providers = () => storage().REMOTE_PROVIDERS;
++
++/** The VOD id a recording file's name carries: vod-<app>-<id>-<ms>, vod-<id>-<ms>, media-<app>-vod-<id>. */
++function vodIdOf(key) {
++ const name = path.basename(String(key || ''));
++ let m = /^vod-(?:[a-z][a-z0-9._]*-)?(\d+)-\d{10,}(?:\.|$)/i.exec(name);
++ if (!m) m = /^media-[a-z0-9._]+-vod-(\d+)\./i.exec(name);
++ return m ? Number(m[1]) : null;
++}
++
++async function heldVod(id) {
++ const rows = await db.all('SELECT id FROM media_objects WHERE legacy_ref LIKE ?', [`legacy:%:vod:${id}`]);
++ const model = require('../objects/model');
++ for (const r of rows) if (await model.isHeld(r.id)) return true;
++ const row = await db.get('SELECT * FROM vods WHERE id = ?', [id]);
++ return !!(row && await model.isHeldRow(row));
++}
++
++/** The size of a VOD's own copy: its key on `provider`, then on the other provider, then its local file; null if none. */
++async function ownCopySize(vod, provider) {
++ const key = storage().keyForVod(vod);
++ for (const p of [provider, ...providers().filter((x) => x !== provider)]) {
++ if (!storage().providerConfigured(p)) continue;
++ const h = await storage().headObject(p, key).catch(() => null);
++ if (h) return Number(h.size);
++ }
++ try { return fs.statSync(storage().localPathForVod(vod)).size; } catch { return null; }
++}
++
++async function run({ dryRun = false, graceDays = GRACE_DAYS, now = Date.now() } = {}) {
++ const cutoff = now - Math.max(0, graceDays) * 86400 * 1000;
++ const vods = new Map((await db.all('SELECT * FROM vods')).map((v) => [Number(v.id), v]));
++ const named = new Set();
++ for (const v of vods.values()) {
++ if (v.storage_key) named.add(v.storage_key);
++ if (v.file_path) named.add('vods/' + path.basename(v.file_path));
++ }
++ for (const r of await db.all("SELECT key FROM media_locations WHERE provider IN ('b2', 'r2')")) named.add(r.key);
++ const localNamed = new Set();
++ for (const v of vods.values()) for (const f of [v.file_path, v.master_file_path]) if (f) localNamed.add(path.basename(f));
++
++ const out = { dry_run: !!dryRun, checked: 0, deleted: { row_gone: 0, duplicate: 0 }, bytes: 0, failed: 0,
++ kept: { young: 0, no_vod_id: 0, held: 0, size_differs: 0, no_own_copy: 0 }, items: [] };
++ const heldCache = new Map();
++ const isHeld = async (id) => { if (!heldCache.has(id)) heldCache.set(id, await heldVod(id)); return heldCache.get(id); };
++ const remove = async (where, key, size, reason) => {
++ out.items.push({ where, key, size, reason });
++ if (dryRun) { out.deleted[reason]++; out.bytes += size; return; }
++ let ok;
++ if (where === 'local') { try { fs.unlinkSync(key); ok = true; } catch (err) { ok = err.code === 'ENOENT'; } }
++ else ok = await storage().deleteObject(where, key);
++ if (ok) { out.deleted[reason]++; out.bytes += size; } else out.failed++;
++ };
++
++ // Remote: only the recording prefixes; every other key in the bucket is out of scope.
++ for (const provider of providers()) {
++ if (!storage().providerConfigured(provider)) continue;
++ for (const prefix of PREFIXES) {
++ let list = [];
++ try { list = await storage().listObjects(provider, prefix); } catch (err) { console.warn(`[Cleanup] list ${provider}:${prefix}: ${err.message}`); continue; }
++ for (const o of list) {
++ out.checked++;
++ if (named.has(o.key)) continue;
++ const id = vodIdOf(o.key);
++ if (id == null) { out.kept.no_vod_id++; continue; }
++ if (o.last_modified && Date.parse(o.last_modified) > cutoff) { out.kept.young++; continue; }
++ if (await isHeld(id)) { out.kept.held++; continue; }
++ const vod = vods.get(id);
++ if (!vod) { await remove(provider, o.key, o.size, 'row_gone'); continue; }
++ const own = await ownCopySize(vod, provider);
++ if (own == null) { out.kept.no_own_copy++; continue; }
++ if (own !== o.size) { out.kept.size_differs++; continue; }
++ await remove(provider, o.key, o.size, 'duplicate');
++ }
++ }
++ }
++
++ // Local: the VOD directory's own files whose VOD is gone (never a duplicate: a recording in progress writes here).
++ const dir = path.resolve(config.vod.path);
++ let names = [];
++ try { names = fs.readdirSync(dir, { withFileTypes: true }).filter((e) => e.isFile()).map((e) => e.name); } catch { /* no directory */ }
++ for (const name of names) {
++ out.checked++;
++ if (localNamed.has(name)) continue;
++ const id = vodIdOf(name);
++ if (id == null) { out.kept.no_vod_id++; continue; }
++ if (vods.has(id)) continue;
++ const file = path.join(dir, name);
++ let st;
++ try { st = fs.statSync(file); } catch { continue; }
++ if (st.mtimeMs > cutoff) { out.kept.young++; continue; }
++ if (await isHeld(id)) { out.kept.held++; continue; }
++ await remove('local', file, st.size, 'row_gone');
++ }
++ return out;
++}
++
++/** One line for the log. */
++function summarize(r) {
++ const gb = (r.bytes / 1e9).toFixed(2);
++ const kept = Object.entries(r.kept).filter(([, n]) => n).map(([k, n]) => `${n} ${k.replace(/_/g, ' ')}`).join(', ') || 'nothing';
++ return `${r.dry_run ? 'would delete' : 'deleted'} ${r.deleted.row_gone} file(s) of deleted VODs and ${r.deleted.duplicate} duplicate(s), ${gb} GB; kept ${kept}${r.failed ? `; ${r.failed} delete(s) failed` : ''} (${r.checked} checked)`;
++}
++
++module.exports = { run, summarize, vodIdOf, PREFIXES, GRACE_DAYS };
ddiff --git a/server/vod/vod-storage.js b/server/vod/vod-storage.js
iindex aadea30..e075538 100644
---- a/server/vod/vod-storage.js
++++ b/server/vod/vod-storage.js
@@@ -826,10 +826,26 @@ async function deleteVodObjects(vod) {
await cleanupSidecar(local);
const key = keyForVod(vod);
for (const p of REMOTE_PROVIDERS) {
-- if (providerConfigured(p)) await deleteObject(p, key);
++ if (providerConfigured(p) && !await deleteObject(p, key)) retryDeleteLater(p, key);
}
}
++// A remote delete that failed is tried again a few times in the background (the owner, 2026-11-03: "retry … up to a
++// certain amount"): after 30 s, 2, 10 and 30 minutes. If all fail, the daily recording clean-up
++// (recording-cleanup.js) still removes the copy its deleted VOD left. Timers are unref'd and nothing is persisted:
++// a restart drops the retries, and the clean-up is the backstop.
++let DELETE_RETRY_MS = [30_000, 120_000, 600_000, 1_800_000];
++function retryDeleteLater(provider, key) {
++ (async () => {
++ for (const ms of DELETE_RETRY_MS) {
++ await new Promise((r) => { const t = setTimeout(r, ms); if (t.unref) t.unref(); });
++ if (await deleteObject(provider, key)) { console.log(`[VodStorage] ${provider}:${key} deleted on a retry`); return; }
++ }
++ console.warn(`[VodStorage] ${provider}:${key} still not deleted after ${DELETE_RETRY_MS.length + 1} tries; the daily recording clean-up will remove it`);
++ })().catch(() => {});
++}
++function _setDeleteRetryDelays(ms) { DELETE_RETRY_MS = ms; }
++
// ── Disk usage ───────────────────────────────────────────────
/**
@@@ -1560,6 +1576,7 @@ module.exports = {
r2Policy,
recordTierDecision,
deleteVodObjects,
++ _setDeleteRetryDelays,
deleteObject,
presignGet,
runSweep,
ddiff --git a/test/recording-cleanup.test.js b/test/recording-cleanup.test.js
nnew file mode 100644
iindex 0000000..7dc1097
---- /dev/null
++++ b/test/recording-cleanup.test.js
@@@ -0,0 +1,147 @@
++'use strict';
++// The recording clean-up (server/vod/recording-cleanup.js; job storage.recordings.cleanup) and the retried remote
++// delete (vod-storage.js deleteVodObjects). Against a fake S3 spoken to by the real AWS SDK:
++// - a file of a deleted VOD goes (B2, R2 and local), unless it is under the grace period or its VOD is held;
++// - a duplicate of a living VOD goes only when the VOD's own copy is there with the same size;
++// - a name with no VOD id, a different-size copy and every other prefix (backups, vods-orphans/) stay;
++// - a dry run deletes nothing; the job is service-wide only and queued daily;
++// - a failed remote delete is tried again until it succeeds.
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const http = require('http');
++
++(async () => {
++ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-media-cleanup-'));
++ const dir = (n) => { const d = path.join(tmp, n); fs.mkdirSync(d, { recursive: true }); return d; };
++ const OLD = '2026-09-21T00:00:00.000Z';
++ const buckets = { 'b2-bucket': new Map(), 'r2-bucket': new Map() };
++ const put = (bucket, key, size, modified = OLD) => buckets[bucket].set(key, { size, modified });
++ const deletes = [];
++ let failDeletes = 0;
++ const xml = (v) => String(v).replace(/&/g, '&amp;').replace(/</g, '&lt;');
++ const s3 = http.createServer((req, res) => {
++ req.resume();
++ req.on('end', () => {
++ const u = new URL(req.url, 'http://x');
++ const [, bucket, ...rest] = decodeURIComponent(u.pathname).split('/');
++ const key = rest.join('/');
++ const store = buckets[bucket];
++ if (!store) { res.writeHead(404); return res.end(); }
++ if (req.method === 'DELETE') {
++ if (failDeletes > 0) { failDeletes--; res.writeHead(500, { 'Content-Type': 'application/xml' }); return res.end('<Error><Code>InternalError</Code></Error>'); }
++ deletes.push(`${bucket}:${key}`); store.delete(key); res.writeHead(204); return res.end();
++ }
++ if (!key && u.searchParams.get('list-type') === '2') {
++ const prefix = u.searchParams.get('prefix') || '';
++ const keys = [...store.keys()].filter((k) => k.startsWith(prefix)).sort();
++ res.writeHead(200, { 'Content-Type': 'application/xml' });
++ return res.end(`<?xml version="1.0" encoding="UTF-8"?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>${bucket}</Name><Prefix>${xml(prefix)}</Prefix><KeyCount>${keys.length}</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>${
++ keys.map((k) => `<Contents><Key>${xml(k)}</Key><LastModified>${store.get(k).modified}</LastModified><ETag>"e"</ETag><Size>${store.get(k).size}</Size><StorageClass>STANDARD</StorageClass></Contents>`).join('')}</ListBucketResult>`);
++ }
++ const obj = store.get(key);
++ if (!obj) { res.writeHead(404); return res.end(); }
++ res.writeHead(200, { 'Content-Length': obj.size, ETag: '"e"' });
++ res.end(req.method === 'GET' ? Buffer.alloc(obj.size) : undefined);
++ });
++ });
++ await new Promise((r) => s3.listen(0, '127.0.0.1', r));
++ const endpoint = `http://127.0.0.1:${s3.address().port}`;
++ const env = {
++ MEDIA_DATA_DIR: path.join(tmp, 'data'), VOD_PATH: dir('vods'), CLIPS_PATH: dir('clips'), FILES_PATH: dir('files'),
++ THUMBNAILS_PATH: dir('thumbnails'), PASTES_PATH: dir('pastes'), OBJECTS_PATH: dir('objects'), ASSETS_PATH: dir('assets'),
++ MEDIA_B2_ENDPOINT: endpoint, MEDIA_B2_BUCKET: 'b2-bucket', MEDIA_B2_KEY_ID: 'k', MEDIA_B2_APP_KEY: 's', MEDIA_B2_REGION: 'us-west-004',
++ MEDIA_R2_ENDPOINT: endpoint, MEDIA_R2_BUCKET: 'r2-bucket', MEDIA_R2_ACCESS_KEY_ID: 'k', MEDIA_R2_SECRET_ACCESS_KEY: 's',
++ AWS_REQUEST_CHECKSUM_CALCULATION: 'WHEN_REQUIRED', AWS_RESPONSE_CHECKSUM_VALIDATION: 'WHEN_REQUIRED',
++ MEDIA_PUBLIC_URL: 'https://media.test',
++ };
++ Object.assign(process.env, env);
++ delete process.env.MEDIA_RECORDING_CLEANUP_DAYS;
++ const db = require('../server/db/database');
++ const model = require('../server/objects/model');
++ const queue = require('../server/jobs/queue');
++ require('../server/jobs/types');
++ const cleanup = require('../server/vod/recording-cleanup');
++ const job = require('../server/jobs/recording-cleanup');
++ const storage = require('../server/vod/vod-storage');
++ await db.upsertApp({ app_id: 'live', api_key: 'live-key-cleanup' });
++ const raw = db.getDb();
++ const ins = raw.prepare(`INSERT INTO vods (id, app_id, title, file_path, file_size, is_public, visibility, duration_seconds, storage_provider, storage_key) OVERRIDING SYSTEM VALUE
++ VALUES (?, 'live', ?, ?, ?, 1, 'public', 60, ?, ?) RETURNING id`);
++ const local = (name, size, ageDays) => { const p = path.join(env.VOD_PATH, name); fs.writeFileSync(p, Buffer.alloc(size)); if (ageDays) { const t = (Date.now() - ageDays * 86400000) / 1000; fs.utimesSync(p, t, t); } return p; };
++
++ // ── Fixtures ──
++ // VOD 10 lives in B2 at its own key; two older copies of it sit under live.vods/ (one the same size, one not).
++ await ins.run(10, 'kept', '/old/vod-live-10-1758000000010.webm', 40, 'b2', 'vods/vod-live-10-1758000000010.webm');
++ put('b2-bucket', 'vods/vod-live-10-1758000000010.webm', 40);
++ put('b2-bucket', 'live.vods/objects/2026/05/01/media-live-vod-10.webm', 40); // duplicate: goes
++ put('b2-bucket', 'live.vods/objects/2026/05/02/media-live-vod-10.webm', 41); // another size: stays
++ // VOD 11 lives locally only; an old B2 copy of another size stays.
++ await ins.run(11, 'local', local('vod-live-11-1758000000011.webm', 30), 30, 'local', null);
++ put('b2-bucket', 'vods/vod-live-11-1758000000099.webm', 31);
++ // Deleted VODs 612 and 613 left copies; 613 is held. 614's copy is new (inside the grace period).
++ put('b2-bucket', 'vods/vod-612-1758000000612.webm', 50);
++ put('r2-bucket', 'vods/vod-612-1758000000612.webm', 50);
++ put('b2-bucket', 'live.vods/objects/2026/05/03/media-hobostreamer-vod-612.webm', 70);
++ put('b2-bucket', 'vods/vod-613-1758000000613.webm', 60);
++ put('b2-bucket', 'vods/vod-614-1758000000614.webm', 60, new Date().toISOString());
++ const heldObj = await model.createObject({ app_id: 'live', kind: 'vod', lifecycle_status: 'deleted', visibility: 'private', size_bytes: 60 });
++ await raw.prepare("UPDATE media_objects SET legacy_ref = 'legacy:live:vod:613' WHERE id = ?").run(heldObj);
++ await model.placeHold({ object_id: heldObj, kind: 'evidence', reason: 'test', created_by: 'test' });
++ // Out of scope: a name with no VOD id, the parked prefix, the backups.
++ put('b2-bucket', 'vods/random-upload.webm', 10);
++ put('b2-bucket', 'vods-orphans/vod-700-1758000000700.webm', 10);
++ put('b2-bucket', 'openvibe-pgbackrest/archive/0001.zst', 10);
++ // Local: a deleted VOD's old file goes, a fresh one stays, a living VOD's file stays.
++ const goneLocal = local('vod-live-612-1758000000612.webm', 20, 10);
++ const freshLocal = local('vod-live-615-1758000000615.webm', 20, 0);
++
++ // ── A dry run lists and deletes nothing ──
++ const dry = await cleanup.run({ dryRun: true });
++ assert.deepStrictEqual(dry.deleted, { row_gone: 4, duplicate: 1 }, JSON.stringify(dry.items));
++ assert.strictEqual(deletes.length, 0, 'a dry run deletes nothing');
++ assert.ok(fs.existsSync(goneLocal));
++
++ // ── The real run ──
++ const r = await cleanup.run();
++ assert.deepStrictEqual(r.deleted, { row_gone: 4, duplicate: 1 });
++ assert.strictEqual(r.bytes, 50 + 50 + 70 + 20 + 40);
++ assert.deepStrictEqual(deletes.sort(), [
++ 'b2-bucket:live.vods/objects/2026/05/01/media-live-vod-10.webm',
++ 'b2-bucket:live.vods/objects/2026/05/03/media-hobostreamer-vod-612.webm',
++ 'b2-bucket:vods/vod-612-1758000000612.webm',
++ 'r2-bucket:vods/vod-612-1758000000612.webm',
++ ].sort());
++ assert.ok(!fs.existsSync(goneLocal) && fs.existsSync(freshLocal), 'the old local file of a deleted VOD goes; a fresh one stays');
++ for (const k of ['vods/vod-live-10-1758000000010.webm', 'live.vods/objects/2026/05/02/media-live-vod-10.webm', 'vods/vod-live-11-1758000000099.webm',
++ 'vods/vod-613-1758000000613.webm', 'vods/vod-614-1758000000614.webm', 'vods/random-upload.webm', 'vods-orphans/vod-700-1758000000700.webm', 'openvibe-pgbackrest/archive/0001.zst']) {
++ assert.ok(buckets['b2-bucket'].has(k), `${k} stays`);
++ }
++ assert.ok(r.kept.held >= 1 && r.kept.young >= 2 && r.kept.size_differs >= 2 && r.kept.no_vod_id >= 1, JSON.stringify(r.kept));
++ assert.match(cleanup.summarize(r), /^deleted 4 file\(s\) of deleted VODs and 1 duplicate\(s\)/);
++ assert.deepStrictEqual([cleanup.vodIdOf('vod-live-2-1758000000002.webm'), cleanup.vodIdOf('vod-1621-1786231355470.webm'), cleanup.vodIdOf('media-hobostreamer-vod-612.webm'), cleanup.vodIdOf('vod-1453-1785217692199.master.mkv'), cleanup.vodIdOf('random.webm')],
++ [2, 1621, 612, 1453, null]);
++
++ // ── The job: service-wide only, queued once a day ──
++ assert.throws(() => job.spec.validate({ appId: 'live', params: {} }), /whole service/);
++ assert.deepStrictEqual(job.spec.validate({ appId: queue.SYSTEM_APP, params: { dry_run: true } }), { dry_run: true });
++ const day = Date.UTC(2026, 10, 3, 12);
++ assert.strictEqual(await job.schedule(day), 1);
++ assert.strictEqual(await job.schedule(day + 3600 * 1000), 0, 'once a day');
++ process.env.MEDIA_RECORDING_CLEANUP_DAYS = '0';
++ assert.strictEqual(await job.schedule(day + 3 * 86400 * 1000), 0, '0 turns it off');
++
++ // ── A failed remote delete is tried again ──
++ storage._setDeleteRetryDelays([20, 20, 20]);
++ put('b2-bucket', 'vods/vod-live-20-1758000000020.webm', 9);
++ deletes.length = 0;
++ failDeletes = 2;
++ await storage.deleteVodObjects({ id: 20, file_path: '/old/vod-live-20-1758000000020.webm', storage_key: 'vods/vod-live-20-1758000000020.webm' });
++ for (let i = 0; i < 50 && buckets['b2-bucket'].has('vods/vod-live-20-1758000000020.webm'); i++) await new Promise((res) => setTimeout(res, 20));
++ assert.ok(!buckets['b2-bucket'].has('vods/vod-live-20-1758000000020.webm'), 'the B2 copy went on a retry');
++
++ console.log('recording clean-up: all checks passed');
++ s3.close();
++ process.exit(0);
++})().catch((e) => { console.error(e); process.exit(1); });