openvibers/openvibe.media Commits
H15 repair (WS-G task 7): scripts/h15-repair.js tries every copy of each object with no good copy before anything is condemned: a B2/R2 copy marked corrupt only because its size differs from the size recorded at creation (no content hash) is downloaded and decoded to the end, and a good one becomes the object (size and sha256 re-recorded); a fragment under 2 s is a recording that failed when it was made (lifecycle failed, not playable); a missing AI-moment frame is extracted again from its VOD; anything else is reported as lost. Dry run by default; --apply takes an online backup and writes a rollback file; --rollback restores
Changes
nnew file mode 100644@@@ -0,0 +1,170 @@++#!/usr/bin/env node++'use strict';++/**++ * H15 repair (roadmap WS-G task 7), for the operator: every ready object with no good copy++ * (scripts/no-good-copy-report.js) gets every copy tried before anything is condemned++ * (server/objects/h15-repair.js has the rules):++ *++ * node scripts/h15-repair.js [--out <report.json>]++ * Dry run (the default): downloads each corrupt B2/R2 copy (to a temporary file; B2/R2 egress),++ * probes and decodes it with ffprobe/ffmpeg, extracts each missing AI-moment frame again from++ * its VOD's copy, and writes the plan: rebaseline, failed_recording, regenerate or lost, with the++ * reason. Changes nothing. The report is for the owner (default <data>/reports/h15-repair-….json).++ * node scripts/h15-repair.js --apply --backup <file.json>++ * First an online backup of the database to <file>.media.db (checked with integrity_check), then++ * the plan: re-records size and sha256 from a good remote copy, marks a failed recording++ * lifecycle_status 'failed' (not playable, hidden from public players), writes a regenerated++ * frame back to its location. <file.json> lists every change with its old values (the rollback++ * file). Nothing is deleted, locally or remotely. Refuses to overwrite either file.++ * node scripts/h15-repair.js --rollback <file.json> [--apply]++ * Restores the old values of the changes the file lists, where the row still holds what the++ * repair wrote (a regenerated file stays: it is the same frame). Without --apply it only reports.++ * (every command takes --db <media.db>; the database is DB_PATH otherwise)++ *++ * B2/R2 credentials come from the environment (the service's /etc/openvibe/media.env names). Run it as++ * the service user so written files belong to it. Exit codes: 0 done, 1 error, 2 bad usage.++ */++const fs = require('fs');++const os = require('os');++const path = require('path');++const crypto = require('crypto');++const { execFile } = require('child_process');++++const args = process.argv.slice(2);++const has = (name) => args.includes(`--${name}`);++const arg = (name) => { const i = args.indexOf(`--${name}`); return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : null; };++const usage = (msg) => { console.error(`h15-repair: ${msg}`); process.exit(2); };++for (const flag of ['db', 'backup', 'rollback', 'out']) if (has(flag) && !arg(flag)) usage(`--${flag} needs a value`);++if (arg('db')) process.env.DB_PATH = path.resolve(arg('db'));++const APPLY = has('apply');++const BACKUP = arg('backup') ? path.resolve(arg('backup')) : null;++const ROLLBACK = arg('rollback') ? path.resolve(arg('rollback')) : null;++if (ROLLBACK && BACKUP) usage('--rollback and --backup do not go together');++if (APPLY && !ROLLBACK && !BACKUP) usage('--apply needs --backup <file.json> (the rollback file; the database copy goes next to it)');++++const config = require('../server/config');++const db = require('../server/db/database');++const copyReport = require('../server/objects/copy-report');++const h15 = require('../server/objects/h15-repair');++const vs = require('../server/vod/vod-storage');++++const run = (cmd, argv, timeoutMs) => new Promise((resolve) => {++ execFile(cmd, argv, { timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => resolve({ err, stdout: String(stdout || ''), stderr: String(stderr || '') }));++});++++/** Download a presigned URL to a file, hashing as it goes. → { size, sha256 } */++async function download(url, file) {++ const res = await fetch(url, { signal: AbortSignal.timeout(30 * 60 * 1000) });++ if (!res.ok || !res.body) throw new Error(`download answered ${res.status}`);++ const hash = crypto.createHash('sha256');++ const out = fs.createWriteStream(file);++ let size = 0;++ for await (const chunk of res.body) { hash.update(chunk); size += chunk.length; if (!out.write(chunk)) await new Promise((r) => out.once('drain', r)); }++ await new Promise((r, j) => out.end((e) => (e ? j(e) : r())));++ return { size, sha256: hash.digest('hex') };++}++++/** ffprobe + a full decode. → { duration, streams, errorLines } | { error } */++async function probe(file) {++ const p = await run('ffprobe', ['-v', 'error', '-show_entries', 'format=duration:stream=codec_name', '-of', 'json', file], 120000);++ if (p.err) return { error: (p.stderr || p.err.message).split('\n')[0] };++ let j = {}; try { j = JSON.parse(p.stdout); } catch { return { error: 'ffprobe output unreadable' }; }++ const d = await run('ffmpeg', ['-nostdin', '-v', 'error', '-i', file, '-f', 'null', '-'], 60 * 60 * 1000);++ return { duration: Number(j.format && j.format.duration) || 0, streams: (j.streams || []).map((s) => s.codec_name).filter(Boolean), errorLines: d.stderr.split('\n').filter(Boolean) };++}++++const sha256File = (file) => new Promise((resolve, reject) => {++ const h = crypto.createHash('sha256');++ fs.createReadStream(file).on('data', (c) => h.update(c)).on('end', () => resolve(h.digest('hex'))).on('error', reject);++});++++/** The frame at `offset` seconds of VOD `vodId`, extracted from its good copy into `file`. → { size, sha256 } */++async function extractFrame(vodId, offset, file) {++ const v = db.getDb().prepare('SELECT id, storage_provider, storage_key, file_path FROM vods WHERE id = ?').get(vodId);++ if (!v) throw new Error(`VOD ${vodId} does not exist`);++ let input = v.file_path && fs.existsSync(v.file_path) ? v.file_path : null;++ if (!input && v.storage_provider && v.storage_key) { const u = await vs.presignGet(v.storage_provider, v.storage_key, 3600); input = typeof u === 'string' ? u : u.url; }++ if (!input) throw new Error(`VOD ${vodId} has no readable copy`);++ const r = await run('ffmpeg', ['-nostdin', '-v', 'error', '-ss', String(offset), '-i', input, '-frames:v', '1', '-q:v', '3', '-y', file], 20 * 60 * 1000);++ if (r.err || !fs.existsSync(file) || !fs.statSync(file).size) throw new Error(`no frame at ${offset} s: ${(r.stderr || (r.err && r.err.message) || '').split('\n')[0]}`);++ return { size: fs.statSync(file).size, sha256: await sha256File(file) };++}++++async function backupDatabase(dest) {++ await db.getDb().backup(dest);++ const Database = require('better-sqlite3');++ const copy = new Database(dest, { readonly: true });++ try { return { path: dest, bytes: fs.statSync(dest).size, integrity_check: copy.pragma('integrity_check', { simple: true }) }; } finally { copy.close(); }++}++++async function main() {++ const h = db.getDb();++ if (ROLLBACK) {++ const file = JSON.parse(fs.readFileSync(ROLLBACK, 'utf8'));++ const out = (file.changes || []).map((c) => (APPLY ? h15.rollback(h, c) : { ...c, would_restore: true }));++ for (const o of out) console.log(`${o.object_id} ${o.action}: ${o.restored ? 'restored' : o.skipped || (o.would_restore ? 'would restore' : '')}`);++ return 0;++ }++ const q = { all: (sql, p = []) => h.prepare(sql).all(...p), get: (sql, p = []) => h.prepare(sql).get(...p) };++ const report = copyReport.buildReport(q, {});++ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'h15-repair-'));++ const steps = [];++ try {++ for (const item of report.objects) {++ const probes = {};++ const facts = {};++ for (const l of item.locations.filter((x) => (x.provider === 'b2' || x.provider === 'r2') && x.state === 'corrupt')) {++ try {++ const u = await vs.presignGet(l.provider, l.key, 3600);++ const file = path.join(tmp, `${item.object_id}.${l.provider}`);++ facts[l.provider] = await download(typeof u === 'string' ? u : u.url, file);++ probes[l.provider] = await probe(file);++ } catch (err) { probes[l.provider] = { error: err.message }; }++ }++ const step = h15.plan(item, probes);++ if (step.action === 'rebaseline') Object.assign(step, { facts: facts[step.provider], probe: { duration: probes[step.provider].duration, streams: probes[step.provider].streams } });++ if (step.action === 'failed_recording') step.probe = probes[step.provider];++ if (step.action === 'regenerate') {++ const loc = item.locations.find((l) => l.provider === 'local');++ const file = path.join(tmp, `${item.object_id}.jpg`);++ try { step.facts = await extractFrame(step.source_vod, step.offset, file); step.file = file; step.target = loc && loc.key; }++ catch (err) { Object.assign(step, { action: 'lost', reason: `the frame could not be extracted again: ${err.message}` }); }++ }++ if (step.probe && step.probe.errorLines) step.probe = { ...step.probe, errorLines: step.probe.errorLines.length };++ steps.push(step);++ console.log(`${item.object_id} ${item.kind} ${item.legacy_ref || ''}: ${step.action} — ${step.reason}`);++ }++ const record = { generated_at: new Date().toISOString(), applied: false, objects: report.count, plan: steps.map(({ file, ...s }) => s) };++ if (!APPLY) {++ const out = arg('out') ? path.resolve(arg('out')) : path.join(path.dirname(path.resolve(config.db.path)), 'reports', `h15-repair-${record.generated_at.replace(/[:.]/g, '-')}.json`);++ fs.mkdirSync(path.dirname(out), { recursive: true });++ fs.writeFileSync(out, JSON.stringify(record, null, 2));++ console.log(`\nDry run: nothing changed. Plan written to ${out}`);++ return 0;++ }++ const copyPath = BACKUP.replace(/\.json$/, '') + '.media.db';++ for (const f of [BACKUP, copyPath]) if (fs.existsSync(f)) usage(`${f} already exists; pick a new --backup name`);++ const dbCopy = await backupDatabase(copyPath);++ if (dbCopy.integrity_check !== 'ok') { console.error(`backup failed integrity_check: ${dbCopy.integrity_check}; nothing changed`); return 1; }++ const changes = [];++ for (const step of steps) {++ if (step.action === 'lost') continue;++ if (step.action === 'regenerate') {++ if (!step.target) { changes.push({ ...step, skipped: 'no local location to write to' }); continue; }++ fs.mkdirSync(path.dirname(step.target), { recursive: true });++ if (!fs.existsSync(step.target)) fs.copyFileSync(step.file, step.target);++ }++ const { file, ...s } = step;++ changes.push(h15.apply(h, s, step.facts));++ }++ Object.assign(record, { applied: true, backup: dbCopy, changes });++ fs.writeFileSync(BACKUP, JSON.stringify(record, null, 2));++ for (const c of changes) console.log(`${c.object_id} ${c.action}: ${c.skipped ? `skipped (${c.skipped})` : 'applied'}`);++ console.log(`\nApplied. Rollback file: ${BACKUP}; database copy: ${copyPath}`);++ return 0;++ } finally {++ fs.rmSync(tmp, { recursive: true, force: true });++ }++}++++main().then((code) => process.exit(code), (err) => { console.error(`h15-repair: ${err.message}`); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,109 @@++'use strict';++/**++ * H15 repair (roadmap WS-G task 7): what to do with each ready object that has no good copy++ * (scripts/no-good-copy-report.js lists them). scripts/h15-repair.js runs it; this file holds the rules++ * and the writes, so they are tested without storage.++ *++ * Every copy is tried before anything is condemned:++ *++ * rebaseline a B2/R2 copy is "corrupt" only because its size differs from the size recorded++ * when the object was made (a recording's size captured mid-upload, no content++ * hash), and that copy decodes to the end with a real duration. The copy is the++ * object: size and sha256 are re-recorded from it and the location is present.++ * failed_recording the only bytes are a fragment (under MIN_SECONDS, or nothing decodes): the++ * recording failed when it was made, nothing was lost since. lifecycle_status++ * becomes 'failed', so readiness says not playable and public players hide it.++ * regenerate an AI-moment screenshot (ai-moment-vod<vod>-<seconds>.jpg) whose file is gone:++ * the same frame is extracted again from that VOD's good copy and written back.++ * lost nothing above applies: reported for the owner, nothing written.++ *++ * Writes happen only in apply(), each object in its own transaction and only while the row still holds++ * what was read (so a concurrent change wins); every change is returned with its old values for the++ * rollback file.++ */++const MIN_SECONDS = 2;++const BENIGN = /non monotonically increasing dts|Last message repeated/i;++const SHOT_RE = /ai-moment-vod(\d+)-(\d+)\.jpg$/;++++/** The decode verdict of one copy: { duration, streams, errorLines } → { ok, reason } */++function judgeProbe(p) {++ if (!p || p.error) return { ok: false, reason: `unreadable: ${(p && p.error) || 'no probe'}` };++ if (!p.streams || !p.streams.length) return { ok: false, reason: 'no audio or video stream' };++ const serious = (p.errorLines || []).filter((l) => l.trim() && !BENIGN.test(l));++ if (serious.length > 5) return { ok: false, reason: `${serious.length} decode errors (first: ${serious[0].slice(0, 120)})` };++ if (!(p.duration >= MIN_SECONDS)) return { ok: false, fragment: true, reason: `only ${Number(p.duration || 0).toFixed(1)} s decode` };++ return { ok: true, reason: `decodes to the end: ${p.duration.toFixed(1)} s, ${p.streams.join('+')}${serious.length ? `, ${serious.length} minor decode error(s)` : ''}` };++}++++/**++ * The plan for one report item. probes: provider → probe of that copy (only for corrupt remote copies).++ * → { object_id, action, provider?, reason, source_vod?, offset? }++ */++function plan(item, probes = {}) {++ // What the plan was based on: apply() writes only while the row still holds it.++ const base = { object_id: item.object_id, kind: item.kind, legacy_ref: item.legacy_ref || null, read: { size_bytes: item.size_bytes, content_hash: item.content_hash || null, lifecycle_status: 'ready' } };++ const shot = item.kind === 'screenshot' && (item.locations || []).map((l) => SHOT_RE.exec(String(l.key || ''))).find(Boolean);++ if (shot && (item.locations || []).every((l) => l.state === 'missing')) {++ return { ...base, action: 'regenerate', source_vod: Number(shot[1]), offset: Number(shot[2]), reason: `the frame at ${shot[2]} s of VOD ${shot[1]} can be extracted again` };++ }++ const remote = (item.locations || []).filter((l) => (l.provider === 'b2' || l.provider === 'r2') && l.state === 'corrupt');++ if (remote.length && !item.content_hash) {++ let fragment = null;++ for (const l of remote) {++ const v = judgeProbe(probes[l.provider]);++ if (v.ok) return { ...base, action: 'rebaseline', provider: l.provider, key: l.key, reason: `${l.provider} copy ${v.reason}; only its size differed from the size recorded at creation (${item.size_bytes} bytes)` };++ if (v.fragment) fragment = { provider: l.provider, reason: v.reason };++ }++ if (fragment) return { ...base, action: 'failed_recording', provider: fragment.provider, reason: `the only copy is a fragment (${fragment.reason}): the recording failed when it was made` };++ }++ return { ...base, action: 'lost', reason: 'no copy decodes and nothing can regenerate it' };++}++++/**++ * Apply one planned change. h = better-sqlite3 handle. facts: { size, sha256 } of the good bytes (rebaseline,++ * regenerate). → the change with old values, or { skipped } when the row moved on.++ */++function apply(h, step, facts = {}) {++ const cur = h.prepare('SELECT id, size_bytes, content_hash, lifecycle_status FROM media_objects WHERE id = ?').get(step.object_id);++ if (!cur) return { ...step, skipped: 'object gone' };++ const obj = { ...cur, ...(step.read || {}) };++ const tx = h.transaction(() => {++ if (step.action === 'rebaseline' || step.action === 'regenerate') {++ const provider = step.action === 'regenerate' ? 'local' : step.provider;++ const loc = h.prepare('SELECT id, state, size_bytes FROM media_locations WHERE object_id = ? AND provider = ?').get(obj.id, provider);++ if (!loc) return { ...step, skipped: `no ${provider} location` };++ const r = h.prepare("UPDATE media_objects SET size_bytes = ?, content_hash = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND size_bytes IS ? AND content_hash IS ?")++ .run(facts.size, facts.sha256, obj.id, obj.size_bytes, obj.content_hash);++ if (!r.changes) return { ...step, skipped: 'the object changed since it was read' };++ h.prepare("UPDATE media_locations SET state = 'present', size_bytes = ?, verified_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(facts.size, loc.id);++ return { ...step, old: { size_bytes: obj.size_bytes, content_hash: obj.content_hash, location_state: loc.state, location_size: loc.size_bytes }, new: { size_bytes: facts.size, content_hash: facts.sha256, location_state: 'present' } };++ }++ if (step.action === 'failed_recording') {++ const r = h.prepare("UPDATE media_objects SET lifecycle_status = 'failed', updated_at = CURRENT_TIMESTAMP WHERE id = ? AND lifecycle_status = ?").run(obj.id, obj.lifecycle_status);++ if (!r.changes) return { ...step, skipped: 'the object changed since it was read' };++ return { ...step, old: { lifecycle_status: obj.lifecycle_status }, new: { lifecycle_status: 'failed' } };++ }++ return { ...step, skipped: 'nothing to write' };++ });++ return tx();++}++++/** Undo one applied change where the row still holds what the repair wrote. → { restored } | { skipped } */++function rollback(h, change) {++ if (!change || !change.old || !change.new) return { ...change, skipped: 'not an applied change' };++ const tx = h.transaction(() => {++ if (change.action === 'failed_recording') {++ const r = h.prepare('UPDATE media_objects SET lifecycle_status = ? WHERE id = ? AND lifecycle_status = ?').run(change.old.lifecycle_status, change.object_id, change.new.lifecycle_status);++ return r.changes ? { ...change, restored: true } : { ...change, skipped: 'the row changed since' };++ }++ const r = h.prepare('UPDATE media_objects SET size_bytes = ?, content_hash = ? WHERE id = ? AND size_bytes IS ? AND content_hash IS ?')++ .run(change.old.size_bytes, change.old.content_hash, change.object_id, change.new.size_bytes, change.new.content_hash);++ if (!r.changes) return { ...change, skipped: 'the row changed since' };++ const provider = change.action === 'regenerate' ? 'local' : change.provider;++ h.prepare('UPDATE media_locations SET state = ?, size_bytes = ? WHERE object_id = ? AND provider = ?').run(change.old.location_state, change.old.location_size, change.object_id, provider);++ return { ...change, restored: true };++ });++ return tx();++}++++module.exports = { plan, apply, rollback, judgeProbe, MIN_SECONDS, SHOT_RE };nnew file mode 100644@@@ -0,0 +1,53 @@++'use strict';++// H15 repair (WS-G task 7): the plan for each object with no good copy, and its writes and rollback.++const assert = require('assert');++const Database = require('better-sqlite3');++const { plan, apply, rollback, judgeProbe } = require('../server/objects/h15-repair');++++const good = { duration: 1721.9, streams: ['vp8', 'opus'], errorLines: Array(4700).fill('[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 5 >= 5').concat(['[opus @ 0x2] Error parsing Opus packet header.']) };++const vod = (over = {}) => ({ object_id: 'med_1', kind: 'vod', size_bytes: 13631488, content_hash: null, legacy_ref: 'legacy:live:vod:1277', locations: [{ provider: 'b2', key: 'vods/a.webm', state: 'corrupt', size_bytes: 14609471 }], ...over });++++// ── The verdict on a decoded copy ──++assert.strictEqual(judgeProbe(good).ok, true, 'timestamp warnings of a live recording do not condemn it');++assert.strictEqual(judgeProbe({ duration: 0.6, streams: ['vp8'], errorLines: [] }).fragment, true);++assert.strictEqual(judgeProbe({ error: 'Invalid data found' }).ok, false);++assert.strictEqual(judgeProbe({ duration: 400, streams: ['vp8'], errorLines: Array(9).fill('[vp8 @ 1] Truncated packet') }).ok, false, 'real decode errors do');++assert.strictEqual(judgeProbe({ duration: 400, streams: [], errorLines: [] }).ok, false);++++// ── The plan ──++assert.strictEqual(plan(vod(), { b2: good }).action, 'rebaseline');++assert.match(plan(vod(), { b2: good }).reason, /only its size differed/);++assert.strictEqual(plan(vod(), { b2: { duration: 0.6, streams: ['vp8'], errorLines: [] } }).action, 'failed_recording');++assert.strictEqual(plan(vod(), { b2: { error: 'moov atom not found' } }).action, 'lost');++assert.strictEqual(plan(vod({ content_hash: 'abc' }), { b2: good }).action, 'lost', 'a recorded hash that disagrees is real corruption: never rebaselined');++const shot = { object_id: 'med_2', kind: 'screenshot', size_bytes: 0, content_hash: null, locations: [{ provider: 'local', key: '/data/pastes/screenshots/ai-moment-vod1691-32230.jpg', state: 'missing' }] };++assert.deepStrictEqual((({ action, source_vod, offset }) => ({ action, source_vod, offset }))(plan(shot)), { action: 'regenerate', source_vod: 1691, offset: 32230 });++assert.strictEqual(plan({ ...shot, locations: [{ provider: 'local', key: '/x/other.jpg', state: 'missing' }] }).action, 'lost');++++// ── Writes, only while the row holds what was read, and their rollback ──++const h = new Database(':memory:');++h.exec(`CREATE TABLE media_objects (id TEXT PRIMARY KEY, size_bytes INTEGER, content_hash TEXT, lifecycle_status TEXT, updated_at TEXT);++ CREATE TABLE media_locations (id INTEGER PRIMARY KEY, object_id TEXT, provider TEXT, key TEXT, state TEXT, size_bytes INTEGER, verified_at TEXT, updated_at TEXT);`);++h.prepare("INSERT INTO media_objects VALUES ('med_1', 13631488, NULL, 'ready', NULL), ('med_3', 262144, NULL, 'ready', NULL), ('med_2', 0, NULL, 'ready', NULL)").run();++h.prepare("INSERT INTO media_locations (object_id, provider, key, state, size_bytes) VALUES ('med_1', 'b2', 'vods/a.webm', 'corrupt', 14609471), ('med_3', 'b2', 'vods/b.webm', 'corrupt', 218283), ('med_2', 'local', '/x.jpg', 'missing', NULL)").run();++++const c1 = apply(h, plan(vod(), { b2: good }), { size: 14609471, sha256: 'f'.repeat(64) });++assert.deepStrictEqual(c1.old, { size_bytes: 13631488, content_hash: null, location_state: 'corrupt', location_size: 14609471 });++assert.deepStrictEqual(h.prepare("SELECT size_bytes, content_hash FROM media_objects WHERE id = 'med_1'").get(), { size_bytes: 14609471, content_hash: 'f'.repeat(64) });++assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_1'").get().state, 'present');++assert.match(apply(h, plan(vod(), { b2: good }), { size: 1, sha256: 'e'.repeat(64) }).skipped, /changed since/, 'a second run finds the row moved on');++++const c3 = apply(h, { object_id: 'med_3', action: 'failed_recording', provider: 'b2' });++assert.strictEqual(h.prepare("SELECT lifecycle_status FROM media_objects WHERE id = 'med_3'").get().lifecycle_status, 'failed');++++const c2 = apply(h, plan(shot), { size: 5120, sha256: 'a'.repeat(64) });++assert.strictEqual(c2.new.location_state, 'present');++assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_2'").get().state, 'present');++++for (const c of [c1, c2, c3]) assert.strictEqual(rollback(h, c).restored, true);++assert.deepStrictEqual(h.prepare("SELECT size_bytes, content_hash, lifecycle_status FROM media_objects WHERE id = 'med_1'").get(), { size_bytes: 13631488, content_hash: null, lifecycle_status: 'ready' });++assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_1'").get().state, 'corrupt');++assert.strictEqual(h.prepare("SELECT lifecycle_status FROM media_objects WHERE id = 'med_3'").get().lifecycle_status, 'ready');++assert.strictEqual(rollback(h, c1).skipped, 'the row changed since', 'rolling back twice changes nothing');++++console.log('h15 repair: all checks passed');
new file mode 100644
new file mode 100644
@@ -0,0 +1,170 @@
@@ -0,0 +1,170 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * H15 repair (roadmap WS-G task 7), for the operator: every ready object with no good copy
+ * (scripts/no-good-copy-report.js) gets every copy tried before anything is condemned
+ * (server/objects/h15-repair.js has the rules):
+ *
+ * node scripts/h15-repair.js [--out <report.json>]
+ * Dry run (the default): downloads each corrupt B2/R2 copy (to a temporary file; B2/R2 egress),
+ * probes and decodes it with ffprobe/ffmpeg, extracts each missing AI-moment frame again from
+ * its VOD's copy, and writes the plan: rebaseline, failed_recording, regenerate or lost, with the
+ * reason. Changes nothing. The report is for the owner (default <data>/reports/h15-repair-….json).
+ * node scripts/h15-repair.js --apply --backup <file.json>
+ * First an online backup of the database to <file>.media.db (checked with integrity_check), then
+ * the plan: re-records size and sha256 from a good remote copy, marks a failed recording
+ * lifecycle_status 'failed' (not playable, hidden from public players), writes a regenerated
+ * frame back to its location. <file.json> lists every change with its old values (the rollback
+ * file). Nothing is deleted, locally or remotely. Refuses to overwrite either file.
+ * node scripts/h15-repair.js --rollback <file.json> [--apply]
+ * Restores the old values of the changes the file lists, where the row still holds what the
+ * repair wrote (a regenerated file stays: it is the same frame). Without --apply it only reports.
+ * (every command takes --db <media.db>; the database is DB_PATH otherwise)
+ *
+ * B2/R2 credentials come from the environment (the service's /etc/openvibe/media.env names). Run it as
+ * the service user so written files belong to it. Exit codes: 0 done, 1 error, 2 bad usage.
+ */
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const crypto = require('crypto');
+const { execFile } = require('child_process');
+
+const args = process.argv.slice(2);
+const has = (name) => args.includes(`--${name}`);
+const arg = (name) => { const i = args.indexOf(`--${name}`); return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : null; };
+const usage = (msg) => { console.error(`h15-repair: ${msg}`); process.exit(2); };
+for (const flag of ['db', 'backup', 'rollback', 'out']) if (has(flag) && !arg(flag)) usage(`--${flag} needs a value`);
+if (arg('db')) process.env.DB_PATH = path.resolve(arg('db'));
+const APPLY = has('apply');
+const BACKUP = arg('backup') ? path.resolve(arg('backup')) : null;
+const ROLLBACK = arg('rollback') ? path.resolve(arg('rollback')) : null;
+if (ROLLBACK && BACKUP) usage('--rollback and --backup do not go together');
+if (APPLY && !ROLLBACK && !BACKUP) usage('--apply needs --backup <file.json> (the rollback file; the database copy goes next to it)');
+
+const config = require('../server/config');
+const db = require('../server/db/database');
+const copyReport = require('../server/objects/copy-report');
+const h15 = require('../server/objects/h15-repair');
+const vs = require('../server/vod/vod-storage');
+
+const run = (cmd, argv, timeoutMs) => new Promise((resolve) => {
+ execFile(cmd, argv, { timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => resolve({ err, stdout: String(stdout || ''), stderr: String(stderr || '') }));
+});
+
+/** Download a presigned URL to a file, hashing as it goes. → { size, sha256 } */
+async function download(url, file) {
+ const res = await fetch(url, { signal: AbortSignal.timeout(30 * 60 * 1000) });
+ if (!res.ok || !res.body) throw new Error(`download answered ${res.status}`);
+ const hash = crypto.createHash('sha256');
+ const out = fs.createWriteStream(file);
+ let size = 0;
+ for await (const chunk of res.body) { hash.update(chunk); size += chunk.length; if (!out.write(chunk)) await new Promise((r) => out.once('drain', r)); }
+ await new Promise((r, j) => out.end((e) => (e ? j(e) : r())));
+ return { size, sha256: hash.digest('hex') };
+}
+
+/** ffprobe + a full decode. → { duration, streams, errorLines } | { error } */
+async function probe(file) {
+ const p = await run('ffprobe', ['-v', 'error', '-show_entries', 'format=duration:stream=codec_name', '-of', 'json', file], 120000);
+ if (p.err) return { error: (p.stderr || p.err.message).split('\n')[0] };
+ let j = {}; try { j = JSON.parse(p.stdout); } catch { return { error: 'ffprobe output unreadable' }; }
+ const d = await run('ffmpeg', ['-nostdin', '-v', 'error', '-i', file, '-f', 'null', '-'], 60 * 60 * 1000);
+ return { duration: Number(j.format && j.format.duration) || 0, streams: (j.streams || []).map((s) => s.codec_name).filter(Boolean), errorLines: d.stderr.split('\n').filter(Boolean) };
+}
+
+const sha256File = (file) => new Promise((resolve, reject) => {
+ const h = crypto.createHash('sha256');
+ fs.createReadStream(file).on('data', (c) => h.update(c)).on('end', () => resolve(h.digest('hex'))).on('error', reject);
+});
+
+/** The frame at `offset` seconds of VOD `vodId`, extracted from its good copy into `file`. → { size, sha256 } */
+async function extractFrame(vodId, offset, file) {
+ const v = db.getDb().prepare('SELECT id, storage_provider, storage_key, file_path FROM vods WHERE id = ?').get(vodId);
+ if (!v) throw new Error(`VOD ${vodId} does not exist`);
+ let input = v.file_path && fs.existsSync(v.file_path) ? v.file_path : null;
+ if (!input && v.storage_provider && v.storage_key) { const u = await vs.presignGet(v.storage_provider, v.storage_key, 3600); input = typeof u === 'string' ? u : u.url; }
+ if (!input) throw new Error(`VOD ${vodId} has no readable copy`);
+ const r = await run('ffmpeg', ['-nostdin', '-v', 'error', '-ss', String(offset), '-i', input, '-frames:v', '1', '-q:v', '3', '-y', file], 20 * 60 * 1000);
+ if (r.err || !fs.existsSync(file) || !fs.statSync(file).size) throw new Error(`no frame at ${offset} s: ${(r.stderr || (r.err && r.err.message) || '').split('\n')[0]}`);
+ return { size: fs.statSync(file).size, sha256: await sha256File(file) };
+}
+
+async function backupDatabase(dest) {
+ await db.getDb().backup(dest);
+ const Database = require('better-sqlite3');
+ const copy = new Database(dest, { readonly: true });
+ try { return { path: dest, bytes: fs.statSync(dest).size, integrity_check: copy.pragma('integrity_check', { simple: true }) }; } finally { copy.close(); }
+}
+
+async function main() {
+ const h = db.getDb();
+ if (ROLLBACK) {
+ const file = JSON.parse(fs.readFileSync(ROLLBACK, 'utf8'));
+ const out = (file.changes || []).map((c) => (APPLY ? h15.rollback(h, c) : { ...c, would_restore: true }));
+ for (const o of out) console.log(`${o.object_id} ${o.action}: ${o.restored ? 'restored' : o.skipped || (o.would_restore ? 'would restore' : '')}`);
+ return 0;
+ }
+ const q = { all: (sql, p = []) => h.prepare(sql).all(...p), get: (sql, p = []) => h.prepare(sql).get(...p) };
+ const report = copyReport.buildReport(q, {});
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'h15-repair-'));
+ const steps = [];
+ try {
+ for (const item of report.objects) {
+ const probes = {};
+ const facts = {};
+ for (const l of item.locations.filter((x) => (x.provider === 'b2' || x.provider === 'r2') && x.state === 'corrupt')) {
+ try {
+ const u = await vs.presignGet(l.provider, l.key, 3600);
+ const file = path.join(tmp, `${item.object_id}.${l.provider}`);
+ facts[l.provider] = await download(typeof u === 'string' ? u : u.url, file);
+ probes[l.provider] = await probe(file);
+ } catch (err) { probes[l.provider] = { error: err.message }; }
+ }
+ const step = h15.plan(item, probes);
+ if (step.action === 'rebaseline') Object.assign(step, { facts: facts[step.provider], probe: { duration: probes[step.provider].duration, streams: probes[step.provider].streams } });
+ if (step.action === 'failed_recording') step.probe = probes[step.provider];
+ if (step.action === 'regenerate') {
+ const loc = item.locations.find((l) => l.provider === 'local');
+ const file = path.join(tmp, `${item.object_id}.jpg`);
+ try { step.facts = await extractFrame(step.source_vod, step.offset, file); step.file = file; step.target = loc && loc.key; }
+ catch (err) { Object.assign(step, { action: 'lost', reason: `the frame could not be extracted again: ${err.message}` }); }
+ }
+ if (step.probe && step.probe.errorLines) step.probe = { ...step.probe, errorLines: step.probe.errorLines.length };
+ steps.push(step);
+ console.log(`${item.object_id} ${item.kind} ${item.legacy_ref || ''}: ${step.action} — ${step.reason}`);
+ }
+ const record = { generated_at: new Date().toISOString(), applied: false, objects: report.count, plan: steps.map(({ file, ...s }) => s) };
+ if (!APPLY) {
+ const out = arg('out') ? path.resolve(arg('out')) : path.join(path.dirname(path.resolve(config.db.path)), 'reports', `h15-repair-${record.generated_at.replace(/[:.]/g, '-')}.json`);
+ fs.mkdirSync(path.dirname(out), { recursive: true });
+ fs.writeFileSync(out, JSON.stringify(record, null, 2));
+ console.log(`\nDry run: nothing changed. Plan written to ${out}`);
+ return 0;
+ }
+ const copyPath = BACKUP.replace(/\.json$/, '') + '.media.db';
+ for (const f of [BACKUP, copyPath]) if (fs.existsSync(f)) usage(`${f} already exists; pick a new --backup name`);
+ const dbCopy = await backupDatabase(copyPath);
+ if (dbCopy.integrity_check !== 'ok') { console.error(`backup failed integrity_check: ${dbCopy.integrity_check}; nothing changed`); return 1; }
+ const changes = [];
+ for (const step of steps) {
+ if (step.action === 'lost') continue;
+ if (step.action === 'regenerate') {
+ if (!step.target) { changes.push({ ...step, skipped: 'no local location to write to' }); continue; }
+ fs.mkdirSync(path.dirname(step.target), { recursive: true });
+ if (!fs.existsSync(step.target)) fs.copyFileSync(step.file, step.target);
+ }
+ const { file, ...s } = step;
+ changes.push(h15.apply(h, s, step.facts));
+ }
+ Object.assign(record, { applied: true, backup: dbCopy, changes });
+ fs.writeFileSync(BACKUP, JSON.stringify(record, null, 2));
+ for (const c of changes) console.log(`${c.object_id} ${c.action}: ${c.skipped ? `skipped (${c.skipped})` : 'applied'}`);
+ console.log(`\nApplied. Rollback file: ${BACKUP}; database copy: ${copyPath}`);
+ return 0;
+ } finally {
+ fs.rmSync(tmp, { recursive: true, force: true });
+ }
+}
+
+main().then((code) => process.exit(code), (err) => { console.error(`h15-repair: ${err.message}`); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,109 @@
@@ -0,0 +1,109 @@
+'use strict';
+/**
+ * H15 repair (roadmap WS-G task 7): what to do with each ready object that has no good copy
+ * (scripts/no-good-copy-report.js lists them). scripts/h15-repair.js runs it; this file holds the rules
+ * and the writes, so they are tested without storage.
+ *
+ * Every copy is tried before anything is condemned:
+ *
+ * rebaseline a B2/R2 copy is "corrupt" only because its size differs from the size recorded
+ * when the object was made (a recording's size captured mid-upload, no content
+ * hash), and that copy decodes to the end with a real duration. The copy is the
+ * object: size and sha256 are re-recorded from it and the location is present.
+ * failed_recording the only bytes are a fragment (under MIN_SECONDS, or nothing decodes): the
+ * recording failed when it was made, nothing was lost since. lifecycle_status
+ * becomes 'failed', so readiness says not playable and public players hide it.
+ * regenerate an AI-moment screenshot (ai-moment-vod<vod>-<seconds>.jpg) whose file is gone:
+ * the same frame is extracted again from that VOD's good copy and written back.
+ * lost nothing above applies: reported for the owner, nothing written.
+ *
+ * Writes happen only in apply(), each object in its own transaction and only while the row still holds
+ * what was read (so a concurrent change wins); every change is returned with its old values for the
+ * rollback file.
+ */
+const MIN_SECONDS = 2;
+const BENIGN = /non monotonically increasing dts|Last message repeated/i;
+const SHOT_RE = /ai-moment-vod(\d+)-(\d+)\.jpg$/;
+
+/** The decode verdict of one copy: { duration, streams, errorLines } → { ok, reason } */
+function judgeProbe(p) {
+ if (!p || p.error) return { ok: false, reason: `unreadable: ${(p && p.error) || 'no probe'}` };
+ if (!p.streams || !p.streams.length) return { ok: false, reason: 'no audio or video stream' };
+ const serious = (p.errorLines || []).filter((l) => l.trim() && !BENIGN.test(l));
+ if (serious.length > 5) return { ok: false, reason: `${serious.length} decode errors (first: ${serious[0].slice(0, 120)})` };
+ if (!(p.duration >= MIN_SECONDS)) return { ok: false, fragment: true, reason: `only ${Number(p.duration || 0).toFixed(1)} s decode` };
+ return { ok: true, reason: `decodes to the end: ${p.duration.toFixed(1)} s, ${p.streams.join('+')}${serious.length ? `, ${serious.length} minor decode error(s)` : ''}` };
+}
+
+/**
+ * The plan for one report item. probes: provider → probe of that copy (only for corrupt remote copies).
+ * → { object_id, action, provider?, reason, source_vod?, offset? }
+ */
+function plan(item, probes = {}) {
+ // What the plan was based on: apply() writes only while the row still holds it.
+ const base = { object_id: item.object_id, kind: item.kind, legacy_ref: item.legacy_ref || null, read: { size_bytes: item.size_bytes, content_hash: item.content_hash || null, lifecycle_status: 'ready' } };
+ const shot = item.kind === 'screenshot' && (item.locations || []).map((l) => SHOT_RE.exec(String(l.key || ''))).find(Boolean);
+ if (shot && (item.locations || []).every((l) => l.state === 'missing')) {
+ return { ...base, action: 'regenerate', source_vod: Number(shot[1]), offset: Number(shot[2]), reason: `the frame at ${shot[2]} s of VOD ${shot[1]} can be extracted again` };
+ }
+ const remote = (item.locations || []).filter((l) => (l.provider === 'b2' || l.provider === 'r2') && l.state === 'corrupt');
+ if (remote.length && !item.content_hash) {
+ let fragment = null;
+ for (const l of remote) {
+ const v = judgeProbe(probes[l.provider]);
+ if (v.ok) return { ...base, action: 'rebaseline', provider: l.provider, key: l.key, reason: `${l.provider} copy ${v.reason}; only its size differed from the size recorded at creation (${item.size_bytes} bytes)` };
+ if (v.fragment) fragment = { provider: l.provider, reason: v.reason };
+ }
+ if (fragment) return { ...base, action: 'failed_recording', provider: fragment.provider, reason: `the only copy is a fragment (${fragment.reason}): the recording failed when it was made` };
+ }
+ return { ...base, action: 'lost', reason: 'no copy decodes and nothing can regenerate it' };
+}
+
+/**
+ * Apply one planned change. h = better-sqlite3 handle. facts: { size, sha256 } of the good bytes (rebaseline,
+ * regenerate). → the change with old values, or { skipped } when the row moved on.
+ */
+function apply(h, step, facts = {}) {
+ const cur = h.prepare('SELECT id, size_bytes, content_hash, lifecycle_status FROM media_objects WHERE id = ?').get(step.object_id);
+ if (!cur) return { ...step, skipped: 'object gone' };
+ const obj = { ...cur, ...(step.read || {}) };
+ const tx = h.transaction(() => {
+ if (step.action === 'rebaseline' || step.action === 'regenerate') {
+ const provider = step.action === 'regenerate' ? 'local' : step.provider;
+ const loc = h.prepare('SELECT id, state, size_bytes FROM media_locations WHERE object_id = ? AND provider = ?').get(obj.id, provider);
+ if (!loc) return { ...step, skipped: `no ${provider} location` };
+ const r = h.prepare("UPDATE media_objects SET size_bytes = ?, content_hash = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND size_bytes IS ? AND content_hash IS ?")
+ .run(facts.size, facts.sha256, obj.id, obj.size_bytes, obj.content_hash);
+ if (!r.changes) return { ...step, skipped: 'the object changed since it was read' };
+ h.prepare("UPDATE media_locations SET state = 'present', size_bytes = ?, verified_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(facts.size, loc.id);
+ return { ...step, old: { size_bytes: obj.size_bytes, content_hash: obj.content_hash, location_state: loc.state, location_size: loc.size_bytes }, new: { size_bytes: facts.size, content_hash: facts.sha256, location_state: 'present' } };
+ }
+ if (step.action === 'failed_recording') {
+ const r = h.prepare("UPDATE media_objects SET lifecycle_status = 'failed', updated_at = CURRENT_TIMESTAMP WHERE id = ? AND lifecycle_status = ?").run(obj.id, obj.lifecycle_status);
+ if (!r.changes) return { ...step, skipped: 'the object changed since it was read' };
+ return { ...step, old: { lifecycle_status: obj.lifecycle_status }, new: { lifecycle_status: 'failed' } };
+ }
+ return { ...step, skipped: 'nothing to write' };
+ });
+ return tx();
+}
+
+/** Undo one applied change where the row still holds what the repair wrote. → { restored } | { skipped } */
+function rollback(h, change) {
+ if (!change || !change.old || !change.new) return { ...change, skipped: 'not an applied change' };
+ const tx = h.transaction(() => {
+ if (change.action === 'failed_recording') {
+ const r = h.prepare('UPDATE media_objects SET lifecycle_status = ? WHERE id = ? AND lifecycle_status = ?').run(change.old.lifecycle_status, change.object_id, change.new.lifecycle_status);
+ return r.changes ? { ...change, restored: true } : { ...change, skipped: 'the row changed since' };
+ }
+ const r = h.prepare('UPDATE media_objects SET size_bytes = ?, content_hash = ? WHERE id = ? AND size_bytes IS ? AND content_hash IS ?')
+ .run(change.old.size_bytes, change.old.content_hash, change.object_id, change.new.size_bytes, change.new.content_hash);
+ if (!r.changes) return { ...change, skipped: 'the row changed since' };
+ const provider = change.action === 'regenerate' ? 'local' : change.provider;
+ h.prepare('UPDATE media_locations SET state = ?, size_bytes = ? WHERE object_id = ? AND provider = ?').run(change.old.location_state, change.old.location_size, change.object_id, provider);
+ return { ...change, restored: true };
+ });
+ return tx();
+}
+
+module.exports = { plan, apply, rollback, judgeProbe, MIN_SECONDS, SHOT_RE };
new file mode 100644
new file mode 100644
@@ -0,0 +1,53 @@
@@ -0,0 +1,53 @@
+'use strict';
+// H15 repair (WS-G task 7): the plan for each object with no good copy, and its writes and rollback.
+const assert = require('assert');
+const Database = require('better-sqlite3');
+const { plan, apply, rollback, judgeProbe } = require('../server/objects/h15-repair');
+
+const good = { duration: 1721.9, streams: ['vp8', 'opus'], errorLines: Array(4700).fill('[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 5 >= 5').concat(['[opus @ 0x2] Error parsing Opus packet header.']) };
+const vod = (over = {}) => ({ object_id: 'med_1', kind: 'vod', size_bytes: 13631488, content_hash: null, legacy_ref: 'legacy:live:vod:1277', locations: [{ provider: 'b2', key: 'vods/a.webm', state: 'corrupt', size_bytes: 14609471 }], ...over });
+
+// ── The verdict on a decoded copy ──
+assert.strictEqual(judgeProbe(good).ok, true, 'timestamp warnings of a live recording do not condemn it');
+assert.strictEqual(judgeProbe({ duration: 0.6, streams: ['vp8'], errorLines: [] }).fragment, true);
+assert.strictEqual(judgeProbe({ error: 'Invalid data found' }).ok, false);
+assert.strictEqual(judgeProbe({ duration: 400, streams: ['vp8'], errorLines: Array(9).fill('[vp8 @ 1] Truncated packet') }).ok, false, 'real decode errors do');
+assert.strictEqual(judgeProbe({ duration: 400, streams: [], errorLines: [] }).ok, false);
+
+// ── The plan ──
+assert.strictEqual(plan(vod(), { b2: good }).action, 'rebaseline');
+assert.match(plan(vod(), { b2: good }).reason, /only its size differed/);
+assert.strictEqual(plan(vod(), { b2: { duration: 0.6, streams: ['vp8'], errorLines: [] } }).action, 'failed_recording');
+assert.strictEqual(plan(vod(), { b2: { error: 'moov atom not found' } }).action, 'lost');
+assert.strictEqual(plan(vod({ content_hash: 'abc' }), { b2: good }).action, 'lost', 'a recorded hash that disagrees is real corruption: never rebaselined');
+const shot = { object_id: 'med_2', kind: 'screenshot', size_bytes: 0, content_hash: null, locations: [{ provider: 'local', key: '/data/pastes/screenshots/ai-moment-vod1691-32230.jpg', state: 'missing' }] };
+assert.deepStrictEqual((({ action, source_vod, offset }) => ({ action, source_vod, offset }))(plan(shot)), { action: 'regenerate', source_vod: 1691, offset: 32230 });
+assert.strictEqual(plan({ ...shot, locations: [{ provider: 'local', key: '/x/other.jpg', state: 'missing' }] }).action, 'lost');
+
+// ── Writes, only while the row holds what was read, and their rollback ──
+const h = new Database(':memory:');
+h.exec(`CREATE TABLE media_objects (id TEXT PRIMARY KEY, size_bytes INTEGER, content_hash TEXT, lifecycle_status TEXT, updated_at TEXT);
+ CREATE TABLE media_locations (id INTEGER PRIMARY KEY, object_id TEXT, provider TEXT, key TEXT, state TEXT, size_bytes INTEGER, verified_at TEXT, updated_at TEXT);`);
+h.prepare("INSERT INTO media_objects VALUES ('med_1', 13631488, NULL, 'ready', NULL), ('med_3', 262144, NULL, 'ready', NULL), ('med_2', 0, NULL, 'ready', NULL)").run();
+h.prepare("INSERT INTO media_locations (object_id, provider, key, state, size_bytes) VALUES ('med_1', 'b2', 'vods/a.webm', 'corrupt', 14609471), ('med_3', 'b2', 'vods/b.webm', 'corrupt', 218283), ('med_2', 'local', '/x.jpg', 'missing', NULL)").run();
+
+const c1 = apply(h, plan(vod(), { b2: good }), { size: 14609471, sha256: 'f'.repeat(64) });
+assert.deepStrictEqual(c1.old, { size_bytes: 13631488, content_hash: null, location_state: 'corrupt', location_size: 14609471 });
+assert.deepStrictEqual(h.prepare("SELECT size_bytes, content_hash FROM media_objects WHERE id = 'med_1'").get(), { size_bytes: 14609471, content_hash: 'f'.repeat(64) });
+assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_1'").get().state, 'present');
+assert.match(apply(h, plan(vod(), { b2: good }), { size: 1, sha256: 'e'.repeat(64) }).skipped, /changed since/, 'a second run finds the row moved on');
+
+const c3 = apply(h, { object_id: 'med_3', action: 'failed_recording', provider: 'b2' });
+assert.strictEqual(h.prepare("SELECT lifecycle_status FROM media_objects WHERE id = 'med_3'").get().lifecycle_status, 'failed');
+
+const c2 = apply(h, plan(shot), { size: 5120, sha256: 'a'.repeat(64) });
+assert.strictEqual(c2.new.location_state, 'present');
+assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_2'").get().state, 'present');
+
+for (const c of [c1, c2, c3]) assert.strictEqual(rollback(h, c).restored, true);
+assert.deepStrictEqual(h.prepare("SELECT size_bytes, content_hash, lifecycle_status FROM media_objects WHERE id = 'med_1'").get(), { size_bytes: 13631488, content_hash: null, lifecycle_status: 'ready' });
+assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_1'").get().state, 'corrupt');
+assert.strictEqual(h.prepare("SELECT lifecycle_status FROM media_objects WHERE id = 'med_3'").get().lifecycle_status, 'ready');
+assert.strictEqual(rollback(h, c1).skipped, 'the row changed since', 'rolling back twice changes nothing');
+
+console.log('h15 repair: all checks passed');