openvibers/openvibe.media Commits

H15 repair (WS-G task 7): scripts/h15-repair.js tries every copy of each object with no good copy before anything is condemned: a B2/R2 copy marked corrupt only because its size differs from the size recorded at creation (no content hash) is downloaded and decoded to the end, and a good one becomes the object (size and sha256 re-recorded); a fragment under 2 s is a recording that failed when it was made (lifecycle failed, not playable); a missing AI-moment frame is extracted again from its VOD; anything else is reported as lost. Dry run by default; --apply takes an online backup and writes a rollback file; --rollback restores

OP OpenVibers committed 25 Sept 2026 · cbe1413040 · 3 files, +332 −0


Changes

ddiff --git a/scripts/h15-repair.js b/scripts/h15-repair.js
nnew file mode 100644
iindex 0000000..5baf407
---- /dev/null
++++ b/scripts/h15-repair.js
@@@ -0,0 +1,170 @@
++#!/usr/bin/env node
++'use strict';
++/**
++ * H15 repair (roadmap WS-G task 7), for the operator: every ready object with no good copy
++ * (scripts/no-good-copy-report.js) gets every copy tried before anything is condemned
++ * (server/objects/h15-repair.js has the rules):
++ *
++ * node scripts/h15-repair.js [--out <report.json>]
++ * Dry run (the default): downloads each corrupt B2/R2 copy (to a temporary file; B2/R2 egress),
++ * probes and decodes it with ffprobe/ffmpeg, extracts each missing AI-moment frame again from
++ * its VOD's copy, and writes the plan: rebaseline, failed_recording, regenerate or lost, with the
++ * reason. Changes nothing. The report is for the owner (default <data>/reports/h15-repair-….json).
++ * node scripts/h15-repair.js --apply --backup <file.json>
++ * First an online backup of the database to <file>.media.db (checked with integrity_check), then
++ * the plan: re-records size and sha256 from a good remote copy, marks a failed recording
++ * lifecycle_status 'failed' (not playable, hidden from public players), writes a regenerated
++ * frame back to its location. <file.json> lists every change with its old values (the rollback
++ * file). Nothing is deleted, locally or remotely. Refuses to overwrite either file.
++ * node scripts/h15-repair.js --rollback <file.json> [--apply]
++ * Restores the old values of the changes the file lists, where the row still holds what the
++ * repair wrote (a regenerated file stays: it is the same frame). Without --apply it only reports.
++ * (every command takes --db <media.db>; the database is DB_PATH otherwise)
++ *
++ * B2/R2 credentials come from the environment (the service's /etc/openvibe/media.env names). Run it as
++ * the service user so written files belong to it. Exit codes: 0 done, 1 error, 2 bad usage.
++ */
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const crypto = require('crypto');
++const { execFile } = require('child_process');
++
++const args = process.argv.slice(2);
++const has = (name) => args.includes(`--${name}`);
++const arg = (name) => { const i = args.indexOf(`--${name}`); return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : null; };
++const usage = (msg) => { console.error(`h15-repair: ${msg}`); process.exit(2); };
++for (const flag of ['db', 'backup', 'rollback', 'out']) if (has(flag) && !arg(flag)) usage(`--${flag} needs a value`);
++if (arg('db')) process.env.DB_PATH = path.resolve(arg('db'));
++const APPLY = has('apply');
++const BACKUP = arg('backup') ? path.resolve(arg('backup')) : null;
++const ROLLBACK = arg('rollback') ? path.resolve(arg('rollback')) : null;
++if (ROLLBACK && BACKUP) usage('--rollback and --backup do not go together');
++if (APPLY && !ROLLBACK && !BACKUP) usage('--apply needs --backup <file.json> (the rollback file; the database copy goes next to it)');
++
++const config = require('../server/config');
++const db = require('../server/db/database');
++const copyReport = require('../server/objects/copy-report');
++const h15 = require('../server/objects/h15-repair');
++const vs = require('../server/vod/vod-storage');
++
++const run = (cmd, argv, timeoutMs) => new Promise((resolve) => {
++ execFile(cmd, argv, { timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => resolve({ err, stdout: String(stdout || ''), stderr: String(stderr || '') }));
++});
++
++/** Download a presigned URL to a file, hashing as it goes. → { size, sha256 } */
++async function download(url, file) {
++ const res = await fetch(url, { signal: AbortSignal.timeout(30 * 60 * 1000) });
++ if (!res.ok || !res.body) throw new Error(`download answered ${res.status}`);
++ const hash = crypto.createHash('sha256');
++ const out = fs.createWriteStream(file);
++ let size = 0;
++ for await (const chunk of res.body) { hash.update(chunk); size += chunk.length; if (!out.write(chunk)) await new Promise((r) => out.once('drain', r)); }
++ await new Promise((r, j) => out.end((e) => (e ? j(e) : r())));
++ return { size, sha256: hash.digest('hex') };
++}
++
++/** ffprobe + a full decode. → { duration, streams, errorLines } | { error } */
++async function probe(file) {
++ const p = await run('ffprobe', ['-v', 'error', '-show_entries', 'format=duration:stream=codec_name', '-of', 'json', file], 120000);
++ if (p.err) return { error: (p.stderr || p.err.message).split('\n')[0] };
++ let j = {}; try { j = JSON.parse(p.stdout); } catch { return { error: 'ffprobe output unreadable' }; }
++ const d = await run('ffmpeg', ['-nostdin', '-v', 'error', '-i', file, '-f', 'null', '-'], 60 * 60 * 1000);
++ return { duration: Number(j.format && j.format.duration) || 0, streams: (j.streams || []).map((s) => s.codec_name).filter(Boolean), errorLines: d.stderr.split('\n').filter(Boolean) };
++}
++
++const sha256File = (file) => new Promise((resolve, reject) => {
++ const h = crypto.createHash('sha256');
++ fs.createReadStream(file).on('data', (c) => h.update(c)).on('end', () => resolve(h.digest('hex'))).on('error', reject);
++});
++
++/** The frame at `offset` seconds of VOD `vodId`, extracted from its good copy into `file`. → { size, sha256 } */
++async function extractFrame(vodId, offset, file) {
++ const v = db.getDb().prepare('SELECT id, storage_provider, storage_key, file_path FROM vods WHERE id = ?').get(vodId);
++ if (!v) throw new Error(`VOD ${vodId} does not exist`);
++ let input = v.file_path && fs.existsSync(v.file_path) ? v.file_path : null;
++ if (!input && v.storage_provider && v.storage_key) { const u = await vs.presignGet(v.storage_provider, v.storage_key, 3600); input = typeof u === 'string' ? u : u.url; }
++ if (!input) throw new Error(`VOD ${vodId} has no readable copy`);
++ const r = await run('ffmpeg', ['-nostdin', '-v', 'error', '-ss', String(offset), '-i', input, '-frames:v', '1', '-q:v', '3', '-y', file], 20 * 60 * 1000);
++ if (r.err || !fs.existsSync(file) || !fs.statSync(file).size) throw new Error(`no frame at ${offset} s: ${(r.stderr || (r.err && r.err.message) || '').split('\n')[0]}`);
++ return { size: fs.statSync(file).size, sha256: await sha256File(file) };
++}
++
++async function backupDatabase(dest) {
++ await db.getDb().backup(dest);
++ const Database = require('better-sqlite3');
++ const copy = new Database(dest, { readonly: true });
++ try { return { path: dest, bytes: fs.statSync(dest).size, integrity_check: copy.pragma('integrity_check', { simple: true }) }; } finally { copy.close(); }
++}
++
++async function main() {
++ const h = db.getDb();
++ if (ROLLBACK) {
++ const file = JSON.parse(fs.readFileSync(ROLLBACK, 'utf8'));
++ const out = (file.changes || []).map((c) => (APPLY ? h15.rollback(h, c) : { ...c, would_restore: true }));
++ for (const o of out) console.log(`${o.object_id} ${o.action}: ${o.restored ? 'restored' : o.skipped || (o.would_restore ? 'would restore' : '')}`);
++ return 0;
++ }
++ const q = { all: (sql, p = []) => h.prepare(sql).all(...p), get: (sql, p = []) => h.prepare(sql).get(...p) };
++ const report = copyReport.buildReport(q, {});
++ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'h15-repair-'));
++ const steps = [];
++ try {
++ for (const item of report.objects) {
++ const probes = {};
++ const facts = {};
++ for (const l of item.locations.filter((x) => (x.provider === 'b2' || x.provider === 'r2') && x.state === 'corrupt')) {
++ try {
++ const u = await vs.presignGet(l.provider, l.key, 3600);
++ const file = path.join(tmp, `${item.object_id}.${l.provider}`);
++ facts[l.provider] = await download(typeof u === 'string' ? u : u.url, file);
++ probes[l.provider] = await probe(file);
++ } catch (err) { probes[l.provider] = { error: err.message }; }
++ }
++ const step = h15.plan(item, probes);
++ if (step.action === 'rebaseline') Object.assign(step, { facts: facts[step.provider], probe: { duration: probes[step.provider].duration, streams: probes[step.provider].streams } });
++ if (step.action === 'failed_recording') step.probe = probes[step.provider];
++ if (step.action === 'regenerate') {
++ const loc = item.locations.find((l) => l.provider === 'local');
++ const file = path.join(tmp, `${item.object_id}.jpg`);
++ try { step.facts = await extractFrame(step.source_vod, step.offset, file); step.file = file; step.target = loc && loc.key; }
++ catch (err) { Object.assign(step, { action: 'lost', reason: `the frame could not be extracted again: ${err.message}` }); }
++ }
++ if (step.probe && step.probe.errorLines) step.probe = { ...step.probe, errorLines: step.probe.errorLines.length };
++ steps.push(step);
++ console.log(`${item.object_id} ${item.kind} ${item.legacy_ref || ''}: ${step.action} — ${step.reason}`);
++ }
++ const record = { generated_at: new Date().toISOString(), applied: false, objects: report.count, plan: steps.map(({ file, ...s }) => s) };
++ if (!APPLY) {
++ const out = arg('out') ? path.resolve(arg('out')) : path.join(path.dirname(path.resolve(config.db.path)), 'reports', `h15-repair-${record.generated_at.replace(/[:.]/g, '-')}.json`);
++ fs.mkdirSync(path.dirname(out), { recursive: true });
++ fs.writeFileSync(out, JSON.stringify(record, null, 2));
++ console.log(`\nDry run: nothing changed. Plan written to ${out}`);
++ return 0;
++ }
++ const copyPath = BACKUP.replace(/\.json$/, '') + '.media.db';
++ for (const f of [BACKUP, copyPath]) if (fs.existsSync(f)) usage(`${f} already exists; pick a new --backup name`);
++ const dbCopy = await backupDatabase(copyPath);
++ if (dbCopy.integrity_check !== 'ok') { console.error(`backup failed integrity_check: ${dbCopy.integrity_check}; nothing changed`); return 1; }
++ const changes = [];
++ for (const step of steps) {
++ if (step.action === 'lost') continue;
++ if (step.action === 'regenerate') {
++ if (!step.target) { changes.push({ ...step, skipped: 'no local location to write to' }); continue; }
++ fs.mkdirSync(path.dirname(step.target), { recursive: true });
++ if (!fs.existsSync(step.target)) fs.copyFileSync(step.file, step.target);
++ }
++ const { file, ...s } = step;
++ changes.push(h15.apply(h, s, step.facts));
++ }
++ Object.assign(record, { applied: true, backup: dbCopy, changes });
++ fs.writeFileSync(BACKUP, JSON.stringify(record, null, 2));
++ for (const c of changes) console.log(`${c.object_id} ${c.action}: ${c.skipped ? `skipped (${c.skipped})` : 'applied'}`);
++ console.log(`\nApplied. Rollback file: ${BACKUP}; database copy: ${copyPath}`);
++ return 0;
++ } finally {
++ fs.rmSync(tmp, { recursive: true, force: true });
++ }
++}
++
++main().then((code) => process.exit(code), (err) => { console.error(`h15-repair: ${err.message}`); process.exit(1); });
ddiff --git a/server/objects/h15-repair.js b/server/objects/h15-repair.js
nnew file mode 100644
iindex 0000000..22076af
---- /dev/null
++++ b/server/objects/h15-repair.js
@@@ -0,0 +1,109 @@
++'use strict';
++/**
++ * H15 repair (roadmap WS-G task 7): what to do with each ready object that has no good copy
++ * (scripts/no-good-copy-report.js lists them). scripts/h15-repair.js runs it; this file holds the rules
++ * and the writes, so they are tested without storage.
++ *
++ * Every copy is tried before anything is condemned:
++ *
++ * rebaseline a B2/R2 copy is "corrupt" only because its size differs from the size recorded
++ * when the object was made (a recording's size captured mid-upload, no content
++ * hash), and that copy decodes to the end with a real duration. The copy is the
++ * object: size and sha256 are re-recorded from it and the location is present.
++ * failed_recording the only bytes are a fragment (under MIN_SECONDS, or nothing decodes): the
++ * recording failed when it was made, nothing was lost since. lifecycle_status
++ * becomes 'failed', so readiness says not playable and public players hide it.
++ * regenerate an AI-moment screenshot (ai-moment-vod<vod>-<seconds>.jpg) whose file is gone:
++ * the same frame is extracted again from that VOD's good copy and written back.
++ * lost nothing above applies: reported for the owner, nothing written.
++ *
++ * Writes happen only in apply(), each object in its own transaction and only while the row still holds
++ * what was read (so a concurrent change wins); every change is returned with its old values for the
++ * rollback file.
++ */
++const MIN_SECONDS = 2;
++const BENIGN = /non monotonically increasing dts|Last message repeated/i;
++const SHOT_RE = /ai-moment-vod(\d+)-(\d+)\.jpg$/;
++
++/** The decode verdict of one copy: { duration, streams, errorLines } → { ok, reason } */
++function judgeProbe(p) {
++ if (!p || p.error) return { ok: false, reason: `unreadable: ${(p && p.error) || 'no probe'}` };
++ if (!p.streams || !p.streams.length) return { ok: false, reason: 'no audio or video stream' };
++ const serious = (p.errorLines || []).filter((l) => l.trim() && !BENIGN.test(l));
++ if (serious.length > 5) return { ok: false, reason: `${serious.length} decode errors (first: ${serious[0].slice(0, 120)})` };
++ if (!(p.duration >= MIN_SECONDS)) return { ok: false, fragment: true, reason: `only ${Number(p.duration || 0).toFixed(1)} s decode` };
++ return { ok: true, reason: `decodes to the end: ${p.duration.toFixed(1)} s, ${p.streams.join('+')}${serious.length ? `, ${serious.length} minor decode error(s)` : ''}` };
++}
++
++/**
++ * The plan for one report item. probes: provider → probe of that copy (only for corrupt remote copies).
++ * → { object_id, action, provider?, reason, source_vod?, offset? }
++ */
++function plan(item, probes = {}) {
++ // What the plan was based on: apply() writes only while the row still holds it.
++ const base = { object_id: item.object_id, kind: item.kind, legacy_ref: item.legacy_ref || null, read: { size_bytes: item.size_bytes, content_hash: item.content_hash || null, lifecycle_status: 'ready' } };
++ const shot = item.kind === 'screenshot' && (item.locations || []).map((l) => SHOT_RE.exec(String(l.key || ''))).find(Boolean);
++ if (shot && (item.locations || []).every((l) => l.state === 'missing')) {
++ return { ...base, action: 'regenerate', source_vod: Number(shot[1]), offset: Number(shot[2]), reason: `the frame at ${shot[2]} s of VOD ${shot[1]} can be extracted again` };
++ }
++ const remote = (item.locations || []).filter((l) => (l.provider === 'b2' || l.provider === 'r2') && l.state === 'corrupt');
++ if (remote.length && !item.content_hash) {
++ let fragment = null;
++ for (const l of remote) {
++ const v = judgeProbe(probes[l.provider]);
++ if (v.ok) return { ...base, action: 'rebaseline', provider: l.provider, key: l.key, reason: `${l.provider} copy ${v.reason}; only its size differed from the size recorded at creation (${item.size_bytes} bytes)` };
++ if (v.fragment) fragment = { provider: l.provider, reason: v.reason };
++ }
++ if (fragment) return { ...base, action: 'failed_recording', provider: fragment.provider, reason: `the only copy is a fragment (${fragment.reason}): the recording failed when it was made` };
++ }
++ return { ...base, action: 'lost', reason: 'no copy decodes and nothing can regenerate it' };
++}
++
++/**
++ * Apply one planned change. h = better-sqlite3 handle. facts: { size, sha256 } of the good bytes (rebaseline,
++ * regenerate). → the change with old values, or { skipped } when the row moved on.
++ */
++function apply(h, step, facts = {}) {
++ const cur = h.prepare('SELECT id, size_bytes, content_hash, lifecycle_status FROM media_objects WHERE id = ?').get(step.object_id);
++ if (!cur) return { ...step, skipped: 'object gone' };
++ const obj = { ...cur, ...(step.read || {}) };
++ const tx = h.transaction(() => {
++ if (step.action === 'rebaseline' || step.action === 'regenerate') {
++ const provider = step.action === 'regenerate' ? 'local' : step.provider;
++ const loc = h.prepare('SELECT id, state, size_bytes FROM media_locations WHERE object_id = ? AND provider = ?').get(obj.id, provider);
++ if (!loc) return { ...step, skipped: `no ${provider} location` };
++ const r = h.prepare("UPDATE media_objects SET size_bytes = ?, content_hash = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND size_bytes IS ? AND content_hash IS ?")
++ .run(facts.size, facts.sha256, obj.id, obj.size_bytes, obj.content_hash);
++ if (!r.changes) return { ...step, skipped: 'the object changed since it was read' };
++ h.prepare("UPDATE media_locations SET state = 'present', size_bytes = ?, verified_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(facts.size, loc.id);
++ return { ...step, old: { size_bytes: obj.size_bytes, content_hash: obj.content_hash, location_state: loc.state, location_size: loc.size_bytes }, new: { size_bytes: facts.size, content_hash: facts.sha256, location_state: 'present' } };
++ }
++ if (step.action === 'failed_recording') {
++ const r = h.prepare("UPDATE media_objects SET lifecycle_status = 'failed', updated_at = CURRENT_TIMESTAMP WHERE id = ? AND lifecycle_status = ?").run(obj.id, obj.lifecycle_status);
++ if (!r.changes) return { ...step, skipped: 'the object changed since it was read' };
++ return { ...step, old: { lifecycle_status: obj.lifecycle_status }, new: { lifecycle_status: 'failed' } };
++ }
++ return { ...step, skipped: 'nothing to write' };
++ });
++ return tx();
++}
++
++/** Undo one applied change where the row still holds what the repair wrote. → { restored } | { skipped } */
++function rollback(h, change) {
++ if (!change || !change.old || !change.new) return { ...change, skipped: 'not an applied change' };
++ const tx = h.transaction(() => {
++ if (change.action === 'failed_recording') {
++ const r = h.prepare('UPDATE media_objects SET lifecycle_status = ? WHERE id = ? AND lifecycle_status = ?').run(change.old.lifecycle_status, change.object_id, change.new.lifecycle_status);
++ return r.changes ? { ...change, restored: true } : { ...change, skipped: 'the row changed since' };
++ }
++ const r = h.prepare('UPDATE media_objects SET size_bytes = ?, content_hash = ? WHERE id = ? AND size_bytes IS ? AND content_hash IS ?')
++ .run(change.old.size_bytes, change.old.content_hash, change.object_id, change.new.size_bytes, change.new.content_hash);
++ if (!r.changes) return { ...change, skipped: 'the row changed since' };
++ const provider = change.action === 'regenerate' ? 'local' : change.provider;
++ h.prepare('UPDATE media_locations SET state = ?, size_bytes = ? WHERE object_id = ? AND provider = ?').run(change.old.location_state, change.old.location_size, change.object_id, provider);
++ return { ...change, restored: true };
++ });
++ return tx();
++}
++
++module.exports = { plan, apply, rollback, judgeProbe, MIN_SECONDS, SHOT_RE };
ddiff --git a/test/h15-repair.test.js b/test/h15-repair.test.js
nnew file mode 100644
iindex 0000000..3315c43
---- /dev/null
++++ b/test/h15-repair.test.js
@@@ -0,0 +1,53 @@
++'use strict';
++// H15 repair (WS-G task 7): the plan for each object with no good copy, and its writes and rollback.
++const assert = require('assert');
++const Database = require('better-sqlite3');
++const { plan, apply, rollback, judgeProbe } = require('../server/objects/h15-repair');
++
++const good = { duration: 1721.9, streams: ['vp8', 'opus'], errorLines: Array(4700).fill('[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 5 >= 5').concat(['[opus @ 0x2] Error parsing Opus packet header.']) };
++const vod = (over = {}) => ({ object_id: 'med_1', kind: 'vod', size_bytes: 13631488, content_hash: null, legacy_ref: 'legacy:live:vod:1277', locations: [{ provider: 'b2', key: 'vods/a.webm', state: 'corrupt', size_bytes: 14609471 }], ...over });
++
++// ── The verdict on a decoded copy ──
++assert.strictEqual(judgeProbe(good).ok, true, 'timestamp warnings of a live recording do not condemn it');
++assert.strictEqual(judgeProbe({ duration: 0.6, streams: ['vp8'], errorLines: [] }).fragment, true);
++assert.strictEqual(judgeProbe({ error: 'Invalid data found' }).ok, false);
++assert.strictEqual(judgeProbe({ duration: 400, streams: ['vp8'], errorLines: Array(9).fill('[vp8 @ 1] Truncated packet') }).ok, false, 'real decode errors do');
++assert.strictEqual(judgeProbe({ duration: 400, streams: [], errorLines: [] }).ok, false);
++
++// ── The plan ──
++assert.strictEqual(plan(vod(), { b2: good }).action, 'rebaseline');
++assert.match(plan(vod(), { b2: good }).reason, /only its size differed/);
++assert.strictEqual(plan(vod(), { b2: { duration: 0.6, streams: ['vp8'], errorLines: [] } }).action, 'failed_recording');
++assert.strictEqual(plan(vod(), { b2: { error: 'moov atom not found' } }).action, 'lost');
++assert.strictEqual(plan(vod({ content_hash: 'abc' }), { b2: good }).action, 'lost', 'a recorded hash that disagrees is real corruption: never rebaselined');
++const shot = { object_id: 'med_2', kind: 'screenshot', size_bytes: 0, content_hash: null, locations: [{ provider: 'local', key: '/data/pastes/screenshots/ai-moment-vod1691-32230.jpg', state: 'missing' }] };
++assert.deepStrictEqual((({ action, source_vod, offset }) => ({ action, source_vod, offset }))(plan(shot)), { action: 'regenerate', source_vod: 1691, offset: 32230 });
++assert.strictEqual(plan({ ...shot, locations: [{ provider: 'local', key: '/x/other.jpg', state: 'missing' }] }).action, 'lost');
++
++// ── Writes, only while the row holds what was read, and their rollback ──
++const h = new Database(':memory:');
++h.exec(`CREATE TABLE media_objects (id TEXT PRIMARY KEY, size_bytes INTEGER, content_hash TEXT, lifecycle_status TEXT, updated_at TEXT);
++ CREATE TABLE media_locations (id INTEGER PRIMARY KEY, object_id TEXT, provider TEXT, key TEXT, state TEXT, size_bytes INTEGER, verified_at TEXT, updated_at TEXT);`);
++h.prepare("INSERT INTO media_objects VALUES ('med_1', 13631488, NULL, 'ready', NULL), ('med_3', 262144, NULL, 'ready', NULL), ('med_2', 0, NULL, 'ready', NULL)").run();
++h.prepare("INSERT INTO media_locations (object_id, provider, key, state, size_bytes) VALUES ('med_1', 'b2', 'vods/a.webm', 'corrupt', 14609471), ('med_3', 'b2', 'vods/b.webm', 'corrupt', 218283), ('med_2', 'local', '/x.jpg', 'missing', NULL)").run();
++
++const c1 = apply(h, plan(vod(), { b2: good }), { size: 14609471, sha256: 'f'.repeat(64) });
++assert.deepStrictEqual(c1.old, { size_bytes: 13631488, content_hash: null, location_state: 'corrupt', location_size: 14609471 });
++assert.deepStrictEqual(h.prepare("SELECT size_bytes, content_hash FROM media_objects WHERE id = 'med_1'").get(), { size_bytes: 14609471, content_hash: 'f'.repeat(64) });
++assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_1'").get().state, 'present');
++assert.match(apply(h, plan(vod(), { b2: good }), { size: 1, sha256: 'e'.repeat(64) }).skipped, /changed since/, 'a second run finds the row moved on');
++
++const c3 = apply(h, { object_id: 'med_3', action: 'failed_recording', provider: 'b2' });
++assert.strictEqual(h.prepare("SELECT lifecycle_status FROM media_objects WHERE id = 'med_3'").get().lifecycle_status, 'failed');
++
++const c2 = apply(h, plan(shot), { size: 5120, sha256: 'a'.repeat(64) });
++assert.strictEqual(c2.new.location_state, 'present');
++assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_2'").get().state, 'present');
++
++for (const c of [c1, c2, c3]) assert.strictEqual(rollback(h, c).restored, true);
++assert.deepStrictEqual(h.prepare("SELECT size_bytes, content_hash, lifecycle_status FROM media_objects WHERE id = 'med_1'").get(), { size_bytes: 13631488, content_hash: null, lifecycle_status: 'ready' });
++assert.strictEqual(h.prepare("SELECT state FROM media_locations WHERE object_id = 'med_1'").get().state, 'corrupt');
++assert.strictEqual(h.prepare("SELECT lifecycle_status FROM media_objects WHERE id = 'med_3'").get().lifecycle_status, 'ready');
++assert.strictEqual(rollback(h, c1).skipped, 'the row changed since', 'rolling back twice changes nothing');
++
++console.log('h15 repair: all checks passed');