openvibers/openvibe.community Commits
Community: move screenshots without losing legacy objects (#264)
Changes
@@@ -2,6 +2,11 @@## Unreleased++- The screenshot move script (2026-11-06) no longer reads a 404 from Media's community tenant as "already gone": for a++ `legacy:live:paste:<slug>` object it keeps `previous_media_ref` and retires the object through Media's internal legacy-paste++ route. `--apply` pages with an id cursor and reaches every paste in one run, `--retire-ids <file>` retires named objects++ whose paste record was lost (one Media id per line), and the dry run prints the moves, retires and refusals it would make.++- Sharing reaches pastes (design 2026-11-06, the pilot): `openvibe-sdk` v0.85.0 and `openvibe-shared` v4.114.0, with`openvibe-contracts` v0.291.0. `migrations/0015_sharing.sql` adds openvibe-sdk/share's tables (expand), and`migrations/0016_sharing_backfill.sql` registers every existing paste once (public→public, unlisted→link,@@@ -13,13 +13,32 @@* The old id is recorded as `previous_media_ref` before the delete, so a failed delete is retried on* the next run instead of being forgotten.*-- * npm run screenshots:private -- --limit 50 # a dry run of 50-- * npm run screenshots:private -- --apply # move every one (batches of 50)-- * npm run screenshots:private -- --apply --keep-old # move, but leave the old objects (an operator's call)++ * The old object may be Community's own (delete it through the community tenant) or an inherited++ * Live-era paste screenshot (`legacy:live:paste:<slug>` in Media's live tenant). Media's++ * tenant-scoped delete answers 404 for the latter and its v2 delete refuses every legacy object, so++ * the script retires those through Media's internal legacy-paste route. A 404 is therefore a failure++ * that keeps `previous_media_ref`: it never means "already gone".++ *++ * --apply pages through every screenshot paste with an id cursor, in batches of --limit (500 by++ * default), so one run reaches every row instead of re-reading the first batch forever. A dry run++ * looks at one batch (50 by default) and prints what it would do: moves, retires and refusals.++ *++ * npm run screenshots:private -- --limit 50 # a dry run of 50++ * npm run screenshots:private -- --apply # move every one (batches of --limit, 500 by default)++ * npm run screenshots:private -- --apply --keep-old # move, but leave the old objects (an operator's call)++ * npm run screenshots:private -- --retire-ids ids.txt # dry run of the ids only++ * npm run screenshots:private -- --apply --retire-ids ids.txt++ *++ * --retire-ids retires only the Media ids in the file, not the paste rows: for an object whose paste++ * record was lost (the three from the 2026-11-06 pilot). One Media id per line; blank lines and `#`++ * comments are ignored, and a line with anything else (the pilot note's "old … new …" lines) is++ * ignored rather than guessed at, so a prose note cannot name an object that must stay.** Idempotent: a paste whose metadata says media_visibility "private" is skipped, except that a-- * recorded previous_media_ref is deleted (or retried) on every run.++ * recorded previous_media_ref is deleted (or retired) on every run. A second retire of the same++ * object is Media's no-op.*/++const fs = require('fs');const { createMediaObjects } = require('../server/media/objects');const config = require('../server/config');@@@ -37,23 +56,65 @@ function oldObjectId(row) {return m ? m[1] : null;}--async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis.fetch, log = console, limit = 50, apply = false, keepOld = false } = {}) {-- const rows = await db.prepare(`SELECT id, slug, owner_subject, screenshot_url, media_ref, metadata FROM pastes-- WHERE type = 'screenshot' AND deleted_at IS NULL AND screenshot_url IS NOT NULL-- ORDER BY id LIMIT ?`).all(limit);-- const out = { seen: 0, moved: 0, deleted: 0, deleteFailed: 0, notDeletable: 0, skipped: 0, failed: 0, failures: [] };++/** The Media ids named in a --retire-ids file: exactly one id per line, deduplicated in order. */++function readRetireIds(file) {++ const ids = [];++ for (const line of String(fs.readFileSync(file, 'utf8')).split(/\r?\n/)) {++ const text = line.replace(/#.*$/, '').trim();++ if (MEDIA_RE.test(text) && !ids.includes(text)) ids.push(text);++ }++ return ids;++}++++/**++ * Move every unprivate screenshot paste, or retire the ids in --retire-ids, or (a dry run) say what++ * would happen. Returns the counts printed by main(); failures never clear previous_media_ref.++ */++async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis.fetch, log = console, limit = 50, apply = false, keepOld = false, retireIds = null } = {}) {++ const pageSize = Math.max(1, parseInt(limit, 10) || 50);++ const out = {++ seen: 0, moved: 0, deleted: 0, retired: 0, alreadyRetired: 0, deleteFailed: 0, retireFailed: 0,++ refused: 0, notDeletable: 0, skipped: 0, failed: 0, batches: 0, dryRun: !apply,++ plan: { move: 0, retire: 0, refuse: 0 }, failures: [],++ };++ const refuse = (row, id, why) => {++ out.notDeletable++;++ out.failures.push({ slug: row && row.slug, object: id, error: why });++ log.warn(`${row ? row.slug : id}: ${why}; the reference is kept and retried next run`);++ };++++ /**++ * Delete Community's own object, or retire a legacy Live paste screenshot on the 404. A 409 or++ * any other answer is a refusal/failure, never "gone". previous_media_ref was written first.++ */const removeOld = async (row, meta, id) => {-- // previous_media_ref is written before the delete: a crash or a refusal leaves a retryable record.const withRef = { ...meta, previous_media_ref: id };await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE id = ?').run(JSON.stringify(withRef), row.id);try {-- await mediaObjects.deleteObject(id);++ try {++ await mediaObjects.deleteObject(id);++ out.deleted++;++ log.log(`deleted old object ${id} of ${row.slug}`);++ } catch (err) {++ if (err.status !== 404) {++ if (err.status === 409 || err.status === 403) out.refused++;++ throw err;++ }++ try {++ const r = await mediaObjects.retireLegacyPaste(id);++ out.retired++;++ if (r && r.changed === false) out.alreadyRetired++;++ log.log(`retired legacy old object ${id} of ${row.slug}`);++ } catch (retireErr) {++ out.retireFailed++;++ if (retireErr.status === 409 || retireErr.status === 403 || retireErr.status === 404) out.refused++;++ throw retireErr;++ }++ }const cleared = { ...withRef };delete cleared.previous_media_ref;await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE id = ?').run(JSON.stringify(cleared), row.id);-- out.deleted++;-- log.log(`deleted old object ${id} of ${row.slug}`);return true;} catch (err) {out.deleteFailed++;@@@ -63,20 +124,40 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis}};-- for (const row of rows) {++ const planRef = (row, id) => {++ if (!MEDIA_RE.test(String(id))) {++ out.plan.refuse++;++ refuse(row, id, 'the old screenshot is not a Media v2 object id');++ return;++ }++ out.plan.retire++;++ log.log(`would delete/retire old object ${id} of ${row.slug}`);++ };++++ const processRow = async (row) => {out.seen++;const meta = parseMeta(row.metadata);const old = oldObjectId(row);// Already private: only a pending old-object delete remains.if (meta.media_visibility === 'private') {if (meta.previous_media_ref && !keepOld) {-- if (!apply) { out.skipped++; continue; }-- if (!MEDIA_RE.test(String(meta.previous_media_ref))) { out.notDeletable++; continue; }++ if (!apply) { planRef(row, String(meta.previous_media_ref)); return; }++ if (!MEDIA_RE.test(String(meta.previous_media_ref))) {++ refuse(row, String(meta.previous_media_ref), 'the recorded old screenshot is not a Media v2 object id');++ return;++ }await removeOld(row, meta, String(meta.previous_media_ref));} else out.skipped++;-- continue;++ return;++ }++ if (!apply) {++ out.plan.move++;++ log.log(`would move ${row.slug} → private (old object ${old || 'not a Media v2 object'})`);++ if (keepOld) out.skipped++;++ else if (old) planRef(row, old);++ else { out.plan.refuse++; refuse(row, null, 'the old screenshot is not a Media v2 object'); }++ return;}-- if (!apply) { out.skipped++; continue; }try {const res = await fetchImpl(row.screenshot_url, { signal: AbortSignal.timeout(30_000) });if (!res.ok) throw new Error(`read ${res.status}`);@@@ -95,13 +176,52 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThisif (!keepOld) {if (old) await removeOld(row, next, old);// A legacy:community:file object has no v2 id to delete; it is left in place and said so.-- else { out.notDeletable++; log.warn(`${row.slug}: the old screenshot is not a Media v2 object; left in place`); }++ else refuse(row, null, 'the old screenshot is not a Media v2 object');}} catch (err) {out.failed++;out.failures.push({ slug: row.slug, error: err.message });log.warn(`could not move ${row.slug}: ${err.message}`);}++ };++++ // A lost paste record: retire exactly the named objects, no paste scan.++ if (retireIds) {++ for (const id of retireIds) {++ out.seen++;++ if (!MEDIA_RE.test(String(id))) {++ out.refused++;++ refuse(null, id, 'not a Media object id');++ continue;++ }++ if (!apply) { out.plan.retire++; log.log(`would retire ${id}`); continue; }++ try {++ const r = await mediaObjects.retireLegacyPaste(id);++ out.retired++;++ if (r && r.changed === false) out.alreadyRetired++;++ log.log(`${r && r.changed === false ? 'already retired' : 'retired'} ${id}`);++ } catch (err) {++ out.retireFailed++;++ if (err.status === 409 || err.status === 403 || err.status === 404) out.refused++;++ out.failures.push({ object: id, error: `retire: ${err.message}` });++ log.warn(`could not retire ${id}: ${err.message}`);++ }++ }++ return out;++ }++++ // Page with an id cursor: the old `ORDER BY id LIMIT` re-read the same first batch on every run.++ let cursor = 0;++ for (;;) {++ const rows = await db.prepare(`SELECT id, slug, owner_subject, screenshot_url, media_ref, metadata FROM pastes++ WHERE type = 'screenshot' AND deleted_at IS NULL AND screenshot_url IS NOT NULL AND id > ?++ ORDER BY id LIMIT ?`).all(cursor, pageSize);++ out.batches++;++ if (!rows.length) break;++ cursor = rows[rows.length - 1].id;++ for (const row of rows) await processRow(row);++ if (!apply) break; // a dry run reports one batch++ if (rows.length < pageSize) break;}return out;}@@@ -109,16 +229,24 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThisasync function main(argv = process.argv.slice(2)) {const apply = argv.includes('--apply');const keepOld = argv.includes('--keep-old');-- const i = argv.indexOf('--limit');-- const limit = i >= 0 ? Math.max(1, parseInt(argv[i + 1], 10) || 50) : (apply ? 500 : 50);++ const li = argv.indexOf('--limit');++ const limit = li >= 0 ? Math.max(1, parseInt(argv[li + 1], 10) || 50) : (apply ? 500 : 50);++ const ri = argv.indexOf('--retire-ids');++ let retireIds = null;++ if (ri >= 0) {++ const file = argv[ri + 1];++ if (!file || file.startsWith('--')) throw new Error('--retire-ids needs a file of Media object ids');++ retireIds = readRetireIds(file);++ if (!retireIds.length) throw new Error(`no Media object ids in ${file}`);++ }const { openDb } = require('../server/db');const db = await openDb(config);try {const mediaObjects = createMediaObjects({ config });if (!mediaObjects.configured) throw new Error('Media needs Community\'s service principal (OV_OAUTH_CLIENT_SECRET)');-- const out = await moveScreenshotsPrivate({ db, mediaObjects, limit, apply, keepOld });++ const out = await moveScreenshotsPrivate({ db, mediaObjects, limit, apply, keepOld, retireIds });console.log(JSON.stringify(out, null, 2));-- if (!apply) console.log('dry run: pass --apply to move them');++ if (!apply) console.log(retireIds ? 'dry run: pass --apply to retire these ids' : 'dry run: pass --apply to move them');} finally {await db.close();}@@@ -126,4 +254,4 @@ async function main(argv = process.argv.slice(2)) {if (require.main === module) main().catch((err) => { console.error(err.message || err); process.exitCode = 1; });--module.exports = { moveScreenshotsPrivate, oldObjectId };++module.exports = { moveScreenshotsPrivate, oldObjectId, readRetireIds };@@@ -204,8 +204,9 @@ async function createApp(opts = {}) {},/*** The screenshot follows a transfer: re-host the same bytes privately, owned by the new owner,-- * then delete the old object. previous_media_ref is recorded before the delete, so a failure-- * is retried by scripts/move-screenshots-private.js instead of leaving the old URL readable.++ * then delete the old object (retire it when it is a legacy Live paste screenshot).++ * previous_media_ref is recorded before the delete, so a failure is retried by++ * scripts/move-screenshots-private.js instead of leaving the old URL readable.*/async moveOwner(row, owner) {if (!owner || !mediaObjects.configured) return false;@@@ -220,7 +221,13 @@ async function createApp(opts = {}) {.run(stored.url, stored.id, JSON.stringify(next), row.slug);if (old) {try {-- await mediaObjects.deleteObject(old);++ try {++ await mediaObjects.deleteObject(old);++ } catch (err) {++ // A 404 is a legacy Live paste screenshot, not "already gone": retire it.++ if (err.status !== 404) throw err;++ await mediaObjects.retireLegacyPaste(old);++ }delete next.previous_media_ref;await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE slug = ?').run(JSON.stringify(next), row.slug);} catch (err) { console.warn(`[Sharing] old screenshot ${old} not deleted (the move script retries): ${err.message}`); }@@@ -9,6 +9,9 @@* POST /api/v2/community/objects init { kind, visibility: 'unlisted', size_bytes, mime_type, filename, content_hash }* PUT /api/v2/community/objects/:id/content the bytes* POST /api/v2/community/objects/:id/complete Media checks size, hash and type → ready++ * DELETE /api/v2/community/objects/:id soft delete (this tenant only; a legacy Live paste answers 404)++ * POST /internal/community/legacy-paste-screenshots/:id/retire++ * the legacy:live:paste:<slug> screenshot in Media's live tenant** Public bytes are served at <media>/o/:id.*/@@@ -27,20 +30,24 @@ function createMediaObjects({ config, fetchImpl = globalThis.fetch } = {}) {}) : null;const base = `${config.mediaInternalUrl}/api/v2/community/objects`;-- async function call(method, path, { json, body, headers = {} } = {}, retried = false) {++ async function request(method, url, { json, body, headers = {} } = {}, retried = false) {const h = { Accept: 'application/json', ...headers, ...(await tokens.authHeaders()) };if (json !== undefined) h['Content-Type'] = 'application/json';-- const res = await fetchImpl(`${base}${path}`, { method, headers: h, body: json !== undefined ? JSON.stringify(json) : body, signal: AbortSignal.timeout(60_000) });-- if (res.status === 401 && !retried) { tokens.invalidate(); return await call(method, path, { json, body, headers }, true); }++ const res = await fetchImpl(url, { method, headers: h, body: json !== undefined ? JSON.stringify(json) : body, signal: AbortSignal.timeout(60_000) });++ if (res.status === 401 && !retried) { tokens.invalidate(); return await request(method, url, { json, body, headers }, true); }const data = await res.json().catch(() => null);if (!res.ok) {-- const err = new Error(`Media ${method} ${path || '/'} ${res.status}: ${(data && (data.detail || data.error)) || 'failed'}`);++ const err = new Error(`Media ${method} ${url.replace(/^https?:\/\/[^/]+/, '')} ${res.status}: ${(data && (data.detail || data.error)) || 'failed'}`);err.status = res.status; err.code = data && data.code;throw err;}return data || {};}++ async function call(method, path, opts = {}) {++ return await request(method, `${base}${path}`, opts);++ }++/*** Store one image. → { id (med_…), url, size_bytes, mime }* owner: the person (usr_…) the object belongs to, when there is one; kind: 'file' (post images) or@@@ -77,20 +84,29 @@ function createMediaObjects({ config, fetchImpl = globalThis.fetch } = {}) {/*** Soft-delete one object from Media (a screenshot's old unlisted copy after it moved private).-- * 404 counts as gone: the outcome the caller wants. Needs media.object.delete in the tenant.++ * Needs media.object.delete in the community tenant. A 404 is NOT "already gone": Media answers++ * it for an object outside the caller's tenant (a legacy Live paste, media.object.legacy_managed++ * on the direct route), and the move script must keep the reference and retire that object++ * through retireLegacyPaste instead.*/-- async function deleteObject(id, retried = false) {-- try {-- await call('DELETE', `/${encodeURIComponent(String(id))}`);-- return { deleted: true };-- } catch (err) {-- if (err.status === 404) return { deleted: true, alreadyGone: true };-- if (err.status === 401 && !retried) { tokens.invalidate(); return await deleteObject(id, true); }-- throw err;-- }++ async function deleteObject(id) {++ await call('DELETE', `/${encodeURIComponent(String(id))}`);++ return { deleted: true };++ }++++ /**++ * Retire one legacy Live paste screenshot through Media's internal route (the object is in the++ * live tenant, so deleteObject answers 404 and the v2 delete would refuse it). Only the exact++ * legacy:live:paste:<slug> screenshot kind is accepted by Media, only for Community's service++ * principal; a repeat is a no-op (changed:false). Needs media.object.delete.++ */++ async function retireLegacyPaste(id) {++ if (!configured) { const e = new Error('Retiring a Media object needs Community\'s service principal (OV_OAUTH_CLIENT_SECRET)'); e.status = 503; throw e; }++ const out = await request('POST', `${config.mediaInternalUrl}/internal/community/legacy-paste-screenshots/${encodeURIComponent(String(id))}/retire`);++ return { retired: true, changed: !!(out && out.changed), ...(out && out.retention_until ? { retention_until: out.retention_until } : {}) };}-- return { configured, uploadImage, downloadUrl, deleteObject };++ return { configured, uploadImage, downloadUrl, deleteObject, retireLegacyPaste };}module.exports = { createMediaObjects };nnew file mode 100644@@@ -0,0 +1,162 @@++'use strict';++// The move script and Community's Media client (server/media/objects.js,++// scripts/move-screenshots-private.js): a 404 from the tenant delete is a legacy object to retire,++// never "already gone"; --apply pages every paste with an id cursor; --retire-ids retires lost-record++// objects; a dry run only reports moves, retires and refusals. Fake Media only: no network.++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const http = require('http');++++const { ids } = require('openvibe-contracts');++const { createMediaObjects } = require('../server/media/objects');++const { moveScreenshotsPrivate, readRetireIds } = require('../scripts/move-screenshots-private');++const { testDb } = require('./helpers/db');++++(async () => {++ // ── Community's Media client: 404 and 409 are errors, and the retire route is its own call ──++ {++ const seen = [];++ const server = http.createServer((req, res) => {++ let body = '';++ req.on('data', (c) => { body += c; });++ req.on('end', () => {++ const json = (status, obj) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(obj)); };++ if (req.url === '/oauth/token') return json(200, { access_token: 'tok', token_type: 'Bearer', expires_in: 300 });++ seen.push({ method: req.method, url: req.url, authorization: req.headers.authorization, body });++ if (req.method === 'DELETE') {++ if (req.url.endsWith('/med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3')) return json(404, { code: 'media.object.not_found' });++ if (req.url.endsWith('/med_01J8Z3Q4R5S6T7V8W9X0Y1Z2B4')) return json(409, { code: 'media.object.held' });++ return json(200, { id: 'ok', lifecycle_status: 'deleted' });++ }++ if (req.method === 'POST' && req.url === '/internal/community/legacy-paste-screenshots/med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3/retire') {++ return json(200, { id: 'med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3', lifecycle_status: 'deleted', changed: true, retention_until: '2026-12-06T00:00:00.000Z' });++ }++ return json(404, { error: 'not found' });++ });++ });++ await new Promise((r) => server.listen(0, '127.0.0.1', r));++ const base = `http://127.0.0.1:${server.address().port}`;++ try {++ const mo = createMediaObjects({ config: { oauth: { clientId: 'community', clientSecret: 'x'.repeat(40) }, networkInternalUrl: base, mediaInternalUrl: base, mediaUrl: 'https://openvibe.media' } });++ assert.deepStrictEqual(await mo.deleteObject('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2C5'), { deleted: true });++ await assert.rejects(() => mo.deleteObject('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3'), (e) => e.status === 404, '404 is not gone');++ await assert.rejects(() => mo.deleteObject('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2B4'), (e) => e.status === 409, '409 stays a refusal');++ const retired = await mo.retireLegacyPaste('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3');++ assert.deepStrictEqual([retired.retired, retired.changed], [true, true]);++ const call = seen.find((s) => s.url.includes('/retire'));++ assert.ok(call && call.authorization && call.authorization.startsWith('Bearer '), 'the retire call carries the service token');++ } finally {++ await new Promise((r) => server.close(r));++ }++ console.log('✅ the Media client: success deletes, 404/409 throw, retire uses the internal route');++ }++++ // ── The script: dry run, cursor paging, 404→retire, 409 keeps the reference, --retire-ids ──++ const db = await testDb();++ try {++ const mediaUrl = 'https://openvibe.media';++ const add = async (slug, oldId, meta = '{}') => {++ await db.prepare(`INSERT INTO pastes (slug, owner_subject, type, title, content, visibility, screenshot_url, media_ref, metadata)++ VALUES (?, ?, 'screenshot', ?, '', 'public', ?, ?, ?)`)++ .run(slug, null, slug, oldId ? `${mediaUrl}/o/${oldId}` : `${mediaUrl}/f/legacy-community-file.png`, oldId || 'legacy:community:file:shot', meta);++ };++ const row = async (slug) => await db.prepare('SELECT screenshot_url, media_ref, metadata FROM pastes WHERE slug = ?').get(slug);++++ const old = Object.fromEntries(['ok1', 'legacy', 'held', 'ok2', 'ok3'].map((k) => [k, ids.newId('media')]));++ await add('row-ok-1', old.ok1);++ await add('row-legacy', old.legacy, JSON.stringify({ media_visibility: 'private', previous_media_ref: old.legacy })); // already moved, pending delete++ await add('row-held', old.held, JSON.stringify({ media_visibility: 'private', previous_media_ref: old.held }));++ await add('row-ok-2', old.ok2);++ await add('row-ok-3', old.ok3);++ await add('row-no-media-id', null); // legacy community file: no v2 id to remove++++ const fake = {++ uploads: 0, deleted: [], retired: [], mode: new Map(), retiredOnce: new Set(),++ async uploadImage() {++ this.uploads++;++ const id = ids.newId('media');++ return { id, url: `${mediaUrl}/o/${id}`, size_bytes: 3, mime: 'image/png' };++ },++ async deleteObject(id) {++ const mode = this.mode.get(id) || 'ok';++ if (mode === '404') { const e = new Error(`Media DELETE 404: not in tenant`); e.status = 404; throw e; }++ if (mode === '409') { const e = new Error(`Media DELETE 409: legacy managed`); e.status = 409; e.code = 'media.object.legacy_managed'; throw e; }++ this.deleted.push(id);++ },++ async retireLegacyPaste(id) {++ if (this.mode.get(id) === 'retire409') { const e = new Error(`Media POST retire 409: not a paste screenshot`); e.status = 409; throw e; }++ this.retired.push(id);++ const changed = !this.retiredOnce.has(id);++ this.retiredOnce.add(id);++ return { retired: true, changed };++ },++ };++ // A legacy Live object answers 404 to Community's delete; the held one is refused.++ fake.mode.set(old.legacy, '404');++ fake.mode.set(old.held, '409');++ const fetchImpl = async () => new Response(Buffer.from('OLD'), { status: 200, headers: { 'content-type': 'image/png' } });++ const logs = [];++ const log = { log: (...a) => logs.push(a.join(' ')), warn: (...a) => logs.push(a.join(' ')) };++++ // Dry run: one batch, plan only — no upload, no delete, no metadata write.++ let out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 10, apply: false });++ assert.deepStrictEqual([out.seen, out.moved], [6, 0], JSON.stringify(out));++ assert.deepStrictEqual(out.plan, { move: 4, retire: 5, refuse: 1 }, JSON.stringify(out.plan));++ assert.strictEqual(fake.uploads, 0, 'a dry run uploads nothing');++ assert.strictEqual(fake.retired.length, 0, 'a dry run retires nothing');++ assert.ok(logs.some((l) => l.startsWith('would move ')) && logs.some((l) => l.startsWith('would delete/retire ')), 'the dry run says what it would do');++ assert.strictEqual(JSON.parse((await row('row-ok-1')).metadata).media_visibility, undefined, 'nothing was written');++ console.log('✅ the dry run plans moves, retires and refusals without changing anything');++++ // Apply with limit 2: 6 rows over more than one batch, the 404 retired, the 409 kept for a retry.++ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: true });++ assert.deepStrictEqual([out.seen, out.moved, out.failed], [6, 4, 0], JSON.stringify(out));++ assert.deepStrictEqual([out.deleted, out.retired, out.deleteFailed, out.refused, out.notDeletable], [3, 1, 1, 1, 1], JSON.stringify(out));++ assert.ok(out.batches > 2, `paged over more than one batch (${out.batches})`);++ assert.strictEqual(fake.uploads, 4, 'only the four not-yet-private rows were re-hosted');++ assert.strictEqual(fake.retired.includes(old.legacy), true, 'the 404 went to the retire route');++ assert.strictEqual(fake.deleted.includes(old.legacy), false);++ const legacyRow = await row('row-legacy');++ assert.strictEqual(JSON.parse(legacyRow.metadata).previous_media_ref, undefined, 'a retired object clears the record');++ const heldRow = await row('row-held');++ assert.strictEqual(JSON.parse(heldRow.metadata).previous_media_ref, old.held, 'a refusal keeps the reference');++ for (const slug of ['row-ok-1', 'row-legacy', 'row-held', 'row-ok-2', 'row-ok-3', 'row-no-media-id']) {++ assert.strictEqual(JSON.parse((await row(slug)).metadata).media_visibility, 'private', `${slug} is private`);++ }++ console.log('✅ apply pages every row: 404→retire, 409 keeps previous_media_ref, no-id row refused');++++ // The next run retries the kept reference, now that Media accepts the delete.++ fake.mode.set(old.held, 'ok');++ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: true });++ assert.deepStrictEqual([out.deleted, out.skipped, out.deleteFailed], [1, 5, 0], JSON.stringify(out));++ assert.strictEqual(JSON.parse((await row('row-held')).metadata).previous_media_ref, undefined, 'the retry cleared it');++ console.log('✅ a later run retries the kept reference and clears it');++++ // --retire-ids: only those ids, no paste scan, dry run first; a repeat is Media's no-op.++ const lost1 = ids.newId('media');++ const lost2 = ids.newId('media');++ const file = path.join(os.tmpdir(), `ov-retire-ids-${process.pid}.txt`);++ fs.writeFileSync(file, `# the 2026-11-06 pilot\n${lost1}\n${lost2}\n-wgXuYn0 old ${lost1} new ${ids.newId('media')}\n# comment\n${lost1}\n`);++ assert.deepStrictEqual(readRetireIds(file), [lost1, lost2], 'one id per line; prose and repeats are ignored');++++ fake.retired.length = 0;++ const uploadsBefore = fake.uploads;++ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: false, retireIds: [lost1, lost2] });++ assert.deepStrictEqual([out.seen, out.plan.retire, fake.retired.length, fake.uploads], [2, 2, 0, uploadsBefore], JSON.stringify(out));++ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: true, retireIds: [lost1, lost2] });++ assert.deepStrictEqual([out.retired, out.retireFailed, out.refused, out.moved], [2, 0, 0, 0], JSON.stringify(out));++ assert.deepStrictEqual(fake.retired, [lost1, lost2], 'no paste rows were scanned or moved');++ assert.strictEqual(fake.uploads, uploadsBefore, 'retire-ids never uploads');++ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, apply: true, retireIds: [lost1] });++ assert.deepStrictEqual([out.retired, out.alreadyRetired, out.retireFailed], [1, 1, 0], JSON.stringify(out));++ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, apply: true, retireIds: ['not-an-id'] });++ assert.deepStrictEqual([out.refused, out.retired], [1, 0], JSON.stringify(out));++ fs.unlinkSync(file);++ console.log('✅ --retire-ids retires only the named ids, dry-runs, and treats a repeat as a no-op');++ } finally {++ await db.close();++ }++ console.log('move screenshots private: all checks passed');++})().catch((err) => { console.error(err); process.exit(1); });
@@ -2,6 +2,11 @@
@@ -2,6 +2,11 @@
## Unreleased
## Unreleased
+- The screenshot move script (2026-11-06) no longer reads a 404 from Media's community tenant as "already gone": for a
+ `legacy:live:paste:<slug>` object it keeps `previous_media_ref` and retires the object through Media's internal legacy-paste
+ route. `--apply` pages with an id cursor and reaches every paste in one run, `--retire-ids <file>` retires named objects
+ whose paste record was lost (one Media id per line), and the dry run prints the moves, retires and refusals it would make.
+
- Sharing reaches pastes (design 2026-11-06, the pilot): `openvibe-sdk` v0.85.0 and `openvibe-shared` v4.114.0, with
- Sharing reaches pastes (design 2026-11-06, the pilot): `openvibe-sdk` v0.85.0 and `openvibe-shared` v4.114.0, with
`openvibe-contracts` v0.291.0. `migrations/0015_sharing.sql` adds openvibe-sdk/share's tables (expand), and
`openvibe-contracts` v0.291.0. `migrations/0015_sharing.sql` adds openvibe-sdk/share's tables (expand), and
`migrations/0016_sharing_backfill.sql` registers every existing paste once (public→public, unlisted→link,
`migrations/0016_sharing_backfill.sql` registers every existing paste once (public→public, unlisted→link,
@@ -13,13 +13,32 @@
@@ -13,13 +13,32 @@
* The old id is recorded as `previous_media_ref` before the delete, so a failed delete is retried on
* The old id is recorded as `previous_media_ref` before the delete, so a failed delete is retried on
* the next run instead of being forgotten.
* the next run instead of being forgotten.
*
*
- * npm run screenshots:private -- --limit 50 # a dry run of 50
- * npm run screenshots:private -- --apply # move every one (batches of 50)
- * npm run screenshots:private -- --apply --keep-old # move, but leave the old objects (an operator's call)
+ * The old object may be Community's own (delete it through the community tenant) or an inherited
+ * Live-era paste screenshot (`legacy:live:paste:<slug>` in Media's live tenant). Media's
+ * tenant-scoped delete answers 404 for the latter and its v2 delete refuses every legacy object, so
+ * the script retires those through Media's internal legacy-paste route. A 404 is therefore a failure
+ * that keeps `previous_media_ref`: it never means "already gone".
+ *
+ * --apply pages through every screenshot paste with an id cursor, in batches of --limit (500 by
+ * default), so one run reaches every row instead of re-reading the first batch forever. A dry run
+ * looks at one batch (50 by default) and prints what it would do: moves, retires and refusals.
+ *
+ * npm run screenshots:private -- --limit 50 # a dry run of 50
+ * npm run screenshots:private -- --apply # move every one (batches of --limit, 500 by default)
+ * npm run screenshots:private -- --apply --keep-old # move, but leave the old objects (an operator's call)
+ * npm run screenshots:private -- --retire-ids ids.txt # dry run of the ids only
+ * npm run screenshots:private -- --apply --retire-ids ids.txt
+ *
+ * --retire-ids retires only the Media ids in the file, not the paste rows: for an object whose paste
+ * record was lost (the three from the 2026-11-06 pilot). One Media id per line; blank lines and `#`
+ * comments are ignored, and a line with anything else (the pilot note's "old … new …" lines) is
+ * ignored rather than guessed at, so a prose note cannot name an object that must stay.
*
*
* Idempotent: a paste whose metadata says media_visibility "private" is skipped, except that a
* Idempotent: a paste whose metadata says media_visibility "private" is skipped, except that a
- * recorded previous_media_ref is deleted (or retried) on every run.
+ * recorded previous_media_ref is deleted (or retired) on every run. A second retire of the same
+ * object is Media's no-op.
*/
*/
+const fs = require('fs');
const { createMediaObjects } = require('../server/media/objects');
const { createMediaObjects } = require('../server/media/objects');
const config = require('../server/config');
const config = require('../server/config');
@@ -37,23 +56,65 @@ function oldObjectId(row) {
@@ -37,23 +56,65 @@ function oldObjectId(row) {
return m ? m[1] : null;
return m ? m[1] : null;
}
}
-async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis.fetch, log = console, limit = 50, apply = false, keepOld = false } = {}) {
- const rows = await db.prepare(`SELECT id, slug, owner_subject, screenshot_url, media_ref, metadata FROM pastes
- WHERE type = 'screenshot' AND deleted_at IS NULL AND screenshot_url IS NOT NULL
- ORDER BY id LIMIT ?`).all(limit);
- const out = { seen: 0, moved: 0, deleted: 0, deleteFailed: 0, notDeletable: 0, skipped: 0, failed: 0, failures: [] };
+/** The Media ids named in a --retire-ids file: exactly one id per line, deduplicated in order. */
+function readRetireIds(file) {
+ const ids = [];
+ for (const line of String(fs.readFileSync(file, 'utf8')).split(/\r?\n/)) {
+ const text = line.replace(/#.*$/, '').trim();
+ if (MEDIA_RE.test(text) && !ids.includes(text)) ids.push(text);
+ }
+ return ids;
+}
+
+/**
+ * Move every unprivate screenshot paste, or retire the ids in --retire-ids, or (a dry run) say what
+ * would happen. Returns the counts printed by main(); failures never clear previous_media_ref.
+ */
+async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis.fetch, log = console, limit = 50, apply = false, keepOld = false, retireIds = null } = {}) {
+ const pageSize = Math.max(1, parseInt(limit, 10) || 50);
+ const out = {
+ seen: 0, moved: 0, deleted: 0, retired: 0, alreadyRetired: 0, deleteFailed: 0, retireFailed: 0,
+ refused: 0, notDeletable: 0, skipped: 0, failed: 0, batches: 0, dryRun: !apply,
+ plan: { move: 0, retire: 0, refuse: 0 }, failures: [],
+ };
+ const refuse = (row, id, why) => {
+ out.notDeletable++;
+ out.failures.push({ slug: row && row.slug, object: id, error: why });
+ log.warn(`${row ? row.slug : id}: ${why}; the reference is kept and retried next run`);
+ };
+
+ /**
+ * Delete Community's own object, or retire a legacy Live paste screenshot on the 404. A 409 or
+ * any other answer is a refusal/failure, never "gone". previous_media_ref was written first.
+ */
const removeOld = async (row, meta, id) => {
const removeOld = async (row, meta, id) => {
- // previous_media_ref is written before the delete: a crash or a refusal leaves a retryable record.
const withRef = { ...meta, previous_media_ref: id };
const withRef = { ...meta, previous_media_ref: id };
await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE id = ?').run(JSON.stringify(withRef), row.id);
await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE id = ?').run(JSON.stringify(withRef), row.id);
try {
try {
- await mediaObjects.deleteObject(id);
+ try {
+ await mediaObjects.deleteObject(id);
+ out.deleted++;
+ log.log(`deleted old object ${id} of ${row.slug}`);
+ } catch (err) {
+ if (err.status !== 404) {
+ if (err.status === 409 || err.status === 403) out.refused++;
+ throw err;
+ }
+ try {
+ const r = await mediaObjects.retireLegacyPaste(id);
+ out.retired++;
+ if (r && r.changed === false) out.alreadyRetired++;
+ log.log(`retired legacy old object ${id} of ${row.slug}`);
+ } catch (retireErr) {
+ out.retireFailed++;
+ if (retireErr.status === 409 || retireErr.status === 403 || retireErr.status === 404) out.refused++;
+ throw retireErr;
+ }
+ }
const cleared = { ...withRef };
const cleared = { ...withRef };
delete cleared.previous_media_ref;
delete cleared.previous_media_ref;
await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE id = ?').run(JSON.stringify(cleared), row.id);
await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE id = ?').run(JSON.stringify(cleared), row.id);
- out.deleted++;
- log.log(`deleted old object ${id} of ${row.slug}`);
return true;
return true;
} catch (err) {
} catch (err) {
out.deleteFailed++;
out.deleteFailed++;
@@ -63,20 +124,40 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis
@@ -63,20 +124,40 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis
}
}
};
};
- for (const row of rows) {
+ const planRef = (row, id) => {
+ if (!MEDIA_RE.test(String(id))) {
+ out.plan.refuse++;
+ refuse(row, id, 'the old screenshot is not a Media v2 object id');
+ return;
+ }
+ out.plan.retire++;
+ log.log(`would delete/retire old object ${id} of ${row.slug}`);
+ };
+
+ const processRow = async (row) => {
out.seen++;
out.seen++;
const meta = parseMeta(row.metadata);
const meta = parseMeta(row.metadata);
const old = oldObjectId(row);
const old = oldObjectId(row);
// Already private: only a pending old-object delete remains.
// Already private: only a pending old-object delete remains.
if (meta.media_visibility === 'private') {
if (meta.media_visibility === 'private') {
if (meta.previous_media_ref && !keepOld) {
if (meta.previous_media_ref && !keepOld) {
- if (!apply) { out.skipped++; continue; }
- if (!MEDIA_RE.test(String(meta.previous_media_ref))) { out.notDeletable++; continue; }
+ if (!apply) { planRef(row, String(meta.previous_media_ref)); return; }
+ if (!MEDIA_RE.test(String(meta.previous_media_ref))) {
+ refuse(row, String(meta.previous_media_ref), 'the recorded old screenshot is not a Media v2 object id');
+ return;
+ }
await removeOld(row, meta, String(meta.previous_media_ref));
await removeOld(row, meta, String(meta.previous_media_ref));
} else out.skipped++;
} else out.skipped++;
- continue;
+ return;
+ }
+ if (!apply) {
+ out.plan.move++;
+ log.log(`would move ${row.slug} → private (old object ${old || 'not a Media v2 object'})`);
+ if (keepOld) out.skipped++;
+ else if (old) planRef(row, old);
+ else { out.plan.refuse++; refuse(row, null, 'the old screenshot is not a Media v2 object'); }
+ return;
}
}
- if (!apply) { out.skipped++; continue; }
try {
try {
const res = await fetchImpl(row.screenshot_url, { signal: AbortSignal.timeout(30_000) });
const res = await fetchImpl(row.screenshot_url, { signal: AbortSignal.timeout(30_000) });
if (!res.ok) throw new Error(`read ${res.status}`);
if (!res.ok) throw new Error(`read ${res.status}`);
@@ -95,13 +176,52 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis
@@ -95,13 +176,52 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis
if (!keepOld) {
if (!keepOld) {
if (old) await removeOld(row, next, old);
if (old) await removeOld(row, next, old);
// A legacy:community:file object has no v2 id to delete; it is left in place and said so.
// A legacy:community:file object has no v2 id to delete; it is left in place and said so.
- else { out.notDeletable++; log.warn(`${row.slug}: the old screenshot is not a Media v2 object; left in place`); }
+ else refuse(row, null, 'the old screenshot is not a Media v2 object');
}
}
} catch (err) {
} catch (err) {
out.failed++;
out.failed++;
out.failures.push({ slug: row.slug, error: err.message });
out.failures.push({ slug: row.slug, error: err.message });
log.warn(`could not move ${row.slug}: ${err.message}`);
log.warn(`could not move ${row.slug}: ${err.message}`);
}
}
+ };
+
+ // A lost paste record: retire exactly the named objects, no paste scan.
+ if (retireIds) {
+ for (const id of retireIds) {
+ out.seen++;
+ if (!MEDIA_RE.test(String(id))) {
+ out.refused++;
+ refuse(null, id, 'not a Media object id');
+ continue;
+ }
+ if (!apply) { out.plan.retire++; log.log(`would retire ${id}`); continue; }
+ try {
+ const r = await mediaObjects.retireLegacyPaste(id);
+ out.retired++;
+ if (r && r.changed === false) out.alreadyRetired++;
+ log.log(`${r && r.changed === false ? 'already retired' : 'retired'} ${id}`);
+ } catch (err) {
+ out.retireFailed++;
+ if (err.status === 409 || err.status === 403 || err.status === 404) out.refused++;
+ out.failures.push({ object: id, error: `retire: ${err.message}` });
+ log.warn(`could not retire ${id}: ${err.message}`);
+ }
+ }
+ return out;
+ }
+
+ // Page with an id cursor: the old `ORDER BY id LIMIT` re-read the same first batch on every run.
+ let cursor = 0;
+ for (;;) {
+ const rows = await db.prepare(`SELECT id, slug, owner_subject, screenshot_url, media_ref, metadata FROM pastes
+ WHERE type = 'screenshot' AND deleted_at IS NULL AND screenshot_url IS NOT NULL AND id > ?
+ ORDER BY id LIMIT ?`).all(cursor, pageSize);
+ out.batches++;
+ if (!rows.length) break;
+ cursor = rows[rows.length - 1].id;
+ for (const row of rows) await processRow(row);
+ if (!apply) break; // a dry run reports one batch
+ if (rows.length < pageSize) break;
}
}
return out;
return out;
}
}
@@ -109,16 +229,24 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis
@@ -109,16 +229,24 @@ async function moveScreenshotsPrivate({ db, mediaObjects, fetchImpl = globalThis
async function main(argv = process.argv.slice(2)) {
async function main(argv = process.argv.slice(2)) {
const apply = argv.includes('--apply');
const apply = argv.includes('--apply');
const keepOld = argv.includes('--keep-old');
const keepOld = argv.includes('--keep-old');
- const i = argv.indexOf('--limit');
- const limit = i >= 0 ? Math.max(1, parseInt(argv[i + 1], 10) || 50) : (apply ? 500 : 50);
+ const li = argv.indexOf('--limit');
+ const limit = li >= 0 ? Math.max(1, parseInt(argv[li + 1], 10) || 50) : (apply ? 500 : 50);
+ const ri = argv.indexOf('--retire-ids');
+ let retireIds = null;
+ if (ri >= 0) {
+ const file = argv[ri + 1];
+ if (!file || file.startsWith('--')) throw new Error('--retire-ids needs a file of Media object ids');
+ retireIds = readRetireIds(file);
+ if (!retireIds.length) throw new Error(`no Media object ids in ${file}`);
+ }
const { openDb } = require('../server/db');
const { openDb } = require('../server/db');
const db = await openDb(config);
const db = await openDb(config);
try {
try {
const mediaObjects = createMediaObjects({ config });
const mediaObjects = createMediaObjects({ config });
if (!mediaObjects.configured) throw new Error('Media needs Community\'s service principal (OV_OAUTH_CLIENT_SECRET)');
if (!mediaObjects.configured) throw new Error('Media needs Community\'s service principal (OV_OAUTH_CLIENT_SECRET)');
- const out = await moveScreenshotsPrivate({ db, mediaObjects, limit, apply, keepOld });
+ const out = await moveScreenshotsPrivate({ db, mediaObjects, limit, apply, keepOld, retireIds });
console.log(JSON.stringify(out, null, 2));
console.log(JSON.stringify(out, null, 2));
- if (!apply) console.log('dry run: pass --apply to move them');
+ if (!apply) console.log(retireIds ? 'dry run: pass --apply to retire these ids' : 'dry run: pass --apply to move them');
} finally {
} finally {
await db.close();
await db.close();
}
}
@@ -126,4 +254,4 @@ async function main(argv = process.argv.slice(2)) {
@@ -126,4 +254,4 @@ async function main(argv = process.argv.slice(2)) {
if (require.main === module) main().catch((err) => { console.error(err.message || err); process.exitCode = 1; });
if (require.main === module) main().catch((err) => { console.error(err.message || err); process.exitCode = 1; });
-module.exports = { moveScreenshotsPrivate, oldObjectId };
+module.exports = { moveScreenshotsPrivate, oldObjectId, readRetireIds };
@@ -204,8 +204,9 @@ async function createApp(opts = {}) {
@@ -204,8 +204,9 @@ async function createApp(opts = {}) {
},
},
/**
/**
* The screenshot follows a transfer: re-host the same bytes privately, owned by the new owner,
* The screenshot follows a transfer: re-host the same bytes privately, owned by the new owner,
- * then delete the old object. previous_media_ref is recorded before the delete, so a failure
- * is retried by scripts/move-screenshots-private.js instead of leaving the old URL readable.
+ * then delete the old object (retire it when it is a legacy Live paste screenshot).
+ * previous_media_ref is recorded before the delete, so a failure is retried by
+ * scripts/move-screenshots-private.js instead of leaving the old URL readable.
*/
*/
async moveOwner(row, owner) {
async moveOwner(row, owner) {
if (!owner || !mediaObjects.configured) return false;
if (!owner || !mediaObjects.configured) return false;
@@ -220,7 +221,13 @@ async function createApp(opts = {}) {
@@ -220,7 +221,13 @@ async function createApp(opts = {}) {
.run(stored.url, stored.id, JSON.stringify(next), row.slug);
.run(stored.url, stored.id, JSON.stringify(next), row.slug);
if (old) {
if (old) {
try {
try {
- await mediaObjects.deleteObject(old);
+ try {
+ await mediaObjects.deleteObject(old);
+ } catch (err) {
+ // A 404 is a legacy Live paste screenshot, not "already gone": retire it.
+ if (err.status !== 404) throw err;
+ await mediaObjects.retireLegacyPaste(old);
+ }
delete next.previous_media_ref;
delete next.previous_media_ref;
await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE slug = ?').run(JSON.stringify(next), row.slug);
await db.prepare('UPDATE pastes SET metadata = ?, updated_at = ov_now() WHERE slug = ?').run(JSON.stringify(next), row.slug);
} catch (err) { console.warn(`[Sharing] old screenshot ${old} not deleted (the move script retries): ${err.message}`); }
} catch (err) { console.warn(`[Sharing] old screenshot ${old} not deleted (the move script retries): ${err.message}`); }
@@ -9,6 +9,9 @@
@@ -9,6 +9,9 @@
* POST /api/v2/community/objects init { kind, visibility: 'unlisted', size_bytes, mime_type, filename, content_hash }
* POST /api/v2/community/objects init { kind, visibility: 'unlisted', size_bytes, mime_type, filename, content_hash }
* PUT /api/v2/community/objects/:id/content the bytes
* PUT /api/v2/community/objects/:id/content the bytes
* POST /api/v2/community/objects/:id/complete Media checks size, hash and type → ready
* POST /api/v2/community/objects/:id/complete Media checks size, hash and type → ready
+ * DELETE /api/v2/community/objects/:id soft delete (this tenant only; a legacy Live paste answers 404)
+ * POST /internal/community/legacy-paste-screenshots/:id/retire
+ * the legacy:live:paste:<slug> screenshot in Media's live tenant
*
*
* Public bytes are served at <media>/o/:id.
* Public bytes are served at <media>/o/:id.
*/
*/
@@ -27,20 +30,24 @@ function createMediaObjects({ config, fetchImpl = globalThis.fetch } = {}) {
@@ -27,20 +30,24 @@ function createMediaObjects({ config, fetchImpl = globalThis.fetch } = {}) {
}) : null;
}) : null;
const base = `${config.mediaInternalUrl}/api/v2/community/objects`;
const base = `${config.mediaInternalUrl}/api/v2/community/objects`;
- async function call(method, path, { json, body, headers = {} } = {}, retried = false) {
+ async function request(method, url, { json, body, headers = {} } = {}, retried = false) {
const h = { Accept: 'application/json', ...headers, ...(await tokens.authHeaders()) };
const h = { Accept: 'application/json', ...headers, ...(await tokens.authHeaders()) };
if (json !== undefined) h['Content-Type'] = 'application/json';
if (json !== undefined) h['Content-Type'] = 'application/json';
- const res = await fetchImpl(`${base}${path}`, { method, headers: h, body: json !== undefined ? JSON.stringify(json) : body, signal: AbortSignal.timeout(60_000) });
- if (res.status === 401 && !retried) { tokens.invalidate(); return await call(method, path, { json, body, headers }, true); }
+ const res = await fetchImpl(url, { method, headers: h, body: json !== undefined ? JSON.stringify(json) : body, signal: AbortSignal.timeout(60_000) });
+ if (res.status === 401 && !retried) { tokens.invalidate(); return await request(method, url, { json, body, headers }, true); }
const data = await res.json().catch(() => null);
const data = await res.json().catch(() => null);
if (!res.ok) {
if (!res.ok) {
- const err = new Error(`Media ${method} ${path || '/'} ${res.status}: ${(data && (data.detail || data.error)) || 'failed'}`);
+ const err = new Error(`Media ${method} ${url.replace(/^https?:\/\/[^/]+/, '')} ${res.status}: ${(data && (data.detail || data.error)) || 'failed'}`);
err.status = res.status; err.code = data && data.code;
err.status = res.status; err.code = data && data.code;
throw err;
throw err;
}
}
return data || {};
return data || {};
}
}
+ async function call(method, path, opts = {}) {
+ return await request(method, `${base}${path}`, opts);
+ }
+
/**
/**
* Store one image. → { id (med_…), url, size_bytes, mime }
* Store one image. → { id (med_…), url, size_bytes, mime }
* owner: the person (usr_…) the object belongs to, when there is one; kind: 'file' (post images) or
* owner: the person (usr_…) the object belongs to, when there is one; kind: 'file' (post images) or
@@ -77,20 +84,29 @@ function createMediaObjects({ config, fetchImpl = globalThis.fetch } = {}) {
@@ -77,20 +84,29 @@ function createMediaObjects({ config, fetchImpl = globalThis.fetch } = {}) {
/**
/**
* Soft-delete one object from Media (a screenshot's old unlisted copy after it moved private).
* Soft-delete one object from Media (a screenshot's old unlisted copy after it moved private).
- * 404 counts as gone: the outcome the caller wants. Needs media.object.delete in the tenant.
+ * Needs media.object.delete in the community tenant. A 404 is NOT "already gone": Media answers
+ * it for an object outside the caller's tenant (a legacy Live paste, media.object.legacy_managed
+ * on the direct route), and the move script must keep the reference and retire that object
+ * through retireLegacyPaste instead.
*/
*/
- async function deleteObject(id, retried = false) {
- try {
- await call('DELETE', `/${encodeURIComponent(String(id))}`);
- return { deleted: true };
- } catch (err) {
- if (err.status === 404) return { deleted: true, alreadyGone: true };
- if (err.status === 401 && !retried) { tokens.invalidate(); return await deleteObject(id, true); }
- throw err;
- }
+ async function deleteObject(id) {
+ await call('DELETE', `/${encodeURIComponent(String(id))}`);
+ return { deleted: true };
+ }
+
+ /**
+ * Retire one legacy Live paste screenshot through Media's internal route (the object is in the
+ * live tenant, so deleteObject answers 404 and the v2 delete would refuse it). Only the exact
+ * legacy:live:paste:<slug> screenshot kind is accepted by Media, only for Community's service
+ * principal; a repeat is a no-op (changed:false). Needs media.object.delete.
+ */
+ async function retireLegacyPaste(id) {
+ if (!configured) { const e = new Error('Retiring a Media object needs Community\'s service principal (OV_OAUTH_CLIENT_SECRET)'); e.status = 503; throw e; }
+ const out = await request('POST', `${config.mediaInternalUrl}/internal/community/legacy-paste-screenshots/${encodeURIComponent(String(id))}/retire`);
+ return { retired: true, changed: !!(out && out.changed), ...(out && out.retention_until ? { retention_until: out.retention_until } : {}) };
}
}
- return { configured, uploadImage, downloadUrl, deleteObject };
+ return { configured, uploadImage, downloadUrl, deleteObject, retireLegacyPaste };
}
}
module.exports = { createMediaObjects };
module.exports = { createMediaObjects };
new file mode 100644
new file mode 100644
@@ -0,0 +1,162 @@
@@ -0,0 +1,162 @@
+'use strict';
+// The move script and Community's Media client (server/media/objects.js,
+// scripts/move-screenshots-private.js): a 404 from the tenant delete is a legacy object to retire,
+// never "already gone"; --apply pages every paste with an id cursor; --retire-ids retires lost-record
+// objects; a dry run only reports moves, retires and refusals. Fake Media only: no network.
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const http = require('http');
+
+const { ids } = require('openvibe-contracts');
+const { createMediaObjects } = require('../server/media/objects');
+const { moveScreenshotsPrivate, readRetireIds } = require('../scripts/move-screenshots-private');
+const { testDb } = require('./helpers/db');
+
+(async () => {
+ // ── Community's Media client: 404 and 409 are errors, and the retire route is its own call ──
+ {
+ const seen = [];
+ const server = http.createServer((req, res) => {
+ let body = '';
+ req.on('data', (c) => { body += c; });
+ req.on('end', () => {
+ const json = (status, obj) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(obj)); };
+ if (req.url === '/oauth/token') return json(200, { access_token: 'tok', token_type: 'Bearer', expires_in: 300 });
+ seen.push({ method: req.method, url: req.url, authorization: req.headers.authorization, body });
+ if (req.method === 'DELETE') {
+ if (req.url.endsWith('/med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3')) return json(404, { code: 'media.object.not_found' });
+ if (req.url.endsWith('/med_01J8Z3Q4R5S6T7V8W9X0Y1Z2B4')) return json(409, { code: 'media.object.held' });
+ return json(200, { id: 'ok', lifecycle_status: 'deleted' });
+ }
+ if (req.method === 'POST' && req.url === '/internal/community/legacy-paste-screenshots/med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3/retire') {
+ return json(200, { id: 'med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3', lifecycle_status: 'deleted', changed: true, retention_until: '2026-12-06T00:00:00.000Z' });
+ }
+ return json(404, { error: 'not found' });
+ });
+ });
+ await new Promise((r) => server.listen(0, '127.0.0.1', r));
+ const base = `http://127.0.0.1:${server.address().port}`;
+ try {
+ const mo = createMediaObjects({ config: { oauth: { clientId: 'community', clientSecret: 'x'.repeat(40) }, networkInternalUrl: base, mediaInternalUrl: base, mediaUrl: 'https://openvibe.media' } });
+ assert.deepStrictEqual(await mo.deleteObject('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2C5'), { deleted: true });
+ await assert.rejects(() => mo.deleteObject('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3'), (e) => e.status === 404, '404 is not gone');
+ await assert.rejects(() => mo.deleteObject('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2B4'), (e) => e.status === 409, '409 stays a refusal');
+ const retired = await mo.retireLegacyPaste('med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3');
+ assert.deepStrictEqual([retired.retired, retired.changed], [true, true]);
+ const call = seen.find((s) => s.url.includes('/retire'));
+ assert.ok(call && call.authorization && call.authorization.startsWith('Bearer '), 'the retire call carries the service token');
+ } finally {
+ await new Promise((r) => server.close(r));
+ }
+ console.log('✅ the Media client: success deletes, 404/409 throw, retire uses the internal route');
+ }
+
+ // ── The script: dry run, cursor paging, 404→retire, 409 keeps the reference, --retire-ids ──
+ const db = await testDb();
+ try {
+ const mediaUrl = 'https://openvibe.media';
+ const add = async (slug, oldId, meta = '{}') => {
+ await db.prepare(`INSERT INTO pastes (slug, owner_subject, type, title, content, visibility, screenshot_url, media_ref, metadata)
+ VALUES (?, ?, 'screenshot', ?, '', 'public', ?, ?, ?)`)
+ .run(slug, null, slug, oldId ? `${mediaUrl}/o/${oldId}` : `${mediaUrl}/f/legacy-community-file.png`, oldId || 'legacy:community:file:shot', meta);
+ };
+ const row = async (slug) => await db.prepare('SELECT screenshot_url, media_ref, metadata FROM pastes WHERE slug = ?').get(slug);
+
+ const old = Object.fromEntries(['ok1', 'legacy', 'held', 'ok2', 'ok3'].map((k) => [k, ids.newId('media')]));
+ await add('row-ok-1', old.ok1);
+ await add('row-legacy', old.legacy, JSON.stringify({ media_visibility: 'private', previous_media_ref: old.legacy })); // already moved, pending delete
+ await add('row-held', old.held, JSON.stringify({ media_visibility: 'private', previous_media_ref: old.held }));
+ await add('row-ok-2', old.ok2);
+ await add('row-ok-3', old.ok3);
+ await add('row-no-media-id', null); // legacy community file: no v2 id to remove
+
+ const fake = {
+ uploads: 0, deleted: [], retired: [], mode: new Map(), retiredOnce: new Set(),
+ async uploadImage() {
+ this.uploads++;
+ const id = ids.newId('media');
+ return { id, url: `${mediaUrl}/o/${id}`, size_bytes: 3, mime: 'image/png' };
+ },
+ async deleteObject(id) {
+ const mode = this.mode.get(id) || 'ok';
+ if (mode === '404') { const e = new Error(`Media DELETE 404: not in tenant`); e.status = 404; throw e; }
+ if (mode === '409') { const e = new Error(`Media DELETE 409: legacy managed`); e.status = 409; e.code = 'media.object.legacy_managed'; throw e; }
+ this.deleted.push(id);
+ },
+ async retireLegacyPaste(id) {
+ if (this.mode.get(id) === 'retire409') { const e = new Error(`Media POST retire 409: not a paste screenshot`); e.status = 409; throw e; }
+ this.retired.push(id);
+ const changed = !this.retiredOnce.has(id);
+ this.retiredOnce.add(id);
+ return { retired: true, changed };
+ },
+ };
+ // A legacy Live object answers 404 to Community's delete; the held one is refused.
+ fake.mode.set(old.legacy, '404');
+ fake.mode.set(old.held, '409');
+ const fetchImpl = async () => new Response(Buffer.from('OLD'), { status: 200, headers: { 'content-type': 'image/png' } });
+ const logs = [];
+ const log = { log: (...a) => logs.push(a.join(' ')), warn: (...a) => logs.push(a.join(' ')) };
+
+ // Dry run: one batch, plan only — no upload, no delete, no metadata write.
+ let out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 10, apply: false });
+ assert.deepStrictEqual([out.seen, out.moved], [6, 0], JSON.stringify(out));
+ assert.deepStrictEqual(out.plan, { move: 4, retire: 5, refuse: 1 }, JSON.stringify(out.plan));
+ assert.strictEqual(fake.uploads, 0, 'a dry run uploads nothing');
+ assert.strictEqual(fake.retired.length, 0, 'a dry run retires nothing');
+ assert.ok(logs.some((l) => l.startsWith('would move ')) && logs.some((l) => l.startsWith('would delete/retire ')), 'the dry run says what it would do');
+ assert.strictEqual(JSON.parse((await row('row-ok-1')).metadata).media_visibility, undefined, 'nothing was written');
+ console.log('✅ the dry run plans moves, retires and refusals without changing anything');
+
+ // Apply with limit 2: 6 rows over more than one batch, the 404 retired, the 409 kept for a retry.
+ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: true });
+ assert.deepStrictEqual([out.seen, out.moved, out.failed], [6, 4, 0], JSON.stringify(out));
+ assert.deepStrictEqual([out.deleted, out.retired, out.deleteFailed, out.refused, out.notDeletable], [3, 1, 1, 1, 1], JSON.stringify(out));
+ assert.ok(out.batches > 2, `paged over more than one batch (${out.batches})`);
+ assert.strictEqual(fake.uploads, 4, 'only the four not-yet-private rows were re-hosted');
+ assert.strictEqual(fake.retired.includes(old.legacy), true, 'the 404 went to the retire route');
+ assert.strictEqual(fake.deleted.includes(old.legacy), false);
+ const legacyRow = await row('row-legacy');
+ assert.strictEqual(JSON.parse(legacyRow.metadata).previous_media_ref, undefined, 'a retired object clears the record');
+ const heldRow = await row('row-held');
+ assert.strictEqual(JSON.parse(heldRow.metadata).previous_media_ref, old.held, 'a refusal keeps the reference');
+ for (const slug of ['row-ok-1', 'row-legacy', 'row-held', 'row-ok-2', 'row-ok-3', 'row-no-media-id']) {
+ assert.strictEqual(JSON.parse((await row(slug)).metadata).media_visibility, 'private', `${slug} is private`);
+ }
+ console.log('✅ apply pages every row: 404→retire, 409 keeps previous_media_ref, no-id row refused');
+
+ // The next run retries the kept reference, now that Media accepts the delete.
+ fake.mode.set(old.held, 'ok');
+ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: true });
+ assert.deepStrictEqual([out.deleted, out.skipped, out.deleteFailed], [1, 5, 0], JSON.stringify(out));
+ assert.strictEqual(JSON.parse((await row('row-held')).metadata).previous_media_ref, undefined, 'the retry cleared it');
+ console.log('✅ a later run retries the kept reference and clears it');
+
+ // --retire-ids: only those ids, no paste scan, dry run first; a repeat is Media's no-op.
+ const lost1 = ids.newId('media');
+ const lost2 = ids.newId('media');
+ const file = path.join(os.tmpdir(), `ov-retire-ids-${process.pid}.txt`);
+ fs.writeFileSync(file, `# the 2026-11-06 pilot\n${lost1}\n${lost2}\n-wgXuYn0 old ${lost1} new ${ids.newId('media')}\n# comment\n${lost1}\n`);
+ assert.deepStrictEqual(readRetireIds(file), [lost1, lost2], 'one id per line; prose and repeats are ignored');
+
+ fake.retired.length = 0;
+ const uploadsBefore = fake.uploads;
+ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: false, retireIds: [lost1, lost2] });
+ assert.deepStrictEqual([out.seen, out.plan.retire, fake.retired.length, fake.uploads], [2, 2, 0, uploadsBefore], JSON.stringify(out));
+ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, limit: 2, apply: true, retireIds: [lost1, lost2] });
+ assert.deepStrictEqual([out.retired, out.retireFailed, out.refused, out.moved], [2, 0, 0, 0], JSON.stringify(out));
+ assert.deepStrictEqual(fake.retired, [lost1, lost2], 'no paste rows were scanned or moved');
+ assert.strictEqual(fake.uploads, uploadsBefore, 'retire-ids never uploads');
+ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, apply: true, retireIds: [lost1] });
+ assert.deepStrictEqual([out.retired, out.alreadyRetired, out.retireFailed], [1, 1, 0], JSON.stringify(out));
+ out = await moveScreenshotsPrivate({ db, mediaObjects: fake, fetchImpl, log, apply: true, retireIds: ['not-an-id'] });
+ assert.deepStrictEqual([out.refused, out.retired], [1, 0], JSON.stringify(out));
+ fs.unlinkSync(file);
+ console.log('✅ --retire-ids retires only the named ids, dry-runs, and treats a repeat as a no-op');
+ } finally {
+ await db.close();
+ }
+ console.log('move screenshots private: all checks passed');
+})().catch((err) => { console.error(err); process.exit(1); });