openvibers/openvibe.media Commits

Store private Locker ciphertext and erase expired Sends (#357)

OP OpenVibers committed 5 days ago · 9ab4d505dd · 5 files, +101 −5

* Add private Locker storage and retryable immediate erasure

* Erase Locker remote replicas with retryable purge

Changes

ddiff --git a/server/auth.js b/server/auth.js
iindex f29ad01..643eaad 100644
---- a/server/auth.js
++++ b/server/auth.js
@@@ -205,6 +205,7 @@ async function ensureTokenOnlyApps() {
for (const [appId, name, quotaGb, origins] of [
['live', 'OpenVibe.Live', 0, ['https://openvibe.live']], // VODs, clips, thumbnails, avatars (C-70)
['community', 'OpenVibe.Community', 10, []],
++ ['locker', 'OpenVibe.Locker', 0, []], // Private ciphertext; Locker reserves each person's quota.
['tools', 'OpenVibe.Tools', 20, ['https://openvibe.tools']], // Wave 11 job results (TOOLS_JOB_RESULTS=media)
['games', 'OpenVibe.Games', 10, ['https://openvibe.games', 'https://play.openvibe.games']], // Wave 12 map-editor assets
['wiki', 'OpenVibe.Wiki', 10, []], // Wave 16 page attachments
ddiff --git a/server/objects/model.js b/server/objects/model.js
iindex 4db9049..a6ba3d9 100644
---- a/server/objects/model.js
++++ b/server/objects/model.js
@@@ -724,11 +724,25 @@ async function purgeExpired({ retentionDays = config.objects.retentionDays } = {
async function purgeOne(obj) {
const md = parseJson(obj.metadata, {});
if (obj.lifecycle_status !== 'deleted' || obj.legacy_ref || md.purged_at || await isHeld(obj.id)) return false;
-- if ((await listLocations(obj.id)).some(l => l.provider === 'r2')) return false;
++ const locations = await listLocations(obj.id);
++ const lockerErasure = obj.app_id === 'locker' && md.no_restore === true;
++ if (!lockerErasure && locations.some(l => l.provider === 'r2')) return false;
const root = path.resolve(config.objects.path) + path.sep;
-- for (const l of await listLocations(obj.id)) {
-- if (l.provider === 'local' && path.resolve(l.key).startsWith(root)) { try { fs.unlinkSync(l.key); } catch { /* already gone */ } }
-- if (l.provider === 'local') await db.run('DELETE FROM media_locations WHERE id = ?', [l.id]);
++ for (const l of locations) {
++ if (l.provider !== 'local') {
++ if (!lockerErasure) continue;
++ // Locker expiry is erasure, not cache demotion: delete canonical and cached remote copies too.
++ const storage = require('../vod/vod-storage');
++ try {
++ if (!await storage.deleteObject(l.provider, l.key) || await storage.headObject(l.provider, l.key)) return false;
++ } catch { return false; }
++ await db.run('DELETE FROM media_locations WHERE id = ?', [l.id]);
++ continue;
++ }
++ // Retain the location until the bytes are gone: a denied or interrupted delete must be retried.
++ if (!path.resolve(l.key).startsWith(root)) return false;
++ try { fs.unlinkSync(l.key); } catch (e) { if (e.code !== 'ENOENT') return false; }
++ await db.run('DELETE FROM media_locations WHERE id = ?', [l.id]);
}
// Its CMAF segments (F3.1) go with the bytes. A durable copy whose delete failed keeps its row, so this purge
// is not marked done and the next pass retries it rather than losing the key.
ddiff --git a/server/objects/routes.js b/server/objects/routes.js
iindex b565831..3b6604a 100644
---- a/server/objects/routes.js
++++ b/server/objects/routes.js
@@@ -324,6 +324,9 @@ router.post('/', upload, limits('media.object.upload', UPLOAD), async (req, res)
const kind = String(b.kind || 'file');
if (!model.KINDS.includes(kind)) return problem(res, 400, 'media.object.invalid', `kind must be one of ${model.KINDS.join(', ')}`);
const visibility = String(b.visibility || 'private');
++ if (req.appId === 'locker' && (kind !== 'file' || visibility !== 'private' || b.mime_type !== 'application/octet-stream')) {
++ return problem(res, 422, 'media.locker.opaque', 'Locker stores private opaque files only');
++ }
if (!model.VISIBILITIES.includes(visibility)) return problem(res, 400, 'media.object.invalid', 'visibility must be public, unlisted or private');
// The namespace: the tenant's root unless the body names one below it; the caller must hold write there.
const named = namespaces.resolveName(req.appRow, b.namespace);
@@@ -732,6 +735,21 @@ router.delete('/:id', remove, limits('media.object.delete', { minute: 60, hour:
if (!canWrite(req, obj)) return problem(res, 403, 'media.object.forbidden', 'Not your object');
if (obj.legacy_ref) return problem(res, 409, 'media.object.legacy_managed', 'Delete vods, clips and files through their v1 routes; legacy screenshots and avatars are deleted through account deletion');
if (await model.isHeld(obj.id)) return problem(res, 409, 'media.object.held', 'Object is under a retention hold');
++ if (req.query.purge === '1') {
++ if (req.appId !== 'locker' || req.principal?.sub !== 'svc:locker') return problem(res, 403, 'media.locker.forbidden', 'Immediate Locker erasure requires its own service principal');
++ // Sends have no restore window. Retry until every encrypted copy is gone; holds still win.
++ let gone = obj.lifecycle_status === 'deleted' ? obj : await model.softDelete(obj, { by: 'svc:locker' });
++ const md = model.parseJson(gone.metadata, {});
++ if (md.purged_at) return res.json(await model.objectPublic(gone));
++ md.no_restore = true;
++ await model.updateObject(obj.id, { metadata: JSON.stringify(md) });
++ const active = await db.all("SELECT id FROM media_uploads WHERE object_id = ? AND status = 'active'", [obj.id]);
++ for (const session of active) await multipart.abort(session.id);
++ gone = await model.getObject(obj.id);
++ if (!await model.purgeOne(gone)) return problem(res, 409, 'media.locker.purge_pending', 'Encrypted copies still need erasure; retry');
++ await namespaces.reconcileChain(req.appRow, obj.namespace);
++ return res.json(await model.objectPublic(await model.getObject(obj.id)));
++ }
if (obj.lifecycle_status === 'deleted') return res.json(await model.objectPublic(obj));
const gone = await model.softDelete(obj, { by: req.principal ? req.principal.sub : `app:${req.appId}` });
await namespaces.reconcileChain(req.appRow, obj.namespace);
ddiff --git a/test/app-tenants.test.js b/test/app-tenants.test.js
iindex faf241e..498b87e 100644
---- a/test/app-tenants.test.js
++++ b/test/app-tenants.test.js
@@@ -36,7 +36,7 @@ const crypto = require('crypto');
await auth.ensureTokenOnlyApps();
// C-71: every first-party tenant exists without seeding, and one boot created carries no app key. An existing row
// (live, written with a test key just above) is left exactly as it was.
-- for (const id of ['community', 'tools', 'games', 'wiki', 'blog', 'inventory', 'media-hub', 'codes', 'host']) {
++ for (const id of ['community', 'tools', 'games', 'wiki', 'blog', 'inventory', 'media-hub', 'codes', 'host', 'locker']) {
const row = await db.getApp(id);
assert.ok(row, `${id} exists without seeding`);
assert.strictEqual(row.api_key_hash, '', `${id} has no app key`);
ddiff --git a/test/locker-tenant.test.js b/test/locker-tenant.test.js
nnew file mode 100644
iindex 0000000..c16b5ba
---- /dev/null
++++ b/test/locker-tenant.test.js
@@@ -0,0 +1,63 @@
++'use strict';
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const crypto = require('crypto');
++(async () => {
++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'media-locker-'));
++ process.env.OBJECTS_PATH = path.join(dir, 'objects');
++ process.env.MEDIA_PUBLIC_URL = 'https://media.test';
++ process.env.MEDIA_SIGNING_SECRET = 'locker-test-signing';
++ const db = require('../server/db/database');
++ const auth = require('../server/auth');
++ const routes = require('../server/objects/routes');
++ const model = require('../server/objects/model');
++ const { serviceAuth } = require('openvibe-contracts');
++ const express = require('express');
++ const keys = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
++ auth._setNetworkPublicKeyForTests(keys.publicKey); await auth.ensureTokenOnlyApps();
++ const tenant = await db.getApp('locker'); assert.strictEqual(tenant.api_key_hash, ''); assert.strictEqual(tenant.quota_bytes, 0);
++ const now = Math.floor(Date.now() / 1000);
++ const token = (sub = 'svc:locker', ns = ['locker']) => serviceAuth.signServiceToken({ iss: 'https://openvibe.network', sub, actor_type: 'service', aud: ['openvibe.media'],
++ cap: ['media.object.upload', 'media.object.read', 'media.object.delete'], ns, iat: now, exp: now + 300, jti: crypto.randomUUID() }, keys.privateKey);
++ const app = express(); app.put('/api/v2/:app/objects/:id/content', ...routes.contentHandlers); app.use(express.json()); app.use('/api/v2/:app/objects', routes); app.use('/o', routes.publicRouter);
++ const server = app.listen(0, '127.0.0.1'); await new Promise((r) => server.once('listening', r));
++ const base = `http://127.0.0.1:${server.address().port}`;
++ const call = (method, p, body, bearer = token(), raw = false) => fetch(base + p, { method, headers: { Authorization: `Bearer ${bearer}`, 'Content-Type': raw ? 'application/octet-stream' : 'application/json' }, body: body === undefined ? undefined : raw ? body : JSON.stringify(body) });
++ try {
++ const obj = '/api/v2/locker/objects'; const init = { kind: 'file', visibility: 'private', mime_type: 'application/octet-stream', size_bytes: 5 };
++ assert.strictEqual((await call('POST', obj, { ...init, visibility: 'public' })).status, 422);
++ assert.strictEqual((await call('POST', obj, { ...init, mime_type: 'image/png' })).status, 422);
++ assert.strictEqual((await call('POST', obj, init, token('svc:community', ['community']))).status, 403);
++ let r = await call('POST', obj, init); assert.strictEqual(r.status, 201, await r.clone().text()); const { id } = await r.json();
++ assert.strictEqual((await call('PUT', `${obj}/${id}/content`, Buffer.from('OVLK!'), token(), true)).status, 200);
++ assert.strictEqual((await call('POST', `${obj}/${id}/complete`, {})).status, 200);
++ const row = await model.getObject(id); const file = model.objectFilePath(row); assert.ok(fs.existsSync(file));
++ assert.strictEqual((await fetch(`${base}/o/${id}`)).status, 404);
++ const storage = require('../server/vod/vod-storage');
++ const originalDelete = storage.deleteObject, originalHead = storage.headObject;
++ const remote = new Set(['b2', 'r2']); let failRemote = true;
++ for (const provider of remote) await model.upsertLocation(id, { provider, bucket: 'locker-test', key: `objects/locker/${id}`, state: 'present', size_bytes: 5, verified: true });
++ storage.deleteObject = async (provider) => { if (failRemote) return false; remote.delete(provider); return true; };
++ storage.headObject = async (provider) => remote.has(provider) ? { size: 5 } : null;
++ const unlink = fs.unlinkSync;
++ fs.unlinkSync = (name) => { if (name === file) throw Object.assign(new Error('denied'), { code: 'EACCES' }); return unlink(name); };
++ try {
++ assert.strictEqual((await call('DELETE', `${obj}/${id}?purge=1`)).status, 409, 'failed erasure remains pending');
++ assert.ok((await model.listLocations(id)).length, 'the durable location is retained for retry');
++ assert.ok(fs.existsSync(file));
++ } finally { fs.unlinkSync = unlink; }
++ try {
++ assert.strictEqual((await call('DELETE', `${obj}/${id}?purge=1`)).status, 409, 'failed remote erasure stays retryable');
++ assert.ok((await model.listLocations(id)).some((l) => l.provider !== 'local'));
++ failRemote = false;
++ assert.strictEqual((await call('DELETE', `${obj}/${id}?purge=1`)).status, 200);
++ assert.equal(remote.size, 0, 'cached and canonical remote ciphertext copies are erased');
++ } finally { storage.deleteObject = originalDelete; storage.headObject = originalHead; }
++ assert.ok(!fs.existsSync(file), 'expired Send ciphertext is erased, not held through a restore window');
++ assert.strictEqual((await call('DELETE', `${obj}/${id}?purge=1`)).status, 200, 'erasure retries are idempotent');
++ assert.strictEqual((await call('POST', `${obj}/${id}/restore`, {})).status, 410);
++ console.log('locker-tenant: private opaque files, namespace isolation, immediate retryable erasure');
++ } finally { await new Promise((r) => server.close(r)); fs.rmSync(dir, { recursive: true, force: true }); }
++})().catch((e) => { console.error(e); process.exitCode = 1; });