Deduplicate Media objects by owner and hash (#385)

ddiff --git a/migrations/0026_owner_content_hash_index.sql b/migrations/0026_owner_content_hash_index.sql
nnew file mode 100644
iindex 0000000..b03258b
---- /dev/null
++++ b/migrations/0026_owner_content_hash_index.sql
@@@ -0,0 +1,9 @@
++-- phase: expand
++-- Owner-scoped lookup index for duplicate preflight. Existing objects without hashes are filled by object.hash.
++CREATE INDEX idx_media_objects_owner_content_hash
++    ON media_objects (app_id, owner_subject, content_hash, created_at DESC)
++    WHERE lifecycle_status = 'ready' AND owner_subject IS NOT NULL AND content_hash IS NOT NULL;
++
++CREATE INDEX idx_media_objects_user_content_hash
++    ON media_objects (app_id, owner_user_id, content_hash, created_at DESC)
++    WHERE lifecycle_status = 'ready' AND owner_user_id IS NOT NULL AND content_hash IS NOT NULL;
ddiff --git a/server/avatars/ingest.js b/server/avatars/ingest.js
iindex 4a82bdb..1e19f47 100644
---- a/server/avatars/ingest.js
++++ b/server/avatars/ingest.js
@@@ -158,7 +158,7 @@ async function readLocalSource(rawUrl, { subject = null } = {}) {
   * Store one avatar's bytes as a native object; returns its id. Throws (with nothing left behind) when the tenant,
   * the namespace or the quota refuses it.
   */
--async function storeAvatar({ db, pic, subject = null, username = null, sourceHost = null, userId = null }) {
++async function storeAvatar({ db, pic, subject = null, username = null, sourceHost = null, userId = null, keepBoth = false }) {
      const model = require('../objects/model');
      const namespaces = require('../objects/namespaces');
      const invariant = require('../objects/invariant');
@@@ -168,6 +168,15 @@ async function storeAvatar({ db, pic, subject = null, username = null, sourceHos
      const { namespace } = namespaces.resolveName(app, AVATAR_NAMESPACE);
      const bytes = pic.buffer.length;
      const sha256 = crypto.createHash('sha256').update(pic.buffer).digest('hex');
++    if (!keepBoth && (SUBJECT_RE.test(String(subject || '')) || userId != null)) {
++        const owner = [];
++        const params = [AVATAR_APP, sha256];
++        if (SUBJECT_RE.test(String(subject || ''))) { owner.push('owner_subject = ?'); params.push(subject); }
++        if (userId != null) { owner.push('owner_user_id = ?'); params.push(Number(userId)); }
++        const existing = await db.get(`SELECT id FROM media_objects WHERE app_id = ? AND kind = 'avatar' AND content_hash = ?
++            AND lifecycle_status = 'ready' AND (${owner.join(' OR ')}) ORDER BY created_at DESC LIMIT 1`, params);
++        if (existing) return { id: existing.id, duplicate: true };
++    }
      // The namespace row, the quota check and the object commit together, as an API upload's init does.
      const made = await db.getDb().tx(async () => {
          const row = await namespaces.ensure(app, namespace);
@@@ -205,7 +214,7 @@ async function storeAvatar({ db, pic, subject = null, username = null, sourceHos
          throw err;
      }
      await namespaces.reconcileChain(app, namespace).catch(() => {});
--    return obj.id;
++    return { id: obj.id, duplicate: false };
  }
  
  function createIngestHandler({ db, config, log = console }) {
@@@ -232,9 +241,10 @@ function createIngestHandler({ db, config, log = console }) {
              const raw = local && local.buffer ? local.buffer : await safeFetchImage(url);
              const pic = await toAvatar(raw, { shape });
              const sourceHost = (() => { try { return new URL(url).hostname; } catch { return null; } })();
--            const id = await storeAvatar({ db, pic, subject, username, sourceHost });
++            const stored = await storeAvatar({ db, pic, subject, username, sourceHost });
++            const id = stored.id;
              await adopt(id);
--            res.json({ ok: true, object_id: id, url: `${config.publicUrl}/o/${id}`, width: pic.width, height: pic.height, bytes: pic.buffer.length });
++            res.json({ ok: true, object_id: id, duplicate: stored.duplicate, url: `${config.publicUrl}/o/${id}`, width: pic.width, height: pic.height, bytes: pic.buffer.length });
          } catch (err) {
              log.warn('[Avatar ingest]', err.message);
              res.status(422).json({ ok: false, error: /unsupported image|Input buffer|corrupt|VipsJpeg|bad seek/i.test(err.message) ? 'That file is not a picture we can read' : err.message });
ddiff --git a/server/objects/routes.js b/server/objects/routes.js
iindex 3b6604a..efc0028 100644
---- a/server/objects/routes.js
++++ b/server/objects/routes.js
@@@ -368,6 +368,19 @@ router.post('/', upload, limits('media.object.upload', UPLOAD), async (req, res)
          }
          if (b.upload_ttl != null && !(Number(b.upload_ttl) >= 60 && Number(b.upload_ttl) <= 86400)) return problem(res, 400, 'media.object.invalid', 'upload_ttl must be 60-86400 seconds');
          const userId = req.authType === 'user' ? req.userId : (Number.isInteger(Number(b.user_id)) && Number(b.user_id) > 0 ? Number(b.user_id) : null);
++        if (b.content_hash && !['1', 'true'].includes(String(b.keep_both || '').toLowerCase()) && (subject || userId)) {
++            const ownerWhere = [], ownerParams = [];
++            if (subject) { ownerWhere.push('owner_subject = ?'); ownerParams.push(subject); }
++            if (userId) { ownerWhere.push('owner_user_id = ?'); ownerParams.push(userId); }
++            const existing = await db.get(`SELECT * FROM media_objects WHERE app_id = ? AND content_hash = ? AND lifecycle_status = 'ready'
++                AND (${ownerWhere.join(' OR ')}) ORDER BY created_at DESC LIMIT 1`, [req.appId, String(b.content_hash).toLowerCase(), ...ownerParams]);
++            if (existing) {
++                const meta = model.parseJson(existing.metadata, {});
++                const address = `${config.publicUrl}/me/objects/${encodeURIComponent(existing.id)}`;
++                return res.json({ duplicate: true, id: existing.id, object: await model.objectPublic(existing), name: meta.filename || meta.title || existing.id,
++                    date: existing.created_at, address, url: address });
++            }
++        }
          const metadata = { ...(b.metadata || {}) };
          const filename = sanitizeFilename(b.filename);
          if (filename) metadata.filename = filename;
@@@ -673,6 +686,26 @@ router.post('/:id/operator-view', reviewView, async (req, res) => {
      }
  });
  
++// A person (or their own service acting for them) can only ask about their own objects in this tenant.
++router.get('/duplicates', read, limits('media.object.duplicates', { minute: 60, hour: 1200 }), async (req, res) => {
++    const hash = String(req.query.sha256 || '').toLowerCase();
++    if (!/^[a-f0-9]{64}$/.test(hash)) return problem(res, 400, 'media.object.invalid', 'sha256 must be a SHA-256 digest');
++    if (req.authType !== 'user' || !Number.isSafeInteger(Number(req.userId)) || Number(req.userId) < 1) {
++        return problem(res, 401, 'media.object.sign_in_required', 'Sign in to check your own uploads');
++    }
++    try {
++        const obj = await db.get(`SELECT * FROM media_objects WHERE app_id = ? AND owner_user_id = ? AND content_hash = ?
++            AND lifecycle_status = 'ready' ORDER BY created_at DESC LIMIT 1`, [req.appId, Number(req.userId), hash]);
++        if (!obj) return res.json({ duplicate: false });
++        const metadata = (() => { try { return JSON.parse(obj.metadata || '{}'); } catch { return {}; } })();
++        return res.json({ duplicate: true, object: { id: obj.id, name: metadata.filename || metadata.title || obj.id,
++            date: obj.created_at, address: `${config.publicUrl}/me/objects/${encodeURIComponent(obj.id)}`, url: `${config.publicUrl}/me/objects/${encodeURIComponent(obj.id)}` } });
++    } catch (err) {
++        console.error('[Objects] duplicate lookup error:', err.message);
++        return problem(res, 500, 'media.object.duplicates_failed', 'Could not check your uploads');
++    }
++});
++
  router.get('/', list, limits('media.object.list'), async (req, res) => {
      try {
          const q = req.query;
ddiff --git a/test/avatar-guard.test.js b/test/avatar-guard.test.js
iindex 530747a..69418e6 100644
---- a/test/avatar-guard.test.js
++++ b/test/avatar-guard.test.js
@@@ -95,8 +95,10 @@ const sharp = require('sharp');
          const nextShot = await store({ kind: 'screenshot', owner_subject: ANA }, png);
          const replaced = await call('POST', '/internal/avatar-ingest', { url: `${config.publicUrl}/o/${nextShot}`, user_id: 513, subject: ANA, product: 'network', ref: '513' });
          assert.strictEqual(replaced.status, 200, JSON.stringify(replaced.body));
--        assert.ok(!await model.avatarRef(avatarId), "the old reference was released with the new one");
--        assert.strictEqual((await call('DELETE', `/api/v2/live/objects/${avatarId}`)).status, 200);
++        assert.strictEqual(replaced.body.duplicate, true, 'identical avatar bytes reuse the existing object');
++        assert.strictEqual(replaced.body.object_id, avatarId);
++        assert.ok(await model.avatarRef(avatarId), 'the reused avatar remains referenced by the profile');
++        assert.strictEqual((await call('DELETE', `/api/v2/live/objects/${avatarId}`)).status, 409);
  
          // 5) An explicit release lets the picture go too.
          const released = await call('POST', '/internal/avatar-release', { product: 'network', ref: '513' });
ddiff --git a/test/avatar-ingest.test.js b/test/avatar-ingest.test.js
iindex 32c96e8..a6389cc 100644
---- a/test/avatar-ingest.test.js
++++ b/test/avatar-ingest.test.js
@@@ -98,6 +98,8 @@ process.env.MEDIA_PUBLIC_URL = 'https://media.test';
              assert.strictEqual(second.status, 200, JSON.stringify(second.body));
              assert.notStrictEqual(second.body.object_id, first.body.object_id);
              await check(second.body, subject);
++            const sameAvatar = await post({ url: 'https://pics.example/me2-again.png', user_id: 8, subject });
++            assert.deepStrictEqual([sameAvatar.status, sameAvatar.body.duplicate, sameAvatar.body.object_id], [200, true, second.body.object_id]);
              const invalid = await post({ url: 'https://pics.example/me3.png', user_id: 9, subject: 'not-a-subject' });
              assert.strictEqual(invalid.status, 200, JSON.stringify(invalid.body));
              await check(invalid.body, null);
ddiff --git a/test/objects-api.test.js b/test/objects-api.test.js
iindex 56e7681..2f603e8 100644
---- a/test/objects-api.test.js
++++ b/test/objects-api.test.js
@@@ -240,6 +240,18 @@ const crypto = require('crypto');
          assert.ok(!(await call('GET', O, { headers: { 'x-ov-user-id': '8' } })).body.objects.some(o => o.id === sevens || o.id === id), 'no enumeration of others\' private objects');
          assert.ok((await call('GET', O, { headers: { 'x-ov-user-id': '7' } })).body.objects.some(o => o.id === sevens));
          assert.strictEqual((await call('DELETE', `${O}/${pubId}`, { headers: { 'x-ov-user-id': '8' } })).status, 403);
++        const repeated = Buffer.from('my own duplicate fixture');
++        const ownedInit = await call('POST', O, { headers: { 'x-ov-user-id': '7' }, body: { kind: 'file', mime_type: 'text/plain', size_bytes: repeated.length, content_hash: sha(repeated) } });
++        const ownedUrl = local(ownedInit.body.upload.url);
++        await call('PUT', ownedUrl, { bearer: null, raw: repeated });
++        await call('POST', `${O}/${ownedInit.body.id}/complete`, { headers: { 'x-ov-user-id': '7' } });
++        const lookup = await call('GET', `${O}/duplicates?sha256=${sha(repeated)}`, { headers: { 'x-ov-user-id': '7' } });
++        assert.deepStrictEqual([lookup.status, lookup.body.duplicate, lookup.body.object.id], [200, true, ownedInit.body.id]);
++        const duplicateInit = await call('POST', O, { headers: { 'x-ov-user-id': '7' }, body: { kind: 'file', size_bytes: repeated.length, content_hash: sha(repeated) } });
++        assert.deepStrictEqual([duplicateInit.status, duplicateInit.body.duplicate, duplicateInit.body.id], [200, true, ownedInit.body.id]);
++        const otherOwner = await call('GET', `${O}/duplicates?sha256=${sha(repeated)}`, { headers: { 'x-ov-user-id': '8' } });
++        const signedOut = await call('GET', `${O}/duplicates?sha256=${sha(repeated)}`);
++        assert.deepStrictEqual([otherOwner.body.duplicate, signedOut.status], [false, 401], 'a lookup cannot confirm another owner\'s upload');
          console.log('✅ acting users: public + own objects only');
  
          // ── Holds, soft delete, restore ──