Add owner-scoped duplicate checks for Pics (#372)

ddiff --git a/migrations/0030_pics_owner_hash_lookup.sql b/migrations/0030_pics_owner_hash_lookup.sql
nnew file mode 100644
iindex 0000000..597d67a
---- /dev/null
++++ b/migrations/0030_pics_owner_hash_lookup.sql
@@@ -0,0 +1,6 @@
++-- phase: expand
++-- Fast exact-byte lookups stay scoped to the person who owns each picture.
++CREATE INDEX pics_images_owner_source_hash ON pics_images (owner, source_sha256, created_at DESC)
++    WHERE owner IS NOT NULL AND state = 'active';
++CREATE INDEX pics_images_owner_hash ON pics_images (owner, sha256, created_at DESC)
++    WHERE owner IS NOT NULL AND state = 'active';
ddiff --git a/public/js/pics.js b/public/js/pics.js
iindex 518eec3..a80d7da 100644
---- a/public/js/pics.js
++++ b/public/js/pics.js
@@@ -79,32 +79,62 @@
          r.main.appendChild(el('div', { className: 'ovp-result-error', text: text }));
      }
  
--    function send(file, r) {
++    function duplicateChoice(file, r, duplicate) {
++        var object = duplicate.object || duplicate;
++        r.main.appendChild(el('p', { className: 'ovp-result-note', text: 'You already uploaded this on ' + (object.date || 'an earlier date') + '.' }));
++        var actions = el('div', { className: 'ovp-result-links' });
++        actions.appendChild(el('a', { className: 'ovp-btn', href: object.address || object.url, text: 'Open it' }));
++        var use = el('button', { className: 'ovp-btn', type: 'button', text: 'Use that one' });
++        use.addEventListener('click', function () {
++            actions.remove(); r.main.querySelectorAll('.ovp-result-note').forEach(function (n) { n.remove(); });
++            if (duplicate.image && duplicate.image.links) showUploaded(r, duplicate.image); else location.href = object.address || object.url;
++        });
++        actions.appendChild(use);
++        var again = el('button', { className: 'ovp-btn', type: 'button', text: 'Upload again' });
++        again.addEventListener('click', function () { actions.remove(); r.main.querySelectorAll('.ovp-result-note').forEach(function (n) { n.remove(); }); queue.push({ file: file, r: r, keepBoth: true }); pump(); });
++        actions.appendChild(again); r.main.appendChild(actions);
++    }
++
++    async function checkDuplicate(file, r) {
++        if (!window.OVUpload || !window.OVUpload.sha256 || !window.crypto || !window.crypto.subtle) return null;
++        var digest = await window.OVUpload.sha256(file, max || undefined);
++        if (!digest) return null;
++        var response = await fetch('/api/v1/images/duplicate?sha256=' + encodeURIComponent(digest), { credentials: 'same-origin', headers: { Accept: 'application/json' } });
++        if (!response.ok) return null; // Anonymous uploads have no private owner lookup.
++        var body = await response.json();
++        return body && body.duplicate ? body : null;
++    }
++
++    function showUploaded(r, body) {
++        var p = r.main.querySelector('.ovp-progress');
++        if (p) p.remove();
++        var name = r.main.querySelector('.ovp-result-name');
++        name.textContent = '';
++        name.appendChild(el('a', { href: body.links.page, text: body.title || 'Your picture' }));
++        var links = body.links;
++        var box = el('div', { className: 'ovp-result-links' });
++        box.appendChild(copyRow('Direct link (the picture itself)', links.direct || ''));
++        if (links.markdown) box.appendChild(copyRow('Markdown', links.markdown));
++        if (links.bbcode) box.appendChild(copyRow('BBCode (forums)', links.bbcode));
++        if (links.html) box.appendChild(copyRow('HTML', links.html));
++        r.main.appendChild(box);
++        r.main.appendChild(el('p', { className: 'ovp-result-note', text: 'Anyone with the link can open it — no account needed.' }));
++        if (body.delete_url) r.main.appendChild(el('div', { className: 'ovp-result-error', text: 'Keep this delete link, shown only now: ' + body.delete_url }));
++    }
++
++    function send(file, r, keepBoth) {
          return new Promise(function (resolve) {
              var xhr = new XMLHttpRequest();
              xhr.open('POST', '/api/v1/images');
              xhr.setRequestHeader('Content-Type', file.type);
              xhr.setRequestHeader('X-Filename', encodeURIComponent(file.name || 'pasted'));
++            if (keepBoth) xhr.setRequestHeader('X-Keep-Both', '1');
              xhr.upload.onprogress = function (e) { if (e.lengthComputable) r.bar.style.width = Math.round(e.loaded / e.total * 100) + '%'; };
              xhr.onload = function () {
                  var body = null;
                  try { body = JSON.parse(xhr.responseText); } catch (e) { body = null; }
--                if (xhr.status === 201 && body && body.links) {
--                    var p = r.main.querySelector('.ovp-progress');
--                    if (p) p.remove();
--                    var name = r.main.querySelector('.ovp-result-name');
--                    name.textContent = '';
--                    name.appendChild(el('a', { href: body.links.page, text: body.title || 'Your picture' }));
--                    // The links of the new picture, side by side: the file itself, and the snippets to embed it.
--                    var links = body.links;
--                    var box = el('div', { className: 'ovp-result-links' });
--                    box.appendChild(copyRow('Direct link (the picture itself)', links.direct || ''));
--                    if (links.markdown) box.appendChild(copyRow('Markdown', links.markdown));
--                    if (links.bbcode) box.appendChild(copyRow('BBCode (forums)', links.bbcode));
--                    if (links.html) box.appendChild(copyRow('HTML', links.html));
--                    r.main.appendChild(box);
--                    r.main.appendChild(el('p', { className: 'ovp-result-note', text: 'Anyone with the link can open it — no account needed.' }));
--                    if (body.delete_url) r.main.appendChild(el('div', { className: 'ovp-result-error', text: 'Keep this delete link, shown only now: ' + body.delete_url }));
++                if ((xhr.status === 201 || xhr.status === 200) && body && body.links) {
++                    showUploaded(r, body);
                  } else if (xhr.status === 401) {
                      fail(r, 'Sign in with OpenVibe to upload.');
                  } else {
@@@ -121,7 +151,11 @@
          if (busy || !queue.length) return;
          busy = true;
          var job = queue.shift();
--        send(job.file, job.r).then(function () { busy = false; pump(); });
++        if (job.keepBoth) { send(job.file, job.r, true).then(function () { busy = false; pump(); }); return; }
++        checkDuplicate(job.file, job.r).then(function (duplicate) {
++            if (duplicate) { duplicateChoice(job.file, job.r, duplicate); return; }
++            return send(job.file, job.r, false);
++        }).catch(function () { return send(job.file, job.r, false); }).then(function () { busy = false; pump(); });
      }
  
      function add(files) {
ddiff --git a/server/pics/api.js b/server/pics/api.js
iindex 20644b2..b548183 100644
---- a/server/pics/api.js
++++ b/server/pics/api.js
@@@ -78,6 +78,14 @@ function createPicsApi(ctx) {
  
      r.use('/uploads', require('./resumable').resumable(ctx, { person, problem, whoOf }));
  
++    r.get('/images/duplicate', person, async (req, res) => {
++        const hash = String(req.query.sha256 || '').toLowerCase();
++        if (!/^[a-f0-9]{64}$/.test(hash)) return problem(req, res, 400, 'pics.hash_invalid', 'Send a SHA-256 digest.');
++        const row = await store.ownedByHash(s, whoOf(req).requester, hash);
++        return res.json(row ? { duplicate: true, object: { id: row.id, name: row.title || row.slug, date: row.created_at,
++            address: `${cfg.url}/i/${row.slug}`, url: `${cfg.url}/i/${row.slug}` }, image: pics.toWire(row, { owner: true }) } : { duplicate: false });
++    });
++
      r.get('/gallery', async (req, res) => {
          const result = await gallery.browse(s, req.query);
          return res.json({ pictures: result.rows.map((row) => gallery.wire(pics, row)), next: result.next, sort: result.sort });
@@@ -119,7 +127,7 @@ function createPicsApi(ctx) {
      }), async (req, res) => {
          if (req.principal.kind === 'anonymous' && !cfg.anonymous) return problem(req, res, 401, 'pics.sign_in_required', 'Sign in with OpenVibe to upload pictures.');
          if (!/^image\//i.test(String(req.get('content-type') || ''))) return problem(req, res, 415, 'pics.not_an_image', 'Send the picture as the body, with its image/* Content-Type.');
--        const out = await pics.upload(whoOf(req), { buffer: Buffer.isBuffer(req.body) ? req.body : null, name: header(req, 'x-filename', 200) }, { title: header(req, 'x-title', 200) });
++        const out = await pics.upload(whoOf(req), { buffer: Buffer.isBuffer(req.body) ? req.body : null, name: header(req, 'x-filename', 200) }, { title: header(req, 'x-title', 200), keepBoth: req.get('x-keep-both') === '1' });
          if (!out.ok) return problem(req, res, out.status, out.code, out.detail);
          res.set('Location', `/api/v1/images/${out.image.slug}`);
          // The same picture again: 200 with the one the person already has (duplicate: true), nothing stored twice.
ddiff --git a/server/pics/pages.js b/server/pics/pages.js
iindex 1de4e36..bbf6853 100644
---- a/server/pics/pages.js
++++ b/server/pics/pages.js
@@@ -271,7 +271,7 @@ function createPicsRoutes(ctx) {
                  seo.jsonLd.faq(faq),
              ],
              styles: ['kit.css', 'showcase.css'],
--            scripts: ['js/pics.js', 'js/pics-gallery.js'],
++            scripts: ['js/pics.js', 'js/pics-gallery.js', 'ov-upload.js'],
              body: html`${raw(showcase.hero({
                  title: 'Share pictures.', accent: 'Discover something new.',
                  lede: 'Drop a picture, get a link to share or embed anywhere: on a forum, in a chat, on your site. Browse what people chose to publish, or keep yours unlisted. Location and camera details are removed from every file.',
ddiff --git a/server/pics/service.js b/server/pics/service.js
iindex d7fd2f9..0f5e804 100644
---- a/server/pics/service.js
++++ b/server/pics/service.js
@@@ -88,7 +88,7 @@ function createPics({ s, media, config, log = console, events = null, sharing =
       * Upload one picture. who: { requester } (a person) or { anonymous: true, ip }. file: { buffer, name }.
       * → { ok, image, deleteKey? } or { ok: false, status, code, detail }.
       */
--    async function upload(who, file, { title = '', description = '' } = {}) {
++    async function upload(who, file, { title = '', description = '', keepBoth = false } = {}) {
          const anonymous = !who || !who.requester;
          if (anonymous && !cfg.anonymous) return { ok: false, status: 401, code: 'pics.sign_in_required', detail: 'Sign in with OpenVibe to upload pictures.' };
          const buf = file && file.buffer;
@@@ -112,9 +112,10 @@ function createPics({ s, media, config, log = console, events = null, sharing =
          }
          // The same picture again, signed in (its exact bytes, or the copy served): the one the person already has, not a
          // second copy of it (VIS-6). A near-duplicate is stored as asked and named after (`similar`), since an edit may be meant.
++        const same = anonymous ? null : await store.ownedByHash(s, who.requester, prepared.sourceSha256)
++            || await store.ownedByHash(s, who.requester, prepared.sha256);
++        if (same && !keepBoth) return { ok: true, image: await store.imageById(s, same.id), duplicate: true };
          const own = anonymous ? [] : await store.ownHashes(s, who.requester);
--        const same = own.find((r) => r.source_sha256 === prepared.sourceSha256 || r.sha256 === prepared.sha256);
--        if (same) return { ok: true, image: await store.imageById(s, same.id), duplicate: true };
          const similar = distinctive(prepared.dhash) ? own.find((r) => images.distance(r.dhash, prepared.dhash) <= OWN_SIMILAR_DISTANCE) : null;
          await store.countUpload(s, uploader, day());
  
ddiff --git a/server/pics/store.js b/server/pics/store.js
iindex 7e08709..86f8d67 100644
---- a/server/pics/store.js
++++ b/server/pics/store.js
@@@ -42,6 +42,9 @@ async function imagesOf(s, owner, { limit = 48, before = null } = {}) {
  const ownHashes = (s, owner, limit = 2000) => s.db.many(
      `SELECT id, slug, sha256, source_sha256, dhash, created_at FROM pics_images WHERE owner = $1 AND state = 'active' ORDER BY id DESC LIMIT $2`,
      [owner, limit]);
++const ownedByHash = (s, owner, hash) => s.db.maybe(
++    `SELECT id, slug, title, created_at FROM pics_images WHERE owner = $1 AND state = 'active'
++     AND (source_sha256 = $2 OR sha256 = $2) ORDER BY created_at DESC LIMIT 1`, [owner, hash]);
  
  const usageOf = async (s, owner) => {
      const r = await s.db.maybe("SELECT count(*)::int AS n, coalesce(sum(bytes), 0)::bigint AS bytes FROM pics_images WHERE owner = $1 AND state <> 'removed'", [owner]);
@@@ -125,7 +128,7 @@ const removeFromAlbum = (s, albumId, imageId) => s.db.exec('DELETE FROM pics_alb
  const albumsHolding = (s, imageId) => s.db.many('SELECT a.* FROM pics_album_images ai JOIN pics_albums a ON a.id = ai.album_id WHERE ai.image_id = $1 ORDER BY a.title', [imageId]);
  
  module.exports = {
--    imageBySlug, imageById, insertImage, updateImage, deleteImage, imagesOf, ownHashes, usageOf, countUpload, uploadsToday,
++    imageBySlug, imageById, insertImage, updateImage, deleteImage, imagesOf, ownHashes, ownedByHash, usageOf, countUpload, uploadsToday,
      blockedSha, blockedDhashes, block, addReport, reportsFor, staffQueue, staffCounts, idleAnonymous,
      albumBySlug, albumsOf, albumCount, insertAlbum, updateAlbum, deleteAlbum, albumImages, albumImageCount, addToAlbum, removeFromAlbum, albumsHolding,
  };
ddiff --git a/test/pics.test.js b/test/pics.test.js
iindex 7fb41ac..dff9d1b 100644
---- a/test/pics.test.js
++++ b/test/pics.test.js
@@@ -62,7 +62,7 @@ function multipart(fields, file) {
          const lee = t.network.addUser('lee');
          const ana = t.network.addUser('ana');
          const boss = t.network.addUser('boss', { role: 'admin' });
--        const up = (as, buf, { type = 'image/jpeg', title = '' } = {}) => t.get('/api/v1/images', { as, method: 'POST', body: buf, headers: { ...SAME, 'content-type': type, ...(title ? { 'x-title': encodeURIComponent(title) } : {}) } });
++        const up = (as, buf, { type = 'image/jpeg', title = '', keepBoth = false } = {}) => t.get('/api/v1/images', { as, method: 'POST', body: buf, headers: { ...SAME, 'content-type': type, ...(title ? { 'x-title': encodeURIComponent(title) } : {}), ...(keepBoth ? { 'x-keep-both': '1' } : {}) } });
          let slug;
          let direct;
  
@@@ -194,13 +194,20 @@ function multipart(fields, file) {
              assert.strictEqual(again.status, 200, again.text);
              assert.deepStrictEqual([again.json().slug, again.json().duplicate, again.json().title], [slug, true, 'The coast'], 'the picture kim already has, unchanged');
              assert.strictEqual(t.media.objects.size, objects, 'nothing stored twice');
++            const hash = require('crypto').createHash('sha256').update(jpegWithSecrets).digest('hex');
++            const lookup = await t.get(`/api/v1/images/duplicate?sha256=${hash}`, { as: kim, headers: PICS });
++            assert.deepStrictEqual([lookup.status, lookup.json().duplicate, lookup.json().image.slug], [200, true, slug]);
++            const strangerLookup = await t.get(`/api/v1/images/duplicate?sha256=${hash}`, { as: lee, headers: PICS });
++            assert.deepStrictEqual([strangerLookup.status, strangerLookup.json().duplicate], [200, false], 'another person cannot learn of the upload');
++            const keepBoth = await up(kim, jpegWithSecrets, { keepBoth: true, title: 'A second copy' });
++            assert.strictEqual(keepBoth.status, 201, 'the owner can explicitly keep a second copy');
              const smaller = await sharp(jpegWithSecrets).rotate().resize(600).jpeg({ quality: 70 }).toBuffer();
              const near = await up(kim, smaller, { title: 'Smaller' });
              assert.strictEqual(near.status, 201, near.text);
--            assert.strictEqual(near.json().similar_to, `https://openvibe.pics/i/${slug}`, 'named after the one it looks like');
--            const page = await t.get(`/i/${near.json().slug}?new=1&like=${slug}`, { as: kim, headers: PICS });
--            assert.ok(page.text.includes(`It looks like a picture you uploaded before: <a href="/i/${slug}">`), 'the page says so');
--            const stranger = await t.get(`/i/${near.json().slug}?new=1&like=${slug}`, { headers: PICS });
++            assert.strictEqual(near.json().similar_to, `https://openvibe.pics/i/${keepBoth.json().slug}`, 'named after the most recent copy it looks like');
++            const page = await t.get(`/i/${near.json().slug}?new=1&like=${keepBoth.json().slug}`, { as: kim, headers: PICS });
++            assert.ok(page.text.includes(`It looks like a picture you uploaded before: <a href="/i/${keepBoth.json().slug}">`), 'the page says so');
++            const stranger = await t.get(`/i/${near.json().slug}?new=1&like=${keepBoth.json().slug}`, { headers: PICS });
              assert.ok(!stranger.text.includes('It looks like a picture you uploaded before'), 'only to its owner');
              const theirs = await up(lee, jpegWithSecrets, { title: 'Lee too' });
              assert.strictEqual(theirs.status, 201, 'another person\'s copy is theirs');