openvibers/openvibe.tours Commits

Place photos: under Media's hourly limit, and a 429 pauses copies (#111)

OP OpenVibers committed 4 days ago · 69a4dbaa8b · 2 files, +22 −2

Media also allows 600 uploads an hour per caller; the backfill reached it and retried 337
times into the limit. Uploads now go one every 6.5 s (about 550 an hour), and a 429 from
Media pauses every copy for 10 minutes (nothing is downloaded meanwhile; the places are
deferred to the next run), logged once.

Changes

ddiff --git a/server/places/photo-cache.js b/server/places/photo-cache.js
iindex 4b7228e..ebee4b8 100644
---- a/server/places/photo-cache.js
++++ b/server/places/photo-cache.js
@@@ -2,8 +2,11 @@
  
  // A checked Commons image is copied once into Media as a public object. The file's Commons page,
  // author and licence stay on the place row; Media serves only the smaller image bytes.
--// Media takes at most 30 uploads a minute from one caller (media.object.upload); one every UPLOAD_GAP_MS stays under it.
--const UPLOAD_GAP_MS = 2_500;
++// Media takes at most 30 uploads a minute and 600 an hour from one caller (media.object.upload, 429 rate_limited).
++// One every UPLOAD_GAP_MS stays under both (about 550 an hour); a 429 pauses every copy for PAUSE_MS.
++const UPLOAD_GAP_MS = 6_500;
++const PAUSE_MS = 10 * 60 * 1000;
++let pausedUntil = 0;
  // One pace for the whole process: the places job and on-demand discovery each make a cache, and together they must
  // stay under Media's limit too.
  let nextUploadAt = 0;
@@@ -29,6 +32,8 @@ function createPhotoCache({ config, s, fetchImpl = globalThis.fetch, log = conso
      }
      async function cache(image) {
          if (!image?.sourceUrl || !image?.url) return image;
++        // Media said "too many" a moment ago: skip (the place is deferred and tried at the next run).
++        if (Date.now() < pausedUntil) return null;
          if (config.drill) return image.provider === 'openverse' ? null : image;
          if (!config.oauth.clientSecret) return null;
          const old = await s.db.maybe('SELECT media_url FROM tours_photo_assets WHERE source_url=$1', [image.sourceUrl]);
@@@ -64,6 +69,11 @@ function createPhotoCache({ config, s, fetchImpl = globalThis.fetch, log = conso
                  ON CONFLICT(source_url) DO NOTHING`, [image.sourceUrl, obj.id, url]);
              return { ...image, url };
          } catch (err) {
++            if (err && (err.status === 429 || /\b429\b|rate_limited/.test(String(err.message)))) {
++                if (Date.now() >= pausedUntil) log.warn(`[Tours photos] Media is rate limiting uploads: copies pause for ${PAUSE_MS / 60000} minutes (${err.message})`);
++                pausedUntil = Date.now() + PAUSE_MS;
++                return null;
++            }
              log.warn(`[Tours photos] Media copy unavailable: ${err.message}`);
              return null;
          }
ddiff --git a/test/photo-cache-widths.test.js b/test/photo-cache-widths.test.js
iindex 657669e..3514349 100644
---- a/test/photo-cache-widths.test.js
++++ b/test/photo-cache-widths.test.js
@@@ -44,5 +44,15 @@ const objectsClient = { upload: async (bytes) => { uploaded.push(bytes.length);
          objectsClient: { upload: async () => { times.push(Date.now()); return { id: 'o', public_url: 'https://openvibe.media/api/v2/tours/objects/o' }; } } });
      await Promise.all([paced.cache({ ...image, sourceUrl: image.sourceUrl + '1' }), paced.cache({ ...image, sourceUrl: image.sourceUrl + '2' })]);
      assert.ok(times.length === 2 && times[1] - times[0] >= 140, 'one upload per gap');
++    // 5. Media says 429: copies pause (the next one downloads nothing) and the pause is logged once.
++    const warned429 = [];
++    let fetched = 0;
++    const limited = createPhotoCache({ config, s: { db }, uploadGapMs: 0, log: { warn: (m) => warned429.push(m) }, fetchImpl: async () => { fetched += 1; return ok(); },
++        objectsClient: { upload: async () => { throw Object.assign(new Error("POST /objects -> 429 rate_limited"), { status: 429 }); } } });
++    assert.strictEqual(await limited.cache({ ...image, sourceUrl: image.sourceUrl + "429a" }), null);
++    const before = fetched;
++    assert.strictEqual(await limited.cache({ ...image, sourceUrl: image.sourceUrl + "429b" }), null);
++    assert.strictEqual(fetched, before, "paused: nothing is downloaded");
++    assert.strictEqual(warned429.filter((m) => /rate limiting uploads/.test(m)).length, 1, "the pause is logged once");
      console.log('photo cache: 960px first, the imageinfo URL as fallback, refusals logged');
  })().catch((e) => { console.error(e); process.exitCode = 1; });