openvibers/openvibe.network Commits

Network: Media's per-zone usage rollups (media.usage.recorded) count in a project's usage; contracts 0.175.0 (#192)

OP OpenVibers committed 29 days ago · bf4c3f5e9b · 8 files, +28 −13

* Network: Media's per-zone usage rollups (media.usage.recorded) count in a project's usage; contracts 0.175.0

Network adds Media's per-zone rollups (`media.usage.recorded`, contracts 0.175.0) to a project's usage, beside Tools' and Events' rollups.

- **Consumer:** `server/developer/usage.js` takes `media` as a third producer. The zone id is the rollup's `dimension`, so the dashboard's daily rows and totals split by zone. Only a `media` source counts for a `media.*` topic, as for the others.
- **Measured quotas** know what Media reports:
  - `media.object.upload`: `byte_hours`, `requests`;
  - `media.object.read`: `requests`, `egress_bytes`;
  - `media.object.list` and `media.object.delete`: `requests`.

  A quota in another unit says which units the service reports.
- **Pin:** `openvibe-contracts` v0.173.0 → v0.175.0. `compat` finds no breaking changes since 0.173.
- **`test/project-usage.test.js`:** three Media rollups recorded (byte-hours, egress, reads with an error), one from the wrong source ignored, the daily rows by zone, and the project's error total including Media's.

**After deploy:**

```sh
sudo node --env-file=/etc/openvibe/network.env scripts/subscribe-events.js --topic media.usage.recorded
```

(from /opt/openvibe.network)

* Network: Zone holds events.subscription.manage (its subscription to media.usage.recorded)

* Network test: the consumer's topics include media.usage.recorded

Changes

ddiff --git a/package-lock.json b/package-lock.json
iindex 7ee7a689..ed9e807c 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -20,7 +20,7 @@
          "iovalkey": "^0.4.0",
          "jsonwebtoken": "^9.0.0",
          "multer": "^1.4.5-lts.1",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.173.0",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.175.0",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.1.4",
          "pg": "^8.23.0",
@@@ -1320,9 +1320,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.173.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.173.0",
--      "integrity": "sha512-6zHbU9zghIYDd3PPGtgMTsIgxrWwsoBHO/rvuLW1rghr6kRK6VwCmSMZRcjQbcPODPH1XrUAFuLg/T1K4nOCMQ==",
++      "version": "0.175.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.175.0",
++      "integrity": "sha512-AJNkb/g7mMQDi1oGPiKpVVEmk+oGvzJYLsgxCzd7bPkxN7aos3y/VKh7+tAlfnAOKTajpOTAz5cvLXcGJQHiiQ==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex 2a483f5b..9adf6e40 100644
---- a/package.json
++++ b/package.json
@@@ -24,7 +24,7 @@
      "iovalkey": "^0.4.0",
      "jsonwebtoken": "^9.0.0",
      "multer": "^1.4.5-lts.1",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.173.0",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.175.0",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.1.4",
      "pg": "^8.23.0",
ddiff --git a/scripts/subscribe-events.js b/scripts/subscribe-events.js
iindex 6c3f46c1..d9aa802f 100644
---- a/scripts/subscribe-events.js
++++ b/scripts/subscribe-events.js
@@@ -4,7 +4,7 @@
   * Create Network's OpenVibe.Events subscriptions for the notification consumer
   * (server/notifications/events-consumer.js): one per topic in TOPICS (deals.watch.matched,
   * trade.alert.triggered, live.stream.started, the moderation audit topics and the project usage
-- * rollups tools.usage.recorded and events.usage.recorded), all delivering to POST /internal/events.
++ * rollups tools.usage.recorded, events.usage.recorded and media.usage.recorded), all delivering to POST /internal/events.
   * Events names the consumer after the calling service: `network`.
   *
   *   sudo node --env-file=/etc/openvibe/network.env scripts/subscribe-events.js \
ddiff --git a/server/developer/usage.js b/server/developer/usage.js
iindex 6e1a2583..1586a00a 100644
---- a/server/developer/usage.js
++++ b/server/developer/usage.js
@@@ -6,6 +6,8 @@
   *
   *   tools.usage.recorded    a Tools satellite's hour of a project's jobs (unit jobs)
   *   events.usage.recorded   OpenVibe.Events: published events (events) and webhook deliveries (deliveries)
++ *   media.usage.recorded    OpenVibe.Media's S3 surface, per zone (dimension: the zone id; resource: its OVRN):
++ *                           stored bytes (byte_hours), requests by capability, read egress (egress_bytes)
   *
   * The Events consumer (server/notifications/events-consumer.js) calls record() inside its inbox
   * transaction, so a redelivery changes nothing. A rollup is the totals of its key (service, project,
@@@ -27,7 +29,7 @@
   */
  const { validate, capabilities } = require('openvibe-contracts');
  
--const SERVICES = Object.freeze(['tools', 'events']);
++const SERVICES = Object.freeze(['tools', 'events', 'media']);
  const TOPICS = Object.freeze(SERVICES.map((s) => `${s}.usage.recorded`));
  const HOUR_MS = 60 * 60 * 1000;
  const DAY_MS = 24 * HOUR_MS;
@@@ -41,6 +43,8 @@ const RECENT_ERRORS = 50;
  const REPORTED = Object.freeze({
      'tools.job.create': ['jobs'], 'tools.tool.run': ['jobs'],
      'events.app.publish': ['events'], 'events.app.subscribe': ['deliveries'],
++    'media.object.upload': ['byte_hours', 'requests'], 'media.object.read': ['requests', 'egress_bytes'],
++    'media.object.list': ['requests'], 'media.object.delete': ['requests'],
  });
  
  const iso = (ms) => new Date(ms).toISOString();
ddiff --git a/server/identity/principals.js b/server/identity/principals.js
iindex 69ce0fe7..85574a1c 100644
---- a/server/identity/principals.js
++++ b/server/identity/principals.js
@@@ -280,6 +280,8 @@ const DEFAULT_GRANTS = [
      ['zone', 'media.zone.provision', 'openvibe.media', []],
      ['zone', 'media.access_key.manage', 'openvibe.media', []],
      ['zone', 'network.project.member.read', SELF_AUDIENCE, []],
++    // Zone sums Media's per-zone rollups (media.usage.recorded) in its usage view: its own Events subscription.
++    ['zone', 'events.subscription.manage', 'openvibe.events', []],
      // Plan T14: OpenVibe.Run publishes run.job.* and hands each job to a paired Node through Bot's internal jobs API
      // (POST /api/v1/jobs, /jobs/:id/cancel, GET /jobs/:id, capability bot.job.dispatch, service-to-service only).
      ['run', 'events.event.publish', 'openvibe.events', []],
ddiff --git a/server/notifications/events-consumer.js b/server/notifications/events-consumer.js
iindex 9b433d8f..0254cf05 100644
---- a/server/notifications/events-consumer.js
++++ b/server/notifications/events-consumer.js
@@@ -20,7 +20,7 @@
   * A started event older than LIVE_STARTED_MAX_AGE (30 min; a replay, or Events catching up after an
   * outage) announces nothing: the stream may be long over.
   *
-- *   <service>.usage.recorded  a developer project's hourly usage rollup from Tools or Events
++ *   <service>.usage.recorded  a developer project's hourly usage rollup from Tools, Events or Media
   *                           (common.usage-recorded@1): kept per project and day for its dashboard
   *                           (server/developer/usage.js), never anyone's notification.
   *
ddiff --git a/test/events-consumer.test.js b/test/events-consumer.test.js
iindex b4c52391..dd74b029 100644
---- a/test/events-consumer.test.js
++++ b/test/events-consumer.test.js
@@@ -59,7 +59,7 @@ const rows = async (userId) => await db.prepare('SELECT * FROM notifications WHE
      // The subscriptions this consumer is meant for, and nothing else.
      assert.deepStrictEqual(TOPICS, ['deals.watch.matched', 'trade.alert.triggered', 'live.stream.started', 'live.stream.ended', 'chat.moderation.action', 'live.moderation.action', 'community.moderation.action', 'space.moderation.action', 'tips.interaction.moderated', 'billing.staff.action',
          ...['media', 'tools', 'games', 'wiki', 'blog', 'news', 'reviews', 'deals', 'coupons', 'trade', 'services'].map((svc) => `${svc}.moderation.action`),
--        'tools.usage.recorded', 'events.usage.recorded']);
++        'tools.usage.recorded', 'events.usage.recorded', 'media.usage.recorded']);
      assert.deepStrictEqual(topicsFrom([]), TOPICS, 'scripts/subscribe-events.js subscribes every topic');
      assert.deepStrictEqual(topicsFrom(['--topic', 'live.stream.started']), ['live.stream.started']);
      assert.throws(() => topicsFrom(['--topic', 'live.*']));
ddiff --git a/test/project-usage.test.js b/test/project-usage.test.js
iindex f8b8b077..12aae821 100644
---- a/test/project-usage.test.js
++++ b/test/project-usage.test.js
@@@ -82,7 +82,7 @@ const JOB = 'job_01JAB2C3D4E5F6G7H8J9K0MNPR';
          assert.match(r.body.freshness, /No usage/);
  
          // ── The consumer subscribes to both rollups, and records them ──
--        assert.ok(TOPICS.includes('tools.usage.recorded') && TOPICS.includes('events.usage.recorded'));
++        assert.ok(['tools', 'events', 'media'].every((svc) => TOPICS.includes(`${svc}.usage.recorded`)));
          // Rollups are counts, never money: the consumer takes no billing topic and no raw usage sample.
          // billing.staff.action is the one exception: a staff audit record (server/admin/moderation-audit.js), no amounts.
          assert.deepStrictEqual(TOPICS.filter((t) => (t.startsWith('billing.') && t !== 'billing.staff.action') || t.includes('usage-sample')), []);
@@@ -112,6 +112,13 @@ const JOB = 'job_01JAB2C3D4E5F6G7H8J9K0MNPR';
              samples: [{ at: new Date(thisHour + 120000).toISOString(), code: 'events.quota_exceeded', status: 429, trace_id: TRACE }],
          }));
          assert.strictEqual((await consumer.apply(published)).outcome, 'recorded');
++        // Media's per-zone rollups (ADR-031 section 8): the zone is the dimension, its OVRN the resource.
++        const ZONE = `zon_${ids.ulid()}`;
++        const zoneHour = (extra) => hour(thisHour, { resource: `ovrn:zone:${P}:object-zone/${ZONE}`, dimension: ZONE.toLowerCase(), errors: 0, ...extra });
++        assert.strictEqual((await consumer.apply(rollup('media', zoneHour({ capability: 'media.object.upload', unit: 'byte_hours', quantity: 3600000 })))).outcome, 'recorded');
++        assert.strictEqual((await consumer.apply(rollup('media', zoneHour({ capability: 'media.object.read', unit: 'egress_bytes', quantity: 52428800 })))).outcome, 'recorded');
++        assert.strictEqual((await consumer.apply(rollup('media', zoneHour({ capability: 'media.object.read', unit: 'requests', quantity: 12, errors: 1, error_codes: { 'media.s3.no_such_key': 1 } })))).outcome, 'recorded');
++        assert.strictEqual((await consumer.apply(rollup('media', zoneHour({ capability: 'media.object.read', unit: 'requests', quantity: 12 }), { source: 'tools' }))).outcome, 'ignored:source', 'only Media sends media rollups');
  
          // What is not a project's rollup is ignored.
          const other = await api('stranger', 'POST', '', { name: 'Someone else' });
@@@ -147,9 +154,11 @@ const JOB = 'job_01JAB2C3D4E5F6G7H8J9K0MNPR';
          assert.strictEqual(quota('tools.job.create').window_start, `${today}T00:00:00.000Z`);
          assert.deepStrictEqual([quota('events.app.publish').used, quota('events.app.publish').remaining], [null, null]);
          assert.match(quota('events.app.publish').note, /minute window/);
--        assert.strictEqual(quota('media.object.upload').used, null);
--        assert.match(quota('media.object.upload').note, /no service reports/);
--        assert.strictEqual(u.errors.total, 64);
++        assert.strictEqual(quota('media.object.upload').used, null, 'a quota in bytes is not what Media reports (byte_hours, requests)');
++        const zoneRow = (unit) => u.daily.find((d) => d.capability === 'media.object.read' && d.unit === unit && d.day === today && d.dimension === ZONE.toLowerCase());
++        assert.deepStrictEqual([zoneRow('egress_bytes').quantity, zoneRow('requests').quantity, zoneRow('requests').errors], [52428800, 12, 1], 'Media\'s rollups add up per zone');
++        assert.match(quota('media.object.upload').note, /the service reports byte_hours, requests for this capability, not bytes/);
++        assert.strictEqual(u.errors.total, 65, 'the tools, events and media errors');
          assert.deepStrictEqual(u.errors.by_code[0], { service: 'events', capability: 'events.app.publish', code: 'events.quota_exceeded', count: 60 });
          assert.deepStrictEqual(u.errors.recent.map((e) => [e.service, e.code, e.status, e.trace_id, e.ref]), [
              ['events', 'events.quota_exceeded', 429, TRACE, null],