openvibers/openvibe.zone Commits

account-data: what Zone holds about a person, and the erasure of it (#87)

OP OpenVibers committed 14 days ago · c3523d478e · 11 files, +158 −18

Zone kept a person's telemetry with no way to ask for it: the two account topics were never
subscribed and no module answered them. Now Zone answers.

- server/account-data.js: the module. Zone holds no table of its own about a person — the zones
  belong to a project, the usage is counted per zone and hour, the idempotency and control logs
  name the caller, a service — so the table map is empty and the SDK covers the telemetry samples
  that name a person (telemetry.json, then the subject NULL on deletion).
- server/http/events.js: the signed POST /internal/events answers the two account topics next to
  Media's usage rollups.
- server/index.js: the subscription covers them too.
- migrations/0004_account_data.sql: the receipts of the deliveries Zone applied.
- test/account-data.test.js: the export carries the person's samples and nobody else's, the
  deletion takes the subject off them and confirms once, an unsigned delivery is refused, and the
  guard finds no table with a person column that is neither covered nor exempted with a reason.

openvibe-sdk 0.74.0 (the guard).

Changes

ddiff --git a/README.md b/README.md
iindex c2f683d..9b64b51 100644
---- a/README.md
++++ b/README.md
@@@ -122,6 +122,6 @@ AGPL-3.0, see LICENSE.
  
  <!-- versions:start -->
  - openvibe-contracts: v0.222.0
--- openvibe-sdk: v0.73.0
++- openvibe-sdk: v0.74.0
  - openvibe-shared: v4.48.0
  <!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 156238c..240412b 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -8,11 +8,11 @@
    "port": 5050,
    "domain": "openvibe.zone",
    "contracts": "openvibe-contracts v0.222.0",
--  "sdk": "openvibe-sdk v0.73.0",
++  "sdk": "openvibe-sdk v0.74.0",
    "shared": "openvibe-shared v4.48.0",
    "packages": [
      "openvibe-contracts v0.222.0",
--    "openvibe-sdk v0.73.0",
++    "openvibe-sdk v0.74.0",
      "openvibe-shared v4.48.0"
    ],
    "summary": "OpenVibe.Zone — private S3-compatible storage zones for OpenVibe projects (ADR-031 amendment). Zone owns the zone record and its control API; OpenVibe.Media owns the objects, bytes, keys, quotas and the S3 surface at s3.openvibe.media.",
ddiff --git a/migrations/0004_account_data.sql b/migrations/0004_account_data.sql
nnew file mode 100644
iindex 0000000..666e571
---- /dev/null
++++ b/migrations/0004_account_data.sql
@@@ -0,0 +1,13 @@
++-- phase: expand
++-- OpenVibe.Zone: the receipts of the ADR-033 account export and deletion deliveries this service applied
++-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.
++-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.
++
++CREATE TABLE IF NOT EXISTS account_data_events (
++    id TEXT PRIMARY KEY,
++    kind TEXT NOT NULL,
++    subject TEXT NOT NULL,
++    outcome JSONB,
++    applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),
++    sent_at TIMESTAMPTZ
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex c4e1809..eb3e4c7 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -17,7 +17,7 @@
          "helmet": "^8.1.0",
          "iovalkey": "^0.4.0",
          "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
--        "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
++        "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
          "pg": "^8.23.0"
        },
@@@ -1163,9 +1163,9 @@
        }
      },
      "node_modules/openvibe-sdk": {
--      "version": "0.73.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
--      "integrity": "sha512-XClxoVm9M6RA8eXlGgYZ0u6q9n5h79mCXqAAEg9fX21KgcqP69RmG+cX+6wrOf7TaEj7eSBWMxHjiIeIa9N7Dg==",
++      "version": "0.74.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
++      "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==",
        "license": "MIT",
        "bin": {
          "openvibe-download": "bin/openvibe-download.js"
ddiff --git a/package.json b/package.json
iindex 947e2ec..09f51df 100644
---- a/package.json
++++ b/package.json
@@@ -24,7 +24,7 @@
      "helmet": "^8.1.0",
      "iovalkey": "^0.4.0",
      "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
--    "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
++    "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
      "pg": "^8.23.0"
    },
ddiff --git a/server/account-data.js b/server/account-data.js
nnew file mode 100644
iindex 0000000..29d8afa
---- /dev/null
++++ b/server/account-data.js
@@@ -0,0 +1,21 @@
++'use strict';
++/**
++ * Account export and deletion → Zone (ADR-033; openvibe-sdk/account-data). What Zone holds about a person:
++ *
++ * - their telemetry: the requests they made against their zones, one aggregated sample per route and minute.
++ *   The SDK covers that table itself (telemetry.json, then the subject NULL on deletion);
++ *
++ * and nothing else. The zones belong to a project environment, the usage is counted per zone, hour and series,
++ * and the idempotency and control logs name the caller (a service), not a person. So Zone's own table map is
++ * empty, and the guard on every table with a person column has nothing left to find here.
++ */
++const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
++
++// Zone keeps no table of its own about a person: the telemetry samples are the SDK's own (see above).
++const TABLES = [];
++
++function create({ db, log = console } = {}) {
++    return createAccountData({ db, service: 'zone', tables: TABLES, log });
++}
++
++module.exports = { create, TABLES, TOPICS };
ddiff --git a/server/app.js b/server/app.js
iindex 333daca..7c07fb5 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -28,6 +28,8 @@ const { createPrincipal } = require('./http/principal');
  const { createApi } = require('./http/api');
  const { createUsage } = require('./zone/usage');
  const { eventsIntake } = require('./http/events');
++const accountDataLib = require('./account-data');
++const { createNetworkSender } = require('openvibe-sdk/account-data');
  const { createPageRoutes } = require('./http/pages');
  const { createServiceReadiness } = require('./observability');
  const { createCallerLimits } = require('./http/caller-limits');
@@@ -58,7 +60,13 @@ async function createApp(opts = {}) {
      const usage = createUsage({ s, log });
      const zones = createZones({ s, clients, config, log, metering: usage });
      const mediaLimits = opts.mediaLimits || require('./zone/media-limits').createMediaLimits({ config, fetchImpl, log });
--    const ctx = { config, s, keys, sso, principal, log, clients, zones, usage, mediaLimits };
++    // The account export and deletion (ADR-033, server/account-data.js): what Zone holds about a person, and the
++    // erasure of it. The network sender posts Zone's part of an export back to Network with Zone's own token.
++    const accountData = opts.accountData || accountDataLib.create({ db: s.db, log });
++    const accountSend = opts.accountSend !== undefined ? opts.accountSend : (config.oauth.clientSecret ? createNetworkSender({
++        networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl,
++    }) : null);
++    const ctx = { config, s, keys, sso, principal, log, clients, zones, usage, mediaLimits, accountData, accountSend };
  
      const app = express();
      app.disable('x-powered-by');
@@@ -104,7 +112,8 @@ async function createApp(opts = {}) {
  
      // ── OpenVibe.Events → Zone (loopback only: nginx answers 404 for /internal/) ──
      // media.usage.recorded: Media's hourly rollups of each zone, kept for the usage view (server/zone/usage.js).
--    app.post('/internal/events', eventsIntake({ secrets: config.events.secrets, usage, log }));
++    // The two account topics (ADR-033): Zone's part of a person's export, and the erasure of what it keeps.
++    app.post('/internal/events', eventsIntake({ secrets: config.events.secrets, usage, accountData, accountSend, log }));
  
      // ── Machine endpoints ───────────────────────────────────
      app.get('/api/health', (_req, res) => res.json({ status: 'ok', service: 'openvibe-zone', version: VERSION }));
ddiff --git a/server/http/events.js b/server/http/events.js
iindex 3dbe608..24fef81 100644
---- a/server/http/events.js
++++ b/server/http/events.js
@@@ -1,14 +1,16 @@
  'use strict';
  /**
-- * POST /internal/events: OpenVibe.Events delivers Zone's subscriptions here (media.usage.recorded). No body parser runs
-- * before it: the v2 signature covers the raw body. Loopback only: a request that came through a proxy is refused.
-- * 2xx once applied (or ignored with a reason); anything else makes Events retry.
++ * POST /internal/events: OpenVibe.Events delivers Zone's subscriptions here (media.usage.recorded, and the two
++ * account topics — network.account.export_requested and network.account.deleted — for ADR-033). No body parser
++ * runs before it: the v2 signature covers the raw body. Loopback only: a request that came through a proxy is
++ * refused. 2xx once applied (or ignored with a reason); anything else makes Events retry.
   */
  const { parseDelivery } = require('openvibe-sdk/events');
++const { TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');
  
  const LIMIT = 256 * 1024;
  
--function eventsIntake({ secrets = [], usage, log = console, now = () => Date.now() }) {
++function eventsIntake({ secrets = [], usage, accountData = null, accountSend = null, log = console, now = () => Date.now() }) {
      const keys = (secrets || []).filter((x) => typeof x === 'string' && x.length >= 32);
      const reply = (res, status, body) => {
          res.status(status).set('Cache-Control', 'no-store').type(status < 300 ? 'application/json' : 'application/problem+json').send(JSON.stringify(body));
@@@ -30,6 +32,18 @@ function eventsIntake({ secrets = [], usage, log = console, now = () => Date.now
          for (const k of keys) { delivery = parseDelivery(raw, req.headers, k, { requireV2: true, now: now() }); if (delivery) break; }
          if (!delivery || !delivery.event || typeof delivery.event.event_type !== 'string') return problem(res, 401, 'zone.bad_signature', 'X-OpenVibe-Signature-V2 does not verify or is outside the replay window');
          const ev = delivery.event;
++        // The account export and deletion (ADR-033, server/account-data.js): Zone's part of a person's export, and
++        // the erasure of what it keeps. The two account topics are answered here, next to Media's usage rollups.
++        if (ACCOUNT_TOPICS.includes(ev.event_type)) {
++            if (!accountData) return problem(res, 503, 'zone.account_disabled', 'no account-data module is configured');
++            try {
++                const outcome = await accountData.apply(ev, { send: accountSend });
++                return reply(res, 200, { event_id: ev.event_id, outcome });
++            } catch (err) {
++                log.warn(`[account-data] event ${ev.event_id} (${ev.event_type}) failed: ${(err && err.message) || err}`);
++                return problem(res, 503, 'zone.retry', 'not applied; Events will retry');
++            }
++        }
          try {
              const outcome = ev.event_type === usage.TOPIC ? await usage.apply(ev) : 'ignored:type';
              return reply(res, 200, { event_id: ev.event_id, outcome });
ddiff --git a/server/index.js b/server/index.js
iindex 17794bd..de54cdf 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -7,7 +7,7 @@
  const { createApp } = require('./app');
  const { gracefulStop } = require('openvibe-sdk/service');
  const { startTelemetryRetention } = require('openvibe-sdk/telemetry');
--const { startSubscriptions } = require('openvibe-sdk/account-data');
++const { startSubscriptions, TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');
  
  /**
   * The process stop (openvibe-sdk/service): the HTTP drain runs, then the timers clear, the zone job stops, the JWKS
@@@ -38,11 +38,12 @@ async function start() {
      ctx.keys.client.start();
      // The zone job (zone/zones.js tick): pushes what Media has not acknowledged and finishes deletions, every 15 s.
      ctx.zones.start();
--    // Media's per-zone rollups (media.usage.recorded) at OpenVibe.Events, created when missing; off without EVENTS_URL,
--    // ZONE_EVENTS_SECRET or the client secret.
++    // Media's per-zone rollups (media.usage.recorded) at OpenVibe.Events, created when missing, and the two account
++    // topics (ADR-033) for a person's export and deletion; off without EVENTS_URL, ZONE_EVENTS_SECRET or the client
++    // secret.
      const subscriptions = startSubscriptions({
          eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.port}/internal/events`, secret: config.events.secrets[0],
--        topics: [ctx.usage.TOPIC],
++        topics: [ctx.usage.TOPIC, ...ACCOUNT_TOPICS],
          networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
      });
      createLifecycle({ server, ctx, extra: [() => ctx.zones.stop(), () => { if (subscriptions) subscriptions.stop(); }] });
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
nnew file mode 100644
iindex 0000000..0db741e
---- /dev/null
++++ b/test/account-data.test.js
@@@ -0,0 +1,68 @@
++'use strict';
++/**
++ * ADR-033: Zone's part of an account export and deletion, through the signed POST /internal/events route with the
++ * stand-in Network. Ada's telemetry goes in her export (her own samples only), her deletion takes the subject off
++ * them, and Bo's samples stay theirs. The guard fails when a new table with a person column is neither covered nor
++ * exempted with a reason.
++ */
++const assert = require('assert');
++const { uncovered } = require('openvibe-sdk/account-data');
++const { signDeliveryHeaders } = require('openvibe-sdk/events');
++const { boot, check, done } = require('./helpers/boot');
++const { TABLES } = require('../server/account-data');
++
++const SECRET = `whsec_${'fixture'.repeat(6)}`;
++const A = 'usr_01JZ0000000000000000000AAA';
++const B = 'usr_01JZ0000000000000000000BBB';
++const EXP = 'exp_01JZ0000000000000000000EXP';
++const DEL = 'del_01JZ0000000000000000000DEX';
++
++const ev = (id, type, payload) => ({ event_id: id, event_type: type, source: 'network', version: 1, timestamp: new Date().toISOString(), payload });
++
++(async () => {
++    const t = await boot({ env: { ZONE_EVENTS_SECRET: SECRET } });
++    const db = t.ctx.s.db;
++    try {
++        await check('every table that names a person is covered or exempted with a reason', async () => {
++            const missing = await uncovered(db, { tables: TABLES, exempt: { account_data_events: 'the ledger of requests itself: which export and deletion Zone applied' } });
++            assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
++        });
++
++        await check('an export carries the person\'s telemetry samples, and nobody else\'s', async () => {
++            for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {
++                await db.exec("INSERT INTO telemetry_samples (at, service, operation, resource, status, latency_ms, subject) VALUES (now(), 'zone', 'http.request', '/zones', '2xx', 12, $1)", [subject]);
++            }
++            const body = JSON.stringify({ event: ev('evt_01JZ0000000000000000000E01', 'network.account.export_requested', { export_id: EXP, subject: A }), seq: 1 });
++            const res = await t.get('/internal/events', { method: 'POST', body, headers: { 'content-type': 'application/json', ...signDeliveryHeaders(body, SECRET) } });
++            assert.strictEqual(res.status, 200, res.text);
++            assert.strictEqual(res.json().outcome, 'exported');
++            const parts = t.network.state.accountParts;
++            const part = parts.find((p) => p.id === EXP);
++            assert.ok(part, 'the part was posted to Network');
++            const samples = part.body.files.find((f) => f.name === 'telemetry.json');
++            assert.ok(samples, 'telemetry.json is in the part');
++            assert.strictEqual(samples.content.length, 1, JSON.stringify(samples.content));
++            assert.strictEqual(samples.content[0].resource, '/zones');
++            assert.ok(!JSON.stringify(part.body).includes(B), 'nobody else\'s rows');
++        });
++
++        await check('a deletion takes the subject off their samples and confirms once', async () => {
++            const body = JSON.stringify({ event: ev('evt_01JZ0000000000000000000D01', 'network.account.deleted', { deletion_id: DEL, subject: A, aliases: [] }), seq: 1 });
++            const res = await t.get('/internal/events', { method: 'POST', body, headers: { 'content-type': 'application/json', ...signDeliveryHeaders(body, SECRET) } });
++            assert.strictEqual(res.status, 200, res.text);
++            assert.strictEqual(res.json().outcome, 'erased');
++            assert.strictEqual(await db.value('SELECT subject FROM telemetry_samples WHERE subject = $1', [A]), null, 'her samples keep no subject');
++            assert.strictEqual(await db.value('SELECT subject FROM telemetry_samples WHERE subject = $1', [B]), B, 'Bo stays');
++            const confirmations = t.network.state.accountConfirmations;
++            const confirmation = confirmations.find((c) => c.id === DEL);
++            assert.ok(confirmation, 'the confirmation was sent');
++        });
++
++        await check('an unsigned delivery is refused', async () => {
++            const body = JSON.stringify({ event: ev('evt_01JZ0000000000000000000E02', 'network.account.export_requested', { export_id: EXP, subject: A }), seq: 1 });
++            const res = await t.get('/internal/events', { method: 'POST', body, headers: { 'content-type': 'application/json' } });
++            assert.strictEqual(res.status, 401, res.text);
++        });
++    } finally { await t.close(); }
++    done();
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/helpers/mocks.js b/test/helpers/mocks.js
iindex 0eb90b2..f3cda17 100644
---- a/test/helpers/mocks.js
++++ b/test/helpers/mocks.js
@@@ -14,6 +14,9 @@
   *             answers what setUsage() stored for the project, or an empty network.project-usage-result@1.
   *             POST /api/v1/projects/:project/export-tokens as Network mints them (owner/admin only,
   *             sub app:app_<project ULID>, read-only caps, purpose export), recorded in exportTokens.
++ *             POST /internal/account-exports/:id/parts and POST /internal/account-deletions/:id/confirmations
++ *             (ADR-033: what a service posts back for a person's export and deletion), recorded in
++ *             accountParts and accountConfirmations.
   * Every request is recorded in `.requests` so tests can assert what was (not) called.
   */
  const http = require('http');
@@@ -56,6 +59,8 @@ async function startNetwork({ sandboxAudiences = ['openvibe.events', 'openvibe.m
          users: new Map(), byUsername: new Map(), projects: new Map(), members: new Map(), apps: new Map(), creds: new Map(),
          grants: new Map(), quotas: new Map(), usage: new Map(), audit: [], codes: new Map(), refresh: new Map(),
          catalog: null, down: false, requests: [], tokenRequests: [], exportTokens: [], exportTtl: 300,
++        // ADR-033: the account export parts a service posts back, and the deletion confirmations it sends.
++        accountParts: [], accountConfirmations: [],
      };
      let issuer = null;
      let nextUserId = 1;
@@@ -298,6 +303,15 @@ async function startNetwork({ sandboxAudiences = ['openvibe.events', 'openvibe.m
              if (!wanted.length) return json(400, { error: 'invalid_scope', error_description: `no grants for audience ${body.audience}` });
              return json(200, { access_token: mintApp(a, body.audience, wanted), token_type: 'Bearer', expires_in: 300, scope: wanted.join(' ') });
          }
++        // ADR-033: a service posts its part of a person's export here, and confirms a deletion here.
++        if (url.pathname.startsWith('/internal/account-exports/') && url.pathname.endsWith('/parts') && req.method === 'POST') {
++            st.accountParts.push({ id: url.pathname.split('/')[3], body: JSON.parse(raw.toString('utf8') || '{}') });
++            return json(201, { ok: true });
++        }
++        if (url.pathname.startsWith('/internal/account-deletions/') && url.pathname.endsWith('/confirmations') && req.method === 'POST') {
++            st.accountConfirmations.push({ id: url.pathname.split('/')[3], body: JSON.parse(raw.toString('utf8') || '{}') });
++            return json(201, { ok: true });
++        }
          if (url.pathname.startsWith('/api/v1/projects')) {
              const u = userFrom(req);
              if (u === 'expired') return problem(json, 401, 'auth.expired', 'access token expired; refresh it');