openvibers/openvibe.media Commits

D11: the view-hash salt leaves media_settings for the sealed config store (#309)

OP OpenVibers committed 8 days ago · bc0578f5f4 · 9 files, +244 −23

* D11: the view-hash salt leaves media_settings for the sealed config store

* D11: blank the legacy salt only when the store holds that exact value

Changes

ddiff --git a/README.md b/README.md
iindex 6284ba0..10d23ba 100644
---- a/README.md
++++ b/README.md
@@@ -45,8 +45,8 @@ without `DATABASE_URL`, development uses an embedded PGlite database in `data/pg
    export and deletion), OpenVibe.Live (`live.lineage.resolve`), OpenVibe.Events (the outbox relay and
    the account and revocation subscriptions)
  - Backblaze B2 and Cloudflare R2 (S3 API) when configured; `ffmpeg`/`ffprobe` on the host
--- `openvibe-contracts` v0.222.0, `openvibe-sdk` v0.67.0 (tokens, events outbox, usage readings, per-actor limits),
--  `openvibe-shared` v4.39.0, pinned by release tarball
++- `openvibe-contracts` v0.288.0, `openvibe-sdk` v0.82.0 (tokens, events outbox, usage readings, per-actor limits),
++  `openvibe-shared` v4.110.0, pinned by release tarball
  
  ## Capabilities
  
@@@ -129,7 +129,7 @@ server/
                           storage.orphans.scan (monthly storage orphan report, service-wide, report only),
                           storage.move.cleanup (the alert for placement moves whose delete keeps failing, service-wide)
    client-ip.js           trust proxy = loopback; req.ip is the only client address
--(openvibe-shared v4.109.2, openvibe-contracts v0.222.0, openvibe-sdk v0.82.0: pinned release tarballs, installed by npm)
++(openvibe-shared v4.110.0, openvibe-contracts v0.288.0, openvibe-sdk v0.82.0: pinned release tarballs, installed by npm)
  scripts/smoke-test.sh    end-to-end smoke test (boots a temp instance)
  scripts/reconcile-objects.js / object-invariant.js / no-good-copy-report.js   object-model operator tools
  scripts/media-jobs.js     list jobs, run the size-invariant scan (dry run by default), approve/cancel proposals
@@@ -755,7 +755,7 @@ Rollback: ovhost puts the previous sha back by itself when `/api/ready` does not
  restart; afterwards `sudo ovhost rollback media --to <sha>`.
  
  <!-- versions:start -->
--- openvibe-contracts: v0.222.0
++- openvibe-contracts: v0.288.0
  - openvibe-sdk: v0.82.0
--- openvibe-shared: v4.109.2
++- openvibe-shared: v4.110.0
  <!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 1810db5..1bdbd8c 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -6,9 +6,9 @@
    "runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, ffmpeg",
    "port": 4100,
    "domain": "openvibe.media",
--  "contracts": "openvibe-contracts v0.222.0",
++  "contracts": "openvibe-contracts v0.288.0",
    "sdk": "openvibe-sdk v0.82.0",
--  "shared": "openvibe-shared v4.109.2",
++  "shared": "openvibe-shared v4.110.0",
    "capabilities": [
      "media.object.upload",
      "media.object.read",
ddiff --git a/package-lock.json b/package-lock.json
iindex ee29a4c..f23080d 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -16,9 +16,9 @@
          "express": "^4.19.2",
          "iovalkey": "^0.4.0",
          "multer": "^1.4.5-lts.1",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.288.0",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.82.0",
--        "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.109.2",
++        "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.110.0",
          "pg": "^8.23.0",
          "sharp": "^0.33.4"
        },
@@@ -2122,9 +2122,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.222.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
--      "integrity": "sha512-wdUQCilq9glRMwfxEswbIj3l0QQqKDXZAi2+CR5Mlh34rZjI1Q+7GlAoR0UtNwZAgtLwXhPxFyJ/aPk8lcDG1Q==",
++      "version": "0.288.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.288.0",
++      "integrity": "sha512-AKRP1yXACYNu7okpf4kCDq4ds0PhsoNvD92VlvBWBCjcSlNimSTQhLWIvSa3D31/kQYz91fodo06ZnwuTanOqg==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
@@@ -2166,9 +2166,9 @@
        }
      },
      "node_modules/openvibe-shared": {
--      "version": "4.109.2",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.109.2",
--      "integrity": "sha512-t9K1yHr6GOJsfamqmgERRTTtiWYWAVmjWBEm4ykTcYjEenbTiqQTwX6CJsZW4lYnqS1IlqEvn+0jG4vXmHSJ3Q==",
++      "version": "4.110.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.110.0",
++      "integrity": "sha512-ZNOSraSgE89jhLTI//lFJALvKCiBJIBwf+V0aCkiIk0VCrPPAytsfsgOgCINU8kXDVdCEnuWWfcA9hZUc54QVA==",
        "license": "MIT",
        "dependencies": {
          "jsonwebtoken": "^9.0.0",
ddiff --git a/package.json b/package.json
iindex c5e0539..af893f5 100644
---- a/package.json
++++ b/package.json
@@@ -20,9 +20,9 @@
      "express": "^4.19.2",
      "iovalkey": "^0.4.0",
      "multer": "^1.4.5-lts.1",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.288.0",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.82.0",
--    "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.109.2",
++    "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.110.0",
      "pg": "^8.23.0",
      "sharp": "^0.33.4"
    },
ddiff --git a/server/admin/site-config.js b/server/admin/site-config.js
nnew file mode 100644
iindex 0000000..72277be
---- /dev/null
++++ b/server/admin/site-config.js
@@@ -0,0 +1,96 @@
++'use strict';
++/**
++ * Media's secret site settings in the revisioned configuration store (D11).
++ *
++ * `media_settings` keeps non-secret settings (storage_tier.*, view cooldowns, job cursors). A secret-class
++ * value copied into it — today the view-hash salt `view_hash_secret` — is moved into the namespace
++ * `media.site_settings`, where openvibe-shared/config seals it when the service has a KEK. The legacy row is
++ * blanked at boot only after the sealed store holds it; without a KEK nothing is blanked and the store reports
++ * sealed:false. Readers go through secretValue() (server/views/service.js) instead of the row.
++ */
++const config = require('openvibe-shared/config');
++const db = require('../db/database');
++
++const SECRET = /(api[_-]?key|secret|token|password|private[_-]?key|salt|access[_-]?key|webhook)/i;
++const KEY = 'view_hash_secret';
++const SYSTEM = { type: 'service', id: 'media' };
++
++const isConfigKey = (key) => key === KEY;
++const classify = (key) => (SECRET.test(key) ? 'secret' : 'internal');
++
++let store = null;
++function getStore() {
++    if (store) return store;
++    store = (async () => {
++        const rows = await db.all('SELECT key, value FROM media_settings WHERE key = ?', [KEY]);
++        const legacy = {};
++        for (const r of rows) if (isConfigKey(r.key)) legacy[r.key] = r.value == null ? '' : String(r.value);
++        return await config.createConfigStore({
++            db: db.getDb(), service: 'media', namespace: 'media.site_settings',
++            classify,
++            legacy: () => legacy,
++            // Nothing non-secret lives in this namespace; a secret is never mirrored into media_settings.
++            onActivate: async () => {},
++            log: { info: (m) => console.log(`[Config] ${m}`), warn: (m) => console.warn(`[Config] ${m}`), error: (m) => console.error(`[Config] ${m}`) },
++        });
++    })();
++    return store;
++}
++
++function hasStore() { return store !== null; }
++async function secretValue(key = KEY) {
++    if (!store) return undefined;
++    return (await store).get(key);
++}
++
++/** Generate the persisting salt through the write-only secret path; an explicit value never touches the row. */
++async function setSecret(value, { reason = 'set view hash salt' } = {}) {
++    return (await getStore()).setSecret(KEY, value, { actor: SYSTEM, reason });
++}
++
++/**
++ * D11: blank the legacy row once the sealed store holds a readable, matching copy. Without a KEK (not sealed)
++ * nothing is blanked; without a store value or with a locked envelope the row is kept. Counts only.
++ */
++async function migrateLegacySecrets(s) {
++    const row = await db.get('SELECT value FROM media_settings WHERE key = ?', [KEY]);
++    if (!row || row.value == null || String(row.value) === '') return { checked: 0, blanked: 0, differs: 0, kept: 0, sealed: !!s.sealed };
++    if (!s.sealed) {
++        console.log('[Config] legacy media_settings: 1 secret value left (no KEK; the store is not sealed)');
++        return { checked: 1, blanked: 0, differs: 0, kept: 1, sealed: false };
++    }
++    const summary = await s.summary();
++    const snap = summary.active;
++    const stored = s.get(KEY);
++    const marker = summary.secrets[KEY] || (typeof s.secretMarker === 'function' ? await s.secretMarker(KEY) : null);
++    const sealedRow = snap && snap.values ? snap.values[KEY] : null;
++    const proven = stored !== undefined && stored !== null && String(stored) !== ''
++        && marker && sealedRow && sealedRow.redacted === true
++        && marker.fingerprint && marker.fingerprint === sealedRow.fingerprint;
++    // Never drop a value the store does not hold: the sealed copy must be exactly this legacy value.
++    if (!proven || String(stored) !== String(row.value)) {
++        console.log('[Config] legacy media_settings: 1 secret value kept (value differs or no sealed copy)');
++        return { checked: 1, blanked: 0, differs: 1, kept: 1, sealed: true };
++    }
++    await db.run('UPDATE media_settings SET value = ?, updated_at = ov_now() WHERE key = ?', ['', KEY]);
++    console.log(`[Config] legacy media_settings: 1 secret value blanked (the store holds the same value); KEK ${s.kek}`);
++    return { checked: 1, blanked: 1, differs: 0, kept: 0, sealed: true };
++}
++
++let initPromise = null;
++/** Boot: load the store, seal old plaintext rows, then blank the legacy copy the sealed store holds. */
++function init({ adopt = true } = {}) {
++    if (initPromise) return initPromise;
++    if (!adopt) {
++        // A restore drill never writes (MEDIA_DRILL): load nothing here; its readers fall back without persisting.
++        return Promise.resolve({ loaded: false, checked: 0, blanked: 0, differs: 0, kept: 0, sealed: false });
++    }
++    initPromise = (async () => {
++        const s = await getStore();
++        if (s.sealed) await s.reseal();
++        return { loaded: true, ...(await migrateLegacySecrets(s)) };
++    })().catch((err) => { initPromise = null; throw err; });
++    return initPromise;
++}
++
++module.exports = { KEY, getStore, init, hasStore, secretValue, setSecret, migrateLegacySecrets, isConfigKey, classify, _reset: () => { store = null; initPromise = null; } };
ddiff --git a/server/index.js b/server/index.js
iindex f4295e4..0014a09 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -48,6 +48,9 @@ const ready = (async () => {
      // PostgreSQL first (migrations run as the owner), then what reads it before the app serves: Network's token cutoffs
      // (isRevoked() is synchronous) and the two revisioned tier policies (their get() reads memory).
      await db.initDb();
++    // D11: load the configuration store, reseal old plaintext under the KEK, then blank the legacy secret copy
++    // it holds. A restore drill writes nothing (MEDIA_DRILL).
++    await require('./admin/site-config').init({ adopt: !drill.enabled });
      // Valkey (ADR-035): per-actor limit counters shared across processes (not in a drill: it connects nowhere).
      const valkey = !drill.enabled && config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix }) : null;
      require('./actor-limits').useValkey(valkey);
ddiff --git a/server/views/service.js b/server/views/service.js
iindex ae9994f..3ffab82 100644
---- a/server/views/service.js
++++ b/server/views/service.js
@@@ -17,6 +17,7 @@
  'use strict';
  const crypto = require('crypto');
  const db = require('../db/database');
++const siteConfig = require('../admin/site-config');
  
  const TABLES = { vod: { table: 'vods', raw: 'view_count' }, clip: { table: 'clips', raw: 'view_count' }, file: { table: 'files', raw: 'view_count' } };
  const BOT_UA = /bot|crawl|spider|slurp|facebookexternalhit|preview|discordbot|twitterbot|whatsapp|telegram|curl\/|wget\/|python-requests|go-http-client|headless/i;
@@@ -30,13 +31,27 @@ async function secret() {
          // Never the value, only the name.
          console.warn('[views] VIEW_HASH_SECRET and MEDIA_SECRET are unset in production; visitor hashes fall back to the persisted per-process secret and past hashes stop matching.');
      }
++    // D11: the salt lives in the sealed configuration store, never in media_settings. The boot has loaded it;
++    // a test that did not boot the server loads it here (the store imports the legacy row once).
      if (!_secret) {
--        // Persist a random secret so visitor hashes stay stable across restarts.
          try {
--            const row = await db.get("SELECT value FROM media_settings WHERE key = 'view_hash_secret'");
--            if (row && row.value) _secret = row.value;
--            else { _secret = crypto.randomBytes(24).toString('hex'); await db.run("INSERT INTO media_settings (key, value, description, type) VALUES ('view_hash_secret', ?, 'Internal: salt for hashed visitor ids (do not share)', 'string') ON CONFLICT (key) DO UPDATE SET value = excluded.value, description = excluded.description, type = excluded.type, updated_at = ov_now()", [_secret]); }
--        } catch { _secret = 'openvibe-media'; }
++            let stored = await siteConfig.secretValue();
++            if (stored === undefined && !require('../drill').enabled) {
++                await siteConfig.getStore();
++                stored = await siteConfig.secretValue();
++            }
++            if (stored) _secret = String(stored);
++        } catch { /* a drill without the store falls through to a per-process salt */ }
++    }
++    if (!_secret) {
++        // Persist a random secret so visitor hashes stay stable across restarts — through the write-only secret
++        // path, never the legacy row. A restore drill writes nothing: its salt is per-process.
++        const fresh = crypto.randomBytes(24).toString('hex');
++        if (require('../drill').enabled) _secret = fresh;
++        else {
++            try { await siteConfig.setSecret(fresh, { reason: 'generated view hash salt' }); _secret = fresh; }
++            catch { _secret = 'openvibe-media'; }
++        }
      }
      return _secret;
  }
@@@ -122,4 +137,4 @@ async function prune(days = 30) {
      try { return (await db.run("DELETE FROM content_visits WHERE last_at < datetime('now', ?)", [`-${days} days`])).changes; } catch { return 0; }
  }
  
--module.exports = { recordView, counts, countsMany, isInitialPlaybackRequest, clientIp, prune, TABLES };
++module.exports = { recordView, counts, countsMany, isInitialPlaybackRequest, clientIp, prune, TABLES, hashSecret: secret };
ddiff --git a/test/legacy-secrets-kek.test.js b/test/legacy-secrets-kek.test.js
nnew file mode 100644
iindex 0000000..a0f567d
---- /dev/null
++++ b/test/legacy-secrets-kek.test.js
@@@ -0,0 +1,72 @@
++'use strict';
++/**
++ * D11: the view-hash salt copied into media_settings is blanked at boot once the sealed config store holds it.
++ * The reader still gets it, a rotated salt goes through the write-only store path, and no summary or log carries
++ * the plaintext.
++ *
++ *   node --import ./test/helpers/pg-preload.mjs test/legacy-secrets-kek.test.js
++ */
++const assert = require('assert');
++process.env.NODE_ENV = 'test';
++process.env.OV_CONFIG_KEKS = `k1:${Buffer.alloc(32, 11).toString('base64')}`;
++process.env.OV_CONFIG_KEK_CURRENT = 'k1';
++delete process.env.VIEW_HASH_SECRET;
++delete process.env.MEDIA_SECRET;
++
++const SECRET = 'sentinel-not-a-secret-view-salt';
++const logs = [];
++const real = { log: console.log, warn: console.warn, error: console.error };
++console.log = (...a) => logs.push(a.map(String).join(' '));
++console.warn = (...a) => logs.push(a.map(String).join(' '));
++console.error = (...a) => logs.push(a.map(String).join(' '));
++
++(async () => {
++    const db = require('../server/db/database');
++    await db.initDb();
++    await db.run("INSERT INTO media_settings (key, value, description, type) VALUES ('view_hash_secret', ?, 'salt', 'string')", [SECRET]);
++
++    const siteConfig = require('../server/admin/site-config');
++    const report = await siteConfig.init();
++    const store = await siteConfig.getStore();
++    assert.strictEqual(store.sealed, true, 'the store is sealed under the KEK');
++    assert.strictEqual(report.blanked, 1);
++    assert.strictEqual(report.differs, 0, 'an equal value is not counted as differing');
++
++    const row = await db.get('SELECT value FROM media_settings WHERE key = ?', ['view_hash_secret']);
++    assert.strictEqual(row.value, '', 'no secret-class value remains in media_settings');
++    assert.strictEqual(await siteConfig.secretValue(), SECRET, 'the store holds the salt');
++    const views = require('../server/views/service');
++    assert.strictEqual(await views.hashSecret(), SECRET, 'the view reader still gets its salt');
++
++    const summary = JSON.stringify(await store.summary());
++    assert.ok(!summary.includes(SECRET), 'the summary carries no value');
++    const stored = await db.get("SELECT values_json FROM config_snapshots WHERE namespace = 'media.site_settings' AND state = 'active'");
++    assert.ok(!stored.values_json.includes(SECRET), 'values_json holds no plaintext');
++    assert.ok(stored.values_json.includes('\"v1:'), 'the secret is stored as an envelope');
++    assert.ok(!logs.join('\n').includes(SECRET), 'no log line carries the value');
++
++    await siteConfig.setSecret('sentinel-rotated', { reason: 'rotate' });
++    assert.strictEqual((await db.get('SELECT value FROM media_settings WHERE key = ?', ['view_hash_secret'])).value, '', 'a rotated salt is not written to the legacy row');
++    assert.strictEqual(await siteConfig.secretValue(), 'sentinel-rotated');
++
++    // A legacy row newer than the store is kept, not blanked: the store does not hold this value.
++    await siteConfig.setSecret('store-side-salt', { reason: 'store side' });
++    await db.run("UPDATE media_settings SET value = ? WHERE key = 'view_hash_secret'", ['legacy-side-salt']);
++    const kept = await siteConfig.migrateLegacySecrets(store);
++    assert.strictEqual(kept.blanked, 0, 'a differing value is never blanked');
++    assert.strictEqual(kept.differs, 1, 'the unequal value is counted');
++    assert.strictEqual((await db.get('SELECT value FROM media_settings WHERE key = ?', ['view_hash_secret'])).value, 'legacy-side-salt', 'the newer legacy value is kept');
++    assert.strictEqual(await siteConfig.secretValue(), 'store-side-salt', 'the reader keeps the store value');
++
++    // An empty store value keeps the row too.
++    await store.deleteSecret('view_hash_secret', { reason: 'empty store' });
++    await db.run("UPDATE media_settings SET value = ? WHERE key = 'view_hash_secret'", ['legacy-only-salt']);
++    const empty = await siteConfig.migrateLegacySecrets(store);
++    assert.strictEqual(empty.blanked, 0, 'nothing is blanked when the store holds no value');
++    assert.strictEqual(empty.differs, 1, 'the empty-store case is counted');
++    assert.strictEqual((await db.get('SELECT value FROM media_settings WHERE key = ?', ['view_hash_secret'])).value, 'legacy-only-salt', 'the only copy is kept');
++    assert.ok(!logs.join('\n').includes('store-side-salt') && !logs.join('\n').includes('legacy-side-salt') && !logs.join('\n').includes('legacy-only-salt'), 'no log line carries either side');
++
++    real.log('legacy secrets (KEK): all checks passed');
++    process.exit(0);
++})().catch((err) => { real.error(err); process.exit(1); });
ddiff --git a/test/legacy-secrets-nokek.test.js b/test/legacy-secrets-nokek.test.js
nnew file mode 100644
iindex 0000000..7029397
---- /dev/null
++++ b/test/legacy-secrets-nokek.test.js
@@@ -0,0 +1,35 @@
++'use strict';
++/**
++ * D11: without a KEK the config store is not sealed and nothing in media_settings is blanked. The reader still
++ * gets the salt (the store holds it, as before).
++ *
++ *   node --import ./test/helpers/pg-preload.mjs test/legacy-secrets-nokek.test.js
++ */
++const assert = require('assert');
++process.env.NODE_ENV = 'test';
++delete process.env.OV_CONFIG_KEKS;
++delete process.env.OV_CONFIG_KEK_CURRENT;
++delete process.env.VIEW_HASH_SECRET;
++delete process.env.MEDIA_SECRET;
++
++const SECRET = 'sentinel-not-a-secret-no-kek';
++const real = { log: console.log, error: console.error };
++console.log = () => {}; console.warn = () => {}; console.error = () => {};
++
++(async () => {
++    const db = require('../server/db/database');
++    await db.initDb();
++    await db.run("INSERT INTO media_settings (key, value, description, type) VALUES ('view_hash_secret', ?, 'salt', 'string')", [SECRET]);
++
++    const siteConfig = require('../server/admin/site-config');
++    const report = await siteConfig.init();
++    const store = await siteConfig.getStore();
++    assert.strictEqual(store.sealed, false);
++    assert.strictEqual(report.blanked, 0, 'nothing is blanked without a sealed store');
++    assert.strictEqual((await db.get('SELECT value FROM media_settings WHERE key = ?', ['view_hash_secret'])).value, SECRET, 'the only copy is still the legacy row');
++    assert.strictEqual(await siteConfig.secretValue(), SECRET);
++    assert.strictEqual(await require('../server/views/service').hashSecret(), SECRET);
++
++    real.log('legacy secrets (no KEK): all checks passed');
++    process.exit(0);
++})().catch((err) => { real.error(err); process.exit(1); });