openvibers/openvibe.media Commits
Plan T4: steps 7 and 8 of the ordered cleanup brief in… (#10)
Changes
@@@ -52,7 +52,7 @@ MEDIA_OWNER_SUBJECT_INTERVAL_MIN=10# never stored in plaintext). allowed_origins guards browser (user-JWT) calls.# MEDIA_APPS_SEED='[{"app_id":"live","name":"OpenVibe.Live","api_key":"change-me","webhook_url":"http://127.0.0.1:3000/api/media-webhook","webhook_secret":"change-me-too","allowed_origins":["https://openvibe.live"],"quota_bytes":0}]'MEDIA_APPS_SEED=--# Fallback short form (no webhooks/origins): "live:key1,games:key2"++# deprecated: use MEDIA_APPS_SEED (fallback short form, no webhooks/origins): "live:key1,games:key2"MEDIA_APP_KEYS=# ── Developer-project tenants (ADR-014) ──────────────────────@@@ -113,7 +113,7 @@ server/client-ip.js trust proxy = loopback; req.ip is the only client address(openvibe-shared v1.25.0, openvibe-contracts v0.71.0, openvibe-sdk v0.12.0: pinned release tarballs, installed by npm)scripts/smoke-test.sh end-to-end smoke test (boots a temp instance)--scripts/backfill-objects.js / reconcile-objects.js / object-invariant.js / no-good-copy-report.js object-model operator tools++scripts/reconcile-objects.js / object-invariant.js / object-drift-report.js / no-good-copy-report.js object-model operator toolsscripts/media-jobs.js list jobs, run the size-invariant scan (dry run by default), approve/cancel proposalsscripts/r2-eviction-drill.js evict one VOD's R2 copy, prove B2 serves it, re-warm R2; JSON artifact (dry run by default)scripts/vod-duration-reconcile.js stored VOD durations vs the real files (dry run by default; --apply --backup; --rollback)@@@ -212,7 +212,7 @@ On every boot the service upserts apps from env (keys are hashed):```bashMEDIA_APPS_SEED='[{"app_id":"live","name":"OpenVibe.Live","api_key":"…","webhook_url":"http://127.0.0.1:3000/internal/media-webhook","webhook_secret":"…","allowed_origins":["https://openvibe.live"],"quota_bytes":0}]'--# fallback short form (no webhooks/origins):++# deprecated: use MEDIA_APPS_SEED (fallback short form, no webhooks/origins)MEDIA_APP_KEYS="live:key1,games:key2"```@@@ -494,10 +494,10 @@ Every stored blob is a **media object** (`med_<ULID>`, `media_objects`) with oneexisting route and response is unchanged, and the old write paths keep themodel current. Full reference: **[docs/object-model.md](docs/object-model.md)**.--- **Backfill** — `node scripts/backfill-objects.js [--dry-run]`: one object per++- **Backfill** — `backfill()` in `server/objects/backfill.js`: one object pervod, clip, file, screenshot, avatar and thumbnail; idempotent; never moves-- bytes or calls B2/R2 (remote copies stay `pending`). The service runs the-- `--only-missing` form 15 s after boot.++ bytes or calls B2/R2 (remote copies stay `pending`). It is a module, not a++ CLI: the service calls it itself with `onlyMissing: true` 15 s after boot.- **API** — `/api/v2/:app/objects`: init → `PUT /:id/content` (sha256, size,quota) → `/:id/complete`; `GET /:id`, cursor `GET /`, soft `DELETE /:id`(bytes kept `MEDIA_DELETE_RETENTION_DAYS`), `/:id/restore`, `/:id/download`@@@ -618,8 +618,9 @@ Reporting a vulnerability: [SECURITY.md](SECURITY.md). The rules the code keeps:`test/rtmp-allowlist.test.js`); avataringestion fetches through the SSRF guard; other calls go to the configured Network, Live, Events, B2and R2 endpoints.--- **Secrets.** `OV_OAUTH_CLIENT_SECRET`, `MEDIA_SIGNING_SECRET`, `MEDIA_SECRET`, `MEDIA_APPS_SEED` /-- `MEDIA_APP_KEYS`, `MEDIA_B2_*`, `MEDIA_R2_*`, `MEDIA_INBOUND_EVENTS_SECRET` and `VIEW_HASH_SECRET` (the++- **Secrets.** `OV_OAUTH_CLIENT_SECRET`, `MEDIA_SIGNING_SECRET`, `MEDIA_SECRET`, `MEDIA_APPS_SEED`++ (and the deprecated `MEDIA_APP_KEYS`), `MEDIA_B2_*`, `MEDIA_R2_*`, `MEDIA_INBOUND_EVENTS_SECRET` and++ `VIEW_HASH_SECRET` (thesecret visitor hashes are derived from; it falls back to `MEDIA_SECRET`) live in`/etc/openvibe/media.env` (0600), by name only.@@@ -75,23 +75,20 @@ the same functions.## Backfill--```--node scripts/backfill-objects.js [--dry-run] [--only-missing] [--json] [--out report.json]--```----The backfill creates one object per vod, clip, file, screenshot/avatar paste and thumbnail, with--relationships and thumbnail variants.++`backfill()` in `server/objects/backfill.js` is a module, not a CLI: it is called from code, and the service calls it++itself with `{ onlyMissing: true }` 15 s after boot. It creates one object per vod, clip, file, screenshot/avatar paste++and thumbnail, with relationships and thumbnail variants.- **Idempotent.** Objects are keyed by `legacy_ref`, so a second run updates them in place and creates nothing.- **Never moves or deletes bytes, and never calls B2/R2.** A local copy is `present` when the file exists and `missing` when it does not. Remote copies stay `pending` until `reconcile-objects.js --verify` checks them.--- **One transaction,** with one savepoint per row: a failing row is reported and leaves nothing half-written. `--dry-run` rolls everything back and only reports.++- **One transaction,** with one savepoint per row: a failing row is reported and leaves nothing half-written. A dry run (`{ dryRun: true }`) rolls everything back and only reports.- **Report:** counts per kind (seen, created, updated, skipped), location states, skipped rows with a reason, and errors. A real run stores the report in `media_settings` under `objects.backfill.last_report`.--- **At boot:** the service runs the `--only-missing` form itself 15 s after start, for rows with no `object_id`. The first boot after the upgrade therefore projects everything. With object-first writes it should find nothing; it stays until the [drift report](#drift-report) shows zero drift across a release.++- **At boot:** the service runs the `onlyMissing` form itself 15 s after start, for rows with no `object_id`. The first boot after the upgrade therefore projects everything. With object-first writes it should find nothing; it stays until the [drift report](#drift-report) shows zero drift across a release.Skip reasons today: `clips-only recording (ephemeral, never published)`, `screenshot paste without a file path`, `external thumbnail url`.--The backfill's own transaction holds row and table locks for its duration, typically seconds. Prefer the boot backfill, or--run the script while traffic is quiet.++The backfill's own transaction holds row and table locks for its duration, typically seconds. Prefer the boot backfill; a++manual call should wait until traffic is quiet.### Keeping the model current: object-first writes@@@ -10,8 +10,9 @@* Browser endpoints additionally check the Origin header against the* app's allowed_origins list.*-- * App seeding: MEDIA_APPS_SEED (JSON array) and/or MEDIA_APP_KEYS ("id:key,…")-- * are upserted on boot; keys are stored as sha256 hashes.++ * App seeding: MEDIA_APPS_SEED (JSON array) and/or the deprecated MEDIA_APP_KEYS++ * ("id:key,…") short form are upserted on boot; keys are stored as sha256 hashes.++ * MEDIA_APP_KEYS is deprecated — move each app to MEDIA_APPS_SEED.** Network principal tokens (service and developer app tokens) are checked per namespace for one of* five verbs: read, list, write, delete, transform (VERBS below; objects/namespaces.js has the rows).@@@ -469,6 +470,7 @@ async function seedApps() {}if (config.apps.seedKeys) {++ console.warn('[Auth] MEDIA_APP_KEYS is deprecated: move each app to MEDIA_APPS_SEED (JSON with app_id, api_key, webhook_url, allowed_origins)');for (const pair of config.apps.seedKeys.split(',')) {const idx = pair.indexOf(':');if (idx < 1) continue;@@@ -95,11 +95,20 @@ const crypto = require('crypto');});for (const k of ['MEDIA_DRILL', 'PASTES_MOVED_TO', 'EVENTS_PUBLISH']) delete process.env[k];++ // MEDIA_APP_KEYS is deprecated (T4): it still seeds, but must warn once per boot and never name the key.++ const warnLines = [];++ const realWarn = console.warn;++ console.warn = (...args) => { warnLines.push(args.join(' ')); return realWarn.apply(console, args); };const { app, server } = await require('../server/index.js').ready;if (!server.listening) await new Promise((r) => server.once('listening', r));++ console.warn = realWarn;++ const depr = warnLines.filter((l) => l.includes('MEDIA_APP_KEYS is deprecated'));++ assert.strictEqual(depr.length, 1, `MEDIA_APP_KEYS warns once per boot (saw ${depr.length})`);++ assert.ok(depr[0].includes('MEDIA_APPS_SEED') && !depr[0].includes(LIVE_KEY), 'the warning points at MEDIA_APPS_SEED and never names a key');const auth = require('../server/auth');auth._setNetworkPublicKeyForTests(keys.publicKey);const db = require('../server/db/database');++ assert.strictEqual((await db.getApp('live')).app_id, 'live', 'MEDIA_APP_KEYS still seeds its app (deprecated, not removed)');const base = `http://127.0.0.1:${server.address().port}`;// ── Credentials ──
@@ -52,7 +52,7 @@ MEDIA_OWNER_SUBJECT_INTERVAL_MIN=10
@@ -52,7 +52,7 @@ MEDIA_OWNER_SUBJECT_INTERVAL_MIN=10
# never stored in plaintext). allowed_origins guards browser (user-JWT) calls.
# never stored in plaintext). allowed_origins guards browser (user-JWT) calls.
# MEDIA_APPS_SEED='[{"app_id":"live","name":"OpenVibe.Live","api_key":"change-me","webhook_url":"http://127.0.0.1:3000/api/media-webhook","webhook_secret":"change-me-too","allowed_origins":["https://openvibe.live"],"quota_bytes":0}]'
# MEDIA_APPS_SEED='[{"app_id":"live","name":"OpenVibe.Live","api_key":"change-me","webhook_url":"http://127.0.0.1:3000/api/media-webhook","webhook_secret":"change-me-too","allowed_origins":["https://openvibe.live"],"quota_bytes":0}]'
MEDIA_APPS_SEED=
MEDIA_APPS_SEED=
-# Fallback short form (no webhooks/origins): "live:key1,games:key2"
+# deprecated: use MEDIA_APPS_SEED (fallback short form, no webhooks/origins): "live:key1,games:key2"
MEDIA_APP_KEYS=
MEDIA_APP_KEYS=
# ── Developer-project tenants (ADR-014) ──────────────────────
# ── Developer-project tenants (ADR-014) ──────────────────────
@@ -113,7 +113,7 @@ server/
@@ -113,7 +113,7 @@ server/
client-ip.js trust proxy = loopback; req.ip is the only client address
client-ip.js trust proxy = loopback; req.ip is the only client address
(openvibe-shared v1.25.0, openvibe-contracts v0.71.0, openvibe-sdk v0.12.0: pinned release tarballs, installed by npm)
(openvibe-shared v1.25.0, openvibe-contracts v0.71.0, openvibe-sdk v0.12.0: pinned release tarballs, installed by npm)
scripts/smoke-test.sh end-to-end smoke test (boots a temp instance)
scripts/smoke-test.sh end-to-end smoke test (boots a temp instance)
-scripts/backfill-objects.js / reconcile-objects.js / object-invariant.js / no-good-copy-report.js object-model operator tools
+scripts/reconcile-objects.js / object-invariant.js / object-drift-report.js / no-good-copy-report.js object-model operator tools
scripts/media-jobs.js list jobs, run the size-invariant scan (dry run by default), approve/cancel proposals
scripts/media-jobs.js list jobs, run the size-invariant scan (dry run by default), approve/cancel proposals
scripts/r2-eviction-drill.js evict one VOD's R2 copy, prove B2 serves it, re-warm R2; JSON artifact (dry run by default)
scripts/r2-eviction-drill.js evict one VOD's R2 copy, prove B2 serves it, re-warm R2; JSON artifact (dry run by default)
scripts/vod-duration-reconcile.js stored VOD durations vs the real files (dry run by default; --apply --backup; --rollback)
scripts/vod-duration-reconcile.js stored VOD durations vs the real files (dry run by default; --apply --backup; --rollback)
@@ -212,7 +212,7 @@ On every boot the service upserts apps from env (keys are hashed):
@@ -212,7 +212,7 @@ On every boot the service upserts apps from env (keys are hashed):
```bash
```bash
MEDIA_APPS_SEED='[{"app_id":"live","name":"OpenVibe.Live","api_key":"…","webhook_url":"http://127.0.0.1:3000/internal/media-webhook","webhook_secret":"…","allowed_origins":["https://openvibe.live"],"quota_bytes":0}]'
MEDIA_APPS_SEED='[{"app_id":"live","name":"OpenVibe.Live","api_key":"…","webhook_url":"http://127.0.0.1:3000/internal/media-webhook","webhook_secret":"…","allowed_origins":["https://openvibe.live"],"quota_bytes":0}]'
-# fallback short form (no webhooks/origins):
+# deprecated: use MEDIA_APPS_SEED (fallback short form, no webhooks/origins)
MEDIA_APP_KEYS="live:key1,games:key2"
MEDIA_APP_KEYS="live:key1,games:key2"
```
```
@@ -494,10 +494,10 @@ Every stored blob is a **media object** (`med_<ULID>`, `media_objects`) with one
@@ -494,10 +494,10 @@ Every stored blob is a **media object** (`med_<ULID>`, `media_objects`) with one
existing route and response is unchanged, and the old write paths keep the
existing route and response is unchanged, and the old write paths keep the
model current. Full reference: **[docs/object-model.md](docs/object-model.md)**.
model current. Full reference: **[docs/object-model.md](docs/object-model.md)**.
-- **Backfill** — `node scripts/backfill-objects.js [--dry-run]`: one object per
+- **Backfill** — `backfill()` in `server/objects/backfill.js`: one object per
vod, clip, file, screenshot, avatar and thumbnail; idempotent; never moves
vod, clip, file, screenshot, avatar and thumbnail; idempotent; never moves
- bytes or calls B2/R2 (remote copies stay `pending`). The service runs the
- `--only-missing` form 15 s after boot.
+ bytes or calls B2/R2 (remote copies stay `pending`). It is a module, not a
+ CLI: the service calls it itself with `onlyMissing: true` 15 s after boot.
- **API** — `/api/v2/:app/objects`: init → `PUT /:id/content` (sha256, size,
- **API** — `/api/v2/:app/objects`: init → `PUT /:id/content` (sha256, size,
quota) → `/:id/complete`; `GET /:id`, cursor `GET /`, soft `DELETE /:id`
quota) → `/:id/complete`; `GET /:id`, cursor `GET /`, soft `DELETE /:id`
(bytes kept `MEDIA_DELETE_RETENTION_DAYS`), `/:id/restore`, `/:id/download`
(bytes kept `MEDIA_DELETE_RETENTION_DAYS`), `/:id/restore`, `/:id/download`
@@ -618,8 +618,9 @@ Reporting a vulnerability: [SECURITY.md](SECURITY.md). The rules the code keeps:
@@ -618,8 +618,9 @@ Reporting a vulnerability: [SECURITY.md](SECURITY.md). The rules the code keeps:
`test/rtmp-allowlist.test.js`); avatar
`test/rtmp-allowlist.test.js`); avatar
ingestion fetches through the SSRF guard; other calls go to the configured Network, Live, Events, B2
ingestion fetches through the SSRF guard; other calls go to the configured Network, Live, Events, B2
and R2 endpoints.
and R2 endpoints.
-- **Secrets.** `OV_OAUTH_CLIENT_SECRET`, `MEDIA_SIGNING_SECRET`, `MEDIA_SECRET`, `MEDIA_APPS_SEED` /
- `MEDIA_APP_KEYS`, `MEDIA_B2_*`, `MEDIA_R2_*`, `MEDIA_INBOUND_EVENTS_SECRET` and `VIEW_HASH_SECRET` (the
+- **Secrets.** `OV_OAUTH_CLIENT_SECRET`, `MEDIA_SIGNING_SECRET`, `MEDIA_SECRET`, `MEDIA_APPS_SEED`
+ (and the deprecated `MEDIA_APP_KEYS`), `MEDIA_B2_*`, `MEDIA_R2_*`, `MEDIA_INBOUND_EVENTS_SECRET` and
+ `VIEW_HASH_SECRET` (the
secret visitor hashes are derived from; it falls back to `MEDIA_SECRET`) live in
secret visitor hashes are derived from; it falls back to `MEDIA_SECRET`) live in
`/etc/openvibe/media.env` (0600), by name only.
`/etc/openvibe/media.env` (0600), by name only.
@@ -75,23 +75,20 @@ the same functions.
@@ -75,23 +75,20 @@ the same functions.
## Backfill
## Backfill
-```
-node scripts/backfill-objects.js [--dry-run] [--only-missing] [--json] [--out report.json]
-```
-
-The backfill creates one object per vod, clip, file, screenshot/avatar paste and thumbnail, with
-relationships and thumbnail variants.
+`backfill()` in `server/objects/backfill.js` is a module, not a CLI: it is called from code, and the service calls it
+itself with `{ onlyMissing: true }` 15 s after boot. It creates one object per vod, clip, file, screenshot/avatar paste
+and thumbnail, with relationships and thumbnail variants.
- **Idempotent.** Objects are keyed by `legacy_ref`, so a second run updates them in place and creates nothing.
- **Idempotent.** Objects are keyed by `legacy_ref`, so a second run updates them in place and creates nothing.
- **Never moves or deletes bytes, and never calls B2/R2.** A local copy is `present` when the file exists and `missing` when it does not. Remote copies stay `pending` until `reconcile-objects.js --verify` checks them.
- **Never moves or deletes bytes, and never calls B2/R2.** A local copy is `present` when the file exists and `missing` when it does not. Remote copies stay `pending` until `reconcile-objects.js --verify` checks them.
-- **One transaction,** with one savepoint per row: a failing row is reported and leaves nothing half-written. `--dry-run` rolls everything back and only reports.
+- **One transaction,** with one savepoint per row: a failing row is reported and leaves nothing half-written. A dry run (`{ dryRun: true }`) rolls everything back and only reports.
- **Report:** counts per kind (seen, created, updated, skipped), location states, skipped rows with a reason, and errors. A real run stores the report in `media_settings` under `objects.backfill.last_report`.
- **Report:** counts per kind (seen, created, updated, skipped), location states, skipped rows with a reason, and errors. A real run stores the report in `media_settings` under `objects.backfill.last_report`.
-- **At boot:** the service runs the `--only-missing` form itself 15 s after start, for rows with no `object_id`. The first boot after the upgrade therefore projects everything. With object-first writes it should find nothing; it stays until the [drift report](#drift-report) shows zero drift across a release.
+- **At boot:** the service runs the `onlyMissing` form itself 15 s after start, for rows with no `object_id`. The first boot after the upgrade therefore projects everything. With object-first writes it should find nothing; it stays until the [drift report](#drift-report) shows zero drift across a release.
Skip reasons today: `clips-only recording (ephemeral, never published)`, `screenshot paste without a file path`, `external thumbnail url`.
Skip reasons today: `clips-only recording (ephemeral, never published)`, `screenshot paste without a file path`, `external thumbnail url`.
-The backfill's own transaction holds row and table locks for its duration, typically seconds. Prefer the boot backfill, or
-run the script while traffic is quiet.
+The backfill's own transaction holds row and table locks for its duration, typically seconds. Prefer the boot backfill; a
+manual call should wait until traffic is quiet.
### Keeping the model current: object-first writes
### Keeping the model current: object-first writes
@@ -10,8 +10,9 @@
@@ -10,8 +10,9 @@
* Browser endpoints additionally check the Origin header against the
* Browser endpoints additionally check the Origin header against the
* app's allowed_origins list.
* app's allowed_origins list.
*
*
- * App seeding: MEDIA_APPS_SEED (JSON array) and/or MEDIA_APP_KEYS ("id:key,…")
- * are upserted on boot; keys are stored as sha256 hashes.
+ * App seeding: MEDIA_APPS_SEED (JSON array) and/or the deprecated MEDIA_APP_KEYS
+ * ("id:key,…") short form are upserted on boot; keys are stored as sha256 hashes.
+ * MEDIA_APP_KEYS is deprecated — move each app to MEDIA_APPS_SEED.
*
*
* Network principal tokens (service and developer app tokens) are checked per namespace for one of
* Network principal tokens (service and developer app tokens) are checked per namespace for one of
* five verbs: read, list, write, delete, transform (VERBS below; objects/namespaces.js has the rows).
* five verbs: read, list, write, delete, transform (VERBS below; objects/namespaces.js has the rows).
@@ -469,6 +470,7 @@ async function seedApps() {
@@ -469,6 +470,7 @@ async function seedApps() {
}
}
if (config.apps.seedKeys) {
if (config.apps.seedKeys) {
+ console.warn('[Auth] MEDIA_APP_KEYS is deprecated: move each app to MEDIA_APPS_SEED (JSON with app_id, api_key, webhook_url, allowed_origins)');
for (const pair of config.apps.seedKeys.split(',')) {
for (const pair of config.apps.seedKeys.split(',')) {
const idx = pair.indexOf(':');
const idx = pair.indexOf(':');
if (idx < 1) continue;
if (idx < 1) continue;
@@ -95,11 +95,20 @@ const crypto = require('crypto');
@@ -95,11 +95,20 @@ const crypto = require('crypto');
});
});
for (const k of ['MEDIA_DRILL', 'PASTES_MOVED_TO', 'EVENTS_PUBLISH']) delete process.env[k];
for (const k of ['MEDIA_DRILL', 'PASTES_MOVED_TO', 'EVENTS_PUBLISH']) delete process.env[k];
+ // MEDIA_APP_KEYS is deprecated (T4): it still seeds, but must warn once per boot and never name the key.
+ const warnLines = [];
+ const realWarn = console.warn;
+ console.warn = (...args) => { warnLines.push(args.join(' ')); return realWarn.apply(console, args); };
const { app, server } = await require('../server/index.js').ready;
const { app, server } = await require('../server/index.js').ready;
if (!server.listening) await new Promise((r) => server.once('listening', r));
if (!server.listening) await new Promise((r) => server.once('listening', r));
+ console.warn = realWarn;
+ const depr = warnLines.filter((l) => l.includes('MEDIA_APP_KEYS is deprecated'));
+ assert.strictEqual(depr.length, 1, `MEDIA_APP_KEYS warns once per boot (saw ${depr.length})`);
+ assert.ok(depr[0].includes('MEDIA_APPS_SEED') && !depr[0].includes(LIVE_KEY), 'the warning points at MEDIA_APPS_SEED and never names a key');
const auth = require('../server/auth');
const auth = require('../server/auth');
auth._setNetworkPublicKeyForTests(keys.publicKey);
auth._setNetworkPublicKeyForTests(keys.publicKey);
const db = require('../server/db/database');
const db = require('../server/db/database');
+ assert.strictEqual((await db.getApp('live')).app_id, 'live', 'MEDIA_APP_KEYS still seeds its app (deprecated, not removed)');
const base = `http://127.0.0.1:${server.address().port}`;
const base = `http://127.0.0.1:${server.address().port}`;
// ── Credentials ──
// ── Credentials ──