openvibers/openvibe.media Commits
Media Fabric F1b, off until the edge host exists: the origin shield. For a public B2 copy Node answers with X-Accel-Redirect into nginx's /_media_shield/b2/ (a 10 MB slice cache with proxy_cache_lock, keyed by object path and slice, never the signature; an hour-long internal presign) instead of redirecting the viewer. It answers only on MEDIA_SHIELD_HOST, a DNS-only edge name: openvibe.media is behind Cloudflare's proxy, whose video terms are unverified, so bytes that went viewer to B2 never start flowing through it. Private and sandbox bytes never enter the cache; deleting a shielded object refreshes its slices through a loopback-only listener. nginx config in deploy/nginx (cache 5 GB, min_free 20 GB so recordings keep their disk; shield logs never carry the signed query); decisions in docs/media-fabric.md. Tests: placement-shield (targeting, edge-host gate, private/sandbox, purge ranges), placement, objects-api.
Changes
nnew file mode 100644@@@ -0,0 +1,49 @@++# Media Fabric F1b — the DNS-only edge host (install in sites-enabled). Viewers' bytes come from here, not through++# Cloudflare's proxy (its terms for video are not verified: plan T4). Node answers every request as on openvibe.media;++# for a public B2 copy it answers with X-Accel-Redirect into /_media_shield/b2/ and nginx serves the bytes from the++# slice cache (MEDIA_SHIELD=b2, MEDIA_SHIELD_HOST=edge.openvibe.media in /etc/openvibe/media.env).++server {++ listen 80;++ listen [::]:80;++ server_name edge.openvibe.media;++ location /.well-known/acme-challenge/ { root /var/www/html; }++ location / { return 301 https://edge.openvibe.media$request_uri; }++}++server {++ listen 443 ssl http2;++ listen [::]:443 ssl http2;++ server_name edge.openvibe.media;++ ssl_certificate /etc/letsencrypt/live/edge.openvibe.media/fullchain.pem;++ ssl_certificate_key /etc/letsencrypt/live/edge.openvibe.media/privkey.pem;++++ location /_media_shield/b2/ {++ internal;++ access_log /var/log/nginx/media-shield.log media_shield;++ rewrite ^/_media_shield/b2/(.*)$ /$1 break;++ slice 10m;++ proxy_cache media_shield;++ proxy_cache_key "b2|$uri|$slice_range";++ proxy_cache_valid 200 206 24h;++ proxy_cache_valid 404 1m;++ # Single flight per object + slice: N viewers of the same range cost one B2 GET.++ proxy_cache_lock on;++ proxy_cache_lock_timeout 10s;++ proxy_cache_lock_age 10s;++ proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;++ proxy_cache_background_update on;++ add_header X-Cache-Status $upstream_cache_status always;++ include snippets/media-shield-b2-upstream.conf;++ }++++ # Everything else is Node, exactly as on openvibe.media (client-IP headers from $remote_addr only).++ location / {++ proxy_pass http://127.0.0.1:4100;++ proxy_http_version 1.1;++ proxy_set_header Host $host;++ proxy_set_header X-Real-IP $remote_addr;++ proxy_set_header X-Forwarded-For $remote_addr;++ proxy_set_header CF-Connecting-IP $remote_addr;++ proxy_set_header X-Forwarded-Proto $scheme;++ proxy_request_buffering off;++ }++}nnew file mode 100644@@@ -0,0 +1,15 @@++# Media Fabric F1b — how the shield reaches B2 (install as /etc/nginx/snippets/media-shield-b2-upstream.conf).++# B2 is addressed path-style, so every presigned URL is signed for this one host (SigV4 signs the Host header).++proxy_set_header Range $slice_range;++proxy_set_header If-Range "";++proxy_set_header Host s3.us-west-004.backblazeb2.com;++proxy_set_header Connection "";++proxy_set_header Cookie "";++proxy_set_header Authorization "";++proxy_http_version 1.1;++proxy_ssl_server_name on;++proxy_ssl_name s3.us-west-004.backblazeb2.com;++proxy_ignore_headers Cache-Control Expires Set-Cookie;++proxy_connect_timeout 5s;++proxy_read_timeout 60s;++proxy_pass https://s3.us-west-004.backblazeb2.com;nnew file mode 100644@@@ -0,0 +1,27 @@++# Media Fabric F1b — origin shield, http-level part (install as /etc/nginx/conf.d/media-shield.conf).++# One slice cache for B2 reads: 10 MB slices keyed by object path + slice, never by the presigned signature.++# Sizing: the cache shares the disk with recordings, so it is small and yields to them: nginx evicts when free space++# drops below 20 GB (recordings' guardian warns at 15 GB), and never holds more than 5 GB.++proxy_cache_path /var/cache/nginx/media-shield levels=1:2 keys_zone=media_shield:64m max_size=5g min_free=20g inactive=24h use_temp_path=off;++++# The signed query is a bearer token: shield logs carry the path only ($uri), never $args or $request_uri.++log_format media_shield '$remote_addr [$time_local] "$request_method $uri" $status $body_bytes_sent cache=$upstream_cache_status range="$http_range" rt=$request_time';++++# Purge (server/placement/shield.js purge): loopback only, bypasses the cache and stores the provider's answer for the++# same key (a deleted object's 404 replaces its slices, cached for a minute).++server {++ listen 127.0.0.1:8479;++ server_name _;++ access_log /var/log/nginx/media-shield.log media_shield;++ location /_media_shield_refresh/b2/ {++ rewrite ^/_media_shield_refresh/b2/(.*)$ /$1 break;++ slice 10m;++ proxy_cache media_shield;++ proxy_cache_key "b2|$uri|$slice_range";++ proxy_cache_bypass 1;++ proxy_cache_valid 200 206 24h;++ proxy_cache_valid 403 404 1m;++ include snippets/media-shield-b2-upstream.conf;++ }++ location / { return 404; }++}@@@ -187,3 +187,21 @@ metric. Budgets per class and provider with a forecast; `provider.cost.thresholdepochs, canary shifts with auto-revert, player QoE beacons, seek SLOs.- **F6 cost truth:** bill imports, forecasts, budget guards, the owner dashboard.- **Archive fabric:** backups and snapshots to B2; optional deep archive with restore drills; never online.++++## F1b decisions (2026-09-29)++++- **Shield B2 only.** B2 is the canonical copy whose ~500 req/s ceiling is worth shielding; R2 reads keep their 302++ until F2's engine can enforce the 70 %-of-port rule (an R2 read through the shield spends the same host bandwidth).++- **A DNS-only edge host.** openvibe.media is behind Cloudflare's proxy and its terms for video are not verified, so++ the shield answers only on `edge.openvibe.media` (grey cloud; `MEDIA_SHIELD_HOST`). Bytes that went viewer → B2 never++ start flowing through Cloudflare.++- **Cache key:** `b2|<bucket>/<key>|<slice range>`: the object path and the 10 MB slice, never the signature. The++ shield's presign lives an hour (it stays inside nginx and a long response fetches later slices with it); the++ viewer-facing 302 keeps its short TTL.++- **Private and sandbox bytes never enter the shield.**++- **Sizing:** one disk with recordings: `max_size=5g`, `min_free=20g` (the recording guardian warns at 15 GB), `inactive=24h`.++- **Purge:** deleting a shielded object refreshes each of its slices through a loopback-only listener++ (`127.0.0.1:8479`, `proxy_cache_bypass`), so the provider's 404 replaces the cached bytes at once.++- **Logs:** shield logs carry the path only: the signed query is a bearer token.++- Config: `deploy/nginx/media-shield.http.conf` (conf.d), `media-shield-b2-upstream.conf` (snippets),++ `edge.openvibe.media.conf` (sites-enabled). Switch on with `MEDIA_SHIELD=b2` and `MEDIA_SHIELD_HOST` after nginx has them.@@@ -773,6 +773,14 @@ publicRouter.get('/:id', async (req, res) => {if (fs.existsSync(loc.key)) return require('../public/routes').streamFileWithRange(req, res, loc.key, headers);continue;}++ // The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through its++ // slice cache, with an hour-long internal presign that never reaches the viewer.++ const shield = require('../placement/shield');++ if (shield.enabled(provider) && !sandbox && obj.visibility !== 'private') {++ const inner = await require('../vod/vod-storage').presignGet(provider, loc.key, shield.SHIELD_TTL_SECONDS, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);++ const t = inner && shield.target({ provider, url: inner, visibility: obj.visibility, sandbox, host: req.hostname });++ if (t) return shield.send(res, t, headers);++ }// The router already signed its first choice; a fallback copy is signed here.const url = (provider === decision.provider && decision.url) || await require('../vod/vod-storage').presignGet(provider, loc.key, 300, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);if (url) { res.set('Cache-Control', 'private, max-age=0'); res.set('X-Robots-Tag', 'noindex'); return res.redirect(302, url); }nnew file mode 100644@@@ -0,0 +1,81 @@++/**++ * OpenVibe.Media — the origin shield (Media Fabric F1b, docs/media-fabric.md "origin shield").++ *++ * Node still authorizes every read and picks the copy (the placement router); for a public copy on a shielded provider++ * it no longer redirects the viewer to a presigned URL. It answers with `X-Accel-Redirect` into nginx's internal++ * `/_media_shield/<provider>/…` location, which proxies the presigned request through a slice cache (10 MB slices,++ * `proxy_cache_lock`): N viewers of the same object range cost one provider GET, the bytes come from the Media host's++ * prepaid bandwidth, and the provider URL never reaches the viewer. The cache key is the object path and the slice,++ * never the signature (deploy/nginx/openvibe.media.conf).++ *++ * MEDIA_SHIELD=b2 providers whose reads go through the shield (comma-separated; empty = off). Set it only++ * once nginx has the locations: a Node that emits X-Accel to an nginx without them 404s.++ * MEDIA_SHIELD_HOST=edge.openvibe.media the ONLY host the shield answers on: a DNS-only (grey-cloud) edge name.++ * openvibe.media is behind Cloudflare's proxy, whose terms for video are not verified (plan++ * T4): bytes that went viewer → B2 must not start flowing through Cloudflare. Unset = off.++ * MEDIA_SHIELD_REFRESH_URL the loopback-only purge listener (default http://127.0.0.1:8479).++ *++ * Rules: private and sandbox bytes never enter the shared cache; B2 only for now (path-style URLs, one signed host; R2++ * waits for F2's port-utilization guard); the shield's presign lives an hour (it stays inside nginx, and a long response++ * fetches later slices with it), the viewer-facing 302 keeps its short TTL.++ *++ * target({ provider, url, visibility, sandbox }) → '/_media_shield/b2/<bucket>/<key>?<signed query>' | null++ * send(res, target, headers) → the 200 + X-Accel-Redirect answer++ * await purge({ provider, key, size, presign }) → refreshes every cached slice of a deleted or replaced object++ */++'use strict';++++const SLICE_BYTES = 10 * 1024 * 1024; // nginx `slice 10m`++const SHIELD_TTL_SECONDS = 3600;++++function providers() {++ return String(process.env.MEDIA_SHIELD || '').split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);++}++function enabled(provider) { return providers().includes(String(provider || '').toLowerCase()); }++++/** The internal location for one presigned URL, or null when this read must not go through the shield. `host` is the++ * request's host: the shield answers only on MEDIA_SHIELD_HOST (a DNS-only edge name), never through Cloudflare. */++function target({ provider, url, visibility = 'public', sandbox = false, host = null }) {++ if (!url || !enabled(provider)) return null;++ const edge = String(process.env.MEDIA_SHIELD_HOST || '').toLowerCase();++ if (!edge || String(host || '').toLowerCase() !== edge) return null;++ if (visibility === 'private' || sandbox) return null; // never into a shared cache++ let u; try { u = new URL(url); } catch { return null; }++ // Path-style URLs only: the pathname is /<bucket>/<key>, the host is the provider's one signed endpoint.++ if (!u.pathname || u.pathname === '/') return null;++ return `/_media_shield/${String(provider).toLowerCase()}${u.pathname}${u.search}`;++}++++/** Hand the read to nginx. Headers the route computed (type, disposition, cache policy) go with it. */++function send(res, targetPath, headers = {}) {++ for (const [k, v] of Object.entries(headers)) if (v != null) res.set(k, v);++ res.set('X-Accel-Redirect', targetPath);++ res.set('X-Media-Source', 'shield');++ return res.status(200).end();++}++++/**++ * Refresh every cached slice of an object after it was deleted or replaced on a shielded provider: one request per++ * 10 MB slice to nginx's loopback-only refresh location, which bypasses the cache and stores the provider's answer++ * (a 404 for a deleted object: cached for a minute, then gone). Never throws; returns what it did.++ */++async function purge({ provider, key, size, presign, fetchImpl = globalThis.fetch, base = process.env.MEDIA_SHIELD_REFRESH_URL || 'http://127.0.0.1:8479', host = process.env.MEDIA_SHIELD_HOST || 'localhost' }) {++ if (!enabled(provider) || !key) return { purged: 0, skipped: true };++ let url; try { url = await presign(provider, key, 300); } catch (err) { return { purged: 0, error: err.message }; }++ const t = target({ provider, url, host: process.env.MEDIA_SHIELD_HOST });++ if (!t) return { purged: 0, skipped: true };++ const refresh = t.replace('/_media_shield/', '/_media_shield_refresh/');++ const slices = Math.max(1, Math.ceil((Number(size) || SLICE_BYTES) / SLICE_BYTES));++ let purged = 0, failed = 0;++ for (let i = 0; i < slices; i++) {++ const range = `bytes=${i * SLICE_BYTES}-${(i + 1) * SLICE_BYTES - 1}`;++ try {++ const r = await fetchImpl(base + refresh, { headers: { Host: host, Range: range }, signal: AbortSignal.timeout(15000) });++ try { await r.arrayBuffer(); } catch { /* drained */ }++ purged++;++ } catch { failed++; }++ }++ return { purged, failed, slices };++}++++module.exports = { target, send, purge, enabled, providers, SLICE_BYTES, SHIELD_TTL_SECONDS };@@@ -174,6 +174,14 @@ async function serveMediaRecord(kind, record, req, res) {const router = require('../placement/router');const plan = await vodStorage.resolvePlayback(record, { session: router.sessionFor(req, `${kind}:${record.id}`) });if (plan?.kind === 'redirect') {++ // The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through++ // its slice cache with an hour-long internal presign; everything else keeps the short-lived 302.++ const shield = require('../placement/shield');++ if (shield.enabled(plan.provider) && visibility !== 'private' && !record.is_recording) {++ const url = await vodStorage.presignGet(plan.provider, vodStorage.keyForVod(record), shield.SHIELD_TTL_SECONDS).catch(() => null);++ const t = url && shield.target({ provider: plan.provider, url, visibility, host: req.hostname });++ if (t) return shield.send(res, t, { 'Cache-Control': visibility === 'public' ? 'public, max-age=300' : 'private, max-age=0', 'X-Robots-Tag': 'noindex' });++ }res.set('Cache-Control', 'private, max-age=0');res.set('X-Robots-Tag', 'noindex');return res.redirect(302, plan.url);@@@ -396,11 +396,19 @@ async function sha256Object(provider, key) {async function deleteObject(provider, key) {const client = clientFor(provider);if (!client) return;++ // A shielded provider's cached slices leave the shield with the object (placement/shield.js): the size decides++ // how many slices to refresh, so it is read before the delete.++ const shield = require('../placement/shield');++ const head = shield.enabled(provider) ? await headObject(provider, key).catch(() => null) : null;try {await client.send(new S3.DeleteObjectCommand({ Bucket: PROVIDER_ENV[provider].bucket, Key: key }));} catch (err) {console.warn(`[VodStorage] Delete ${provider}:${key} failed:`, err.message);++ return;}++ if (head) shield.purge({ provider, key, size: head.size, presign: presignGet })++ .then((r) => { if (r.failed) console.warn(`[VodStorage] shield purge ${provider}:${key}: ${r.failed} of ${r.slices} slices not refreshed`); })++ .catch(() => { /* purge never throws */ });}/**nnew file mode 100644@@@ -0,0 +1,42 @@++'use strict';++// Origin shield (Media Fabric F1b): which reads nginx serves from its slice cache, and the purge of a deleted object.++const assert = require('assert');++++(async () => {++ const shield = require('../server/placement/shield');++ const url = 'https://s3.us-west-004.backblazeb2.com/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc';++++ delete process.env.MEDIA_SHIELD;++ assert.strictEqual(shield.target({ provider: 'b2', url }), null, 'off unless MEDIA_SHIELD names the provider');++++ process.env.MEDIA_SHIELD = 'b2';++ assert.strictEqual(shield.target({ provider: 'b2', url, host: 'edge.openvibe.media' }), null, 'no edge host configured: off');++ process.env.MEDIA_SHIELD_HOST = 'edge.openvibe.media';++ assert.strictEqual(shield.target({ provider: 'b2', url, host: 'openvibe.media' }), null, 'never on the Cloudflare-proxied host');++ const host = 'edge.openvibe.media';++ assert.strictEqual(shield.target({ provider: 'b2', url, host }), '/_media_shield/b2/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc',++ 'the path-style path and the signed query ride into the internal location unchanged');++ assert.strictEqual(shield.target({ provider: 'r2', url, host }), null, 'R2 is not shielded in F1b');++ assert.strictEqual(shield.target({ provider: 'b2', url, host, visibility: 'private' }), null, 'private bytes never enter the shared cache');++ assert.strictEqual(shield.target({ provider: 'b2', url, host, sandbox: true }), null, 'sandbox bytes never enter it either');++ assert.strictEqual(shield.target({ provider: 'b2', url: 'not a url', host }), null);++ assert.strictEqual(shield.target({ provider: 'b2', url: 'https://s3.example.com/', host }), null, 'a URL without an object path is not shielded');++++ // send(): the route's headers plus the internal redirect, an empty 200.++ const res = { h: {}, set(k, v) { this.h[k] = v; }, status(c) { this.code = c; return this; }, end() { this.ended = true; return this; } };++ shield.send(res, '/_media_shield/b2/x', { 'Content-Type': 'video/mp4', 'Cache-Control': 'public, max-age=300', Skip: null });++ assert.deepStrictEqual([res.code, res.ended, res.h['X-Accel-Redirect'], res.h['Content-Type'], 'Skip' in res.h], [200, true, '/_media_shield/b2/x', 'video/mp4', false]);++++ // purge(): one refresh request per 10 MB slice, loopback, with the public Host and each slice's exact range.++ const calls = [];++ const fetchImpl = async (u, opts) => { calls.push([u, opts.headers.Host, opts.headers.Range]); return { arrayBuffer: async () => new ArrayBuffer(0) }; };++ const r = await shield.purge({ provider: 'b2', key: 'vods/a b.mp4', size: 25 * 1024 * 1024, presign: async () => url, fetchImpl });++ assert.deepStrictEqual([r.purged, r.slices, r.failed], [3, 3, 0]);++ assert.ok(calls.every(([u, h]) => u.startsWith('http://127.0.0.1:8479/_media_shield_refresh/b2/ov-vods/vods/a%20b.mp4?') && h === 'edge.openvibe.media'), JSON.stringify(calls[0]));++ assert.deepStrictEqual(calls.map((c) => c[2]), ['bytes=0-10485759', 'bytes=10485760-20971519', 'bytes=20971520-31457279']);++ assert.deepStrictEqual(await shield.purge({ provider: 'r2', key: 'k', size: 1, presign: async () => url, fetchImpl }), { purged: 0, skipped: true }, 'nothing to purge off the shield');++++ delete process.env.MEDIA_SHIELD; delete process.env.MEDIA_SHIELD_HOST;++ console.log('placement-shield: all checks passed');++ process.exit(0);++})().catch((err) => { console.error(err); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,49 @@
@@ -0,0 +1,49 @@
+# Media Fabric F1b — the DNS-only edge host (install in sites-enabled). Viewers' bytes come from here, not through
+# Cloudflare's proxy (its terms for video are not verified: plan T4). Node answers every request as on openvibe.media;
+# for a public B2 copy it answers with X-Accel-Redirect into /_media_shield/b2/ and nginx serves the bytes from the
+# slice cache (MEDIA_SHIELD=b2, MEDIA_SHIELD_HOST=edge.openvibe.media in /etc/openvibe/media.env).
+server {
+ listen 80;
+ listen [::]:80;
+ server_name edge.openvibe.media;
+ location /.well-known/acme-challenge/ { root /var/www/html; }
+ location / { return 301 https://edge.openvibe.media$request_uri; }
+}
+server {
+ listen 443 ssl http2;
+ listen [::]:443 ssl http2;
+ server_name edge.openvibe.media;
+ ssl_certificate /etc/letsencrypt/live/edge.openvibe.media/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/edge.openvibe.media/privkey.pem;
+
+ location /_media_shield/b2/ {
+ internal;
+ access_log /var/log/nginx/media-shield.log media_shield;
+ rewrite ^/_media_shield/b2/(.*)$ /$1 break;
+ slice 10m;
+ proxy_cache media_shield;
+ proxy_cache_key "b2|$uri|$slice_range";
+ proxy_cache_valid 200 206 24h;
+ proxy_cache_valid 404 1m;
+ # Single flight per object + slice: N viewers of the same range cost one B2 GET.
+ proxy_cache_lock on;
+ proxy_cache_lock_timeout 10s;
+ proxy_cache_lock_age 10s;
+ proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+ proxy_cache_background_update on;
+ add_header X-Cache-Status $upstream_cache_status always;
+ include snippets/media-shield-b2-upstream.conf;
+ }
+
+ # Everything else is Node, exactly as on openvibe.media (client-IP headers from $remote_addr only).
+ location / {
+ proxy_pass http://127.0.0.1:4100;
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header CF-Connecting-IP $remote_addr;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_request_buffering off;
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,15 @@
@@ -0,0 +1,15 @@
+# Media Fabric F1b — how the shield reaches B2 (install as /etc/nginx/snippets/media-shield-b2-upstream.conf).
+# B2 is addressed path-style, so every presigned URL is signed for this one host (SigV4 signs the Host header).
+proxy_set_header Range $slice_range;
+proxy_set_header If-Range "";
+proxy_set_header Host s3.us-west-004.backblazeb2.com;
+proxy_set_header Connection "";
+proxy_set_header Cookie "";
+proxy_set_header Authorization "";
+proxy_http_version 1.1;
+proxy_ssl_server_name on;
+proxy_ssl_name s3.us-west-004.backblazeb2.com;
+proxy_ignore_headers Cache-Control Expires Set-Cookie;
+proxy_connect_timeout 5s;
+proxy_read_timeout 60s;
+proxy_pass https://s3.us-west-004.backblazeb2.com;
new file mode 100644
new file mode 100644
@@ -0,0 +1,27 @@
@@ -0,0 +1,27 @@
+# Media Fabric F1b — origin shield, http-level part (install as /etc/nginx/conf.d/media-shield.conf).
+# One slice cache for B2 reads: 10 MB slices keyed by object path + slice, never by the presigned signature.
+# Sizing: the cache shares the disk with recordings, so it is small and yields to them: nginx evicts when free space
+# drops below 20 GB (recordings' guardian warns at 15 GB), and never holds more than 5 GB.
+proxy_cache_path /var/cache/nginx/media-shield levels=1:2 keys_zone=media_shield:64m max_size=5g min_free=20g inactive=24h use_temp_path=off;
+
+# The signed query is a bearer token: shield logs carry the path only ($uri), never $args or $request_uri.
+log_format media_shield '$remote_addr [$time_local] "$request_method $uri" $status $body_bytes_sent cache=$upstream_cache_status range="$http_range" rt=$request_time';
+
+# Purge (server/placement/shield.js purge): loopback only, bypasses the cache and stores the provider's answer for the
+# same key (a deleted object's 404 replaces its slices, cached for a minute).
+server {
+ listen 127.0.0.1:8479;
+ server_name _;
+ access_log /var/log/nginx/media-shield.log media_shield;
+ location /_media_shield_refresh/b2/ {
+ rewrite ^/_media_shield_refresh/b2/(.*)$ /$1 break;
+ slice 10m;
+ proxy_cache media_shield;
+ proxy_cache_key "b2|$uri|$slice_range";
+ proxy_cache_bypass 1;
+ proxy_cache_valid 200 206 24h;
+ proxy_cache_valid 403 404 1m;
+ include snippets/media-shield-b2-upstream.conf;
+ }
+ location / { return 404; }
+}
@@ -187,3 +187,21 @@ metric. Budgets per class and provider with a forecast; `provider.cost.threshold
@@ -187,3 +187,21 @@ metric. Budgets per class and provider with a forecast; `provider.cost.threshold
epochs, canary shifts with auto-revert, player QoE beacons, seek SLOs.
epochs, canary shifts with auto-revert, player QoE beacons, seek SLOs.
- **F6 cost truth:** bill imports, forecasts, budget guards, the owner dashboard.
- **F6 cost truth:** bill imports, forecasts, budget guards, the owner dashboard.
- **Archive fabric:** backups and snapshots to B2; optional deep archive with restore drills; never online.
- **Archive fabric:** backups and snapshots to B2; optional deep archive with restore drills; never online.
+
+## F1b decisions (2026-09-29)
+
+- **Shield B2 only.** B2 is the canonical copy whose ~500 req/s ceiling is worth shielding; R2 reads keep their 302
+ until F2's engine can enforce the 70 %-of-port rule (an R2 read through the shield spends the same host bandwidth).
+- **A DNS-only edge host.** openvibe.media is behind Cloudflare's proxy and its terms for video are not verified, so
+ the shield answers only on `edge.openvibe.media` (grey cloud; `MEDIA_SHIELD_HOST`). Bytes that went viewer → B2 never
+ start flowing through Cloudflare.
+- **Cache key:** `b2|<bucket>/<key>|<slice range>`: the object path and the 10 MB slice, never the signature. The
+ shield's presign lives an hour (it stays inside nginx and a long response fetches later slices with it); the
+ viewer-facing 302 keeps its short TTL.
+- **Private and sandbox bytes never enter the shield.**
+- **Sizing:** one disk with recordings: `max_size=5g`, `min_free=20g` (the recording guardian warns at 15 GB), `inactive=24h`.
+- **Purge:** deleting a shielded object refreshes each of its slices through a loopback-only listener
+ (`127.0.0.1:8479`, `proxy_cache_bypass`), so the provider's 404 replaces the cached bytes at once.
+- **Logs:** shield logs carry the path only: the signed query is a bearer token.
+- Config: `deploy/nginx/media-shield.http.conf` (conf.d), `media-shield-b2-upstream.conf` (snippets),
+ `edge.openvibe.media.conf` (sites-enabled). Switch on with `MEDIA_SHIELD=b2` and `MEDIA_SHIELD_HOST` after nginx has them.
@@ -773,6 +773,14 @@ publicRouter.get('/:id', async (req, res) => {
@@ -773,6 +773,14 @@ publicRouter.get('/:id', async (req, res) => {
if (fs.existsSync(loc.key)) return require('../public/routes').streamFileWithRange(req, res, loc.key, headers);
if (fs.existsSync(loc.key)) return require('../public/routes').streamFileWithRange(req, res, loc.key, headers);
continue;
continue;
}
}
+ // The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through its
+ // slice cache, with an hour-long internal presign that never reaches the viewer.
+ const shield = require('../placement/shield');
+ if (shield.enabled(provider) && !sandbox && obj.visibility !== 'private') {
+ const inner = await require('../vod/vod-storage').presignGet(provider, loc.key, shield.SHIELD_TTL_SECONDS, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);
+ const t = inner && shield.target({ provider, url: inner, visibility: obj.visibility, sandbox, host: req.hostname });
+ if (t) return shield.send(res, t, headers);
+ }
// The router already signed its first choice; a fallback copy is signed here.
// The router already signed its first choice; a fallback copy is signed here.
const url = (provider === decision.provider && decision.url) || await require('../vod/vod-storage').presignGet(provider, loc.key, 300, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);
const url = (provider === decision.provider && decision.url) || await require('../vod/vod-storage').presignGet(provider, loc.key, 300, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);
if (url) { res.set('Cache-Control', 'private, max-age=0'); res.set('X-Robots-Tag', 'noindex'); return res.redirect(302, url); }
if (url) { res.set('Cache-Control', 'private, max-age=0'); res.set('X-Robots-Tag', 'noindex'); return res.redirect(302, url); }
new file mode 100644
new file mode 100644
@@ -0,0 +1,81 @@
@@ -0,0 +1,81 @@
+/**
+ * OpenVibe.Media — the origin shield (Media Fabric F1b, docs/media-fabric.md "origin shield").
+ *
+ * Node still authorizes every read and picks the copy (the placement router); for a public copy on a shielded provider
+ * it no longer redirects the viewer to a presigned URL. It answers with `X-Accel-Redirect` into nginx's internal
+ * `/_media_shield/<provider>/…` location, which proxies the presigned request through a slice cache (10 MB slices,
+ * `proxy_cache_lock`): N viewers of the same object range cost one provider GET, the bytes come from the Media host's
+ * prepaid bandwidth, and the provider URL never reaches the viewer. The cache key is the object path and the slice,
+ * never the signature (deploy/nginx/openvibe.media.conf).
+ *
+ * MEDIA_SHIELD=b2 providers whose reads go through the shield (comma-separated; empty = off). Set it only
+ * once nginx has the locations: a Node that emits X-Accel to an nginx without them 404s.
+ * MEDIA_SHIELD_HOST=edge.openvibe.media the ONLY host the shield answers on: a DNS-only (grey-cloud) edge name.
+ * openvibe.media is behind Cloudflare's proxy, whose terms for video are not verified (plan
+ * T4): bytes that went viewer → B2 must not start flowing through Cloudflare. Unset = off.
+ * MEDIA_SHIELD_REFRESH_URL the loopback-only purge listener (default http://127.0.0.1:8479).
+ *
+ * Rules: private and sandbox bytes never enter the shared cache; B2 only for now (path-style URLs, one signed host; R2
+ * waits for F2's port-utilization guard); the shield's presign lives an hour (it stays inside nginx, and a long response
+ * fetches later slices with it), the viewer-facing 302 keeps its short TTL.
+ *
+ * target({ provider, url, visibility, sandbox }) → '/_media_shield/b2/<bucket>/<key>?<signed query>' | null
+ * send(res, target, headers) → the 200 + X-Accel-Redirect answer
+ * await purge({ provider, key, size, presign }) → refreshes every cached slice of a deleted or replaced object
+ */
+'use strict';
+
+const SLICE_BYTES = 10 * 1024 * 1024; // nginx `slice 10m`
+const SHIELD_TTL_SECONDS = 3600;
+
+function providers() {
+ return String(process.env.MEDIA_SHIELD || '').split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
+}
+function enabled(provider) { return providers().includes(String(provider || '').toLowerCase()); }
+
+/** The internal location for one presigned URL, or null when this read must not go through the shield. `host` is the
+ * request's host: the shield answers only on MEDIA_SHIELD_HOST (a DNS-only edge name), never through Cloudflare. */
+function target({ provider, url, visibility = 'public', sandbox = false, host = null }) {
+ if (!url || !enabled(provider)) return null;
+ const edge = String(process.env.MEDIA_SHIELD_HOST || '').toLowerCase();
+ if (!edge || String(host || '').toLowerCase() !== edge) return null;
+ if (visibility === 'private' || sandbox) return null; // never into a shared cache
+ let u; try { u = new URL(url); } catch { return null; }
+ // Path-style URLs only: the pathname is /<bucket>/<key>, the host is the provider's one signed endpoint.
+ if (!u.pathname || u.pathname === '/') return null;
+ return `/_media_shield/${String(provider).toLowerCase()}${u.pathname}${u.search}`;
+}
+
+/** Hand the read to nginx. Headers the route computed (type, disposition, cache policy) go with it. */
+function send(res, targetPath, headers = {}) {
+ for (const [k, v] of Object.entries(headers)) if (v != null) res.set(k, v);
+ res.set('X-Accel-Redirect', targetPath);
+ res.set('X-Media-Source', 'shield');
+ return res.status(200).end();
+}
+
+/**
+ * Refresh every cached slice of an object after it was deleted or replaced on a shielded provider: one request per
+ * 10 MB slice to nginx's loopback-only refresh location, which bypasses the cache and stores the provider's answer
+ * (a 404 for a deleted object: cached for a minute, then gone). Never throws; returns what it did.
+ */
+async function purge({ provider, key, size, presign, fetchImpl = globalThis.fetch, base = process.env.MEDIA_SHIELD_REFRESH_URL || 'http://127.0.0.1:8479', host = process.env.MEDIA_SHIELD_HOST || 'localhost' }) {
+ if (!enabled(provider) || !key) return { purged: 0, skipped: true };
+ let url; try { url = await presign(provider, key, 300); } catch (err) { return { purged: 0, error: err.message }; }
+ const t = target({ provider, url, host: process.env.MEDIA_SHIELD_HOST });
+ if (!t) return { purged: 0, skipped: true };
+ const refresh = t.replace('/_media_shield/', '/_media_shield_refresh/');
+ const slices = Math.max(1, Math.ceil((Number(size) || SLICE_BYTES) / SLICE_BYTES));
+ let purged = 0, failed = 0;
+ for (let i = 0; i < slices; i++) {
+ const range = `bytes=${i * SLICE_BYTES}-${(i + 1) * SLICE_BYTES - 1}`;
+ try {
+ const r = await fetchImpl(base + refresh, { headers: { Host: host, Range: range }, signal: AbortSignal.timeout(15000) });
+ try { await r.arrayBuffer(); } catch { /* drained */ }
+ purged++;
+ } catch { failed++; }
+ }
+ return { purged, failed, slices };
+}
+
+module.exports = { target, send, purge, enabled, providers, SLICE_BYTES, SHIELD_TTL_SECONDS };
@@ -174,6 +174,14 @@ async function serveMediaRecord(kind, record, req, res) {
@@ -174,6 +174,14 @@ async function serveMediaRecord(kind, record, req, res) {
const router = require('../placement/router');
const router = require('../placement/router');
const plan = await vodStorage.resolvePlayback(record, { session: router.sessionFor(req, `${kind}:${record.id}`) });
const plan = await vodStorage.resolvePlayback(record, { session: router.sessionFor(req, `${kind}:${record.id}`) });
if (plan?.kind === 'redirect') {
if (plan?.kind === 'redirect') {
+ // The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through
+ // its slice cache with an hour-long internal presign; everything else keeps the short-lived 302.
+ const shield = require('../placement/shield');
+ if (shield.enabled(plan.provider) && visibility !== 'private' && !record.is_recording) {
+ const url = await vodStorage.presignGet(plan.provider, vodStorage.keyForVod(record), shield.SHIELD_TTL_SECONDS).catch(() => null);
+ const t = url && shield.target({ provider: plan.provider, url, visibility, host: req.hostname });
+ if (t) return shield.send(res, t, { 'Cache-Control': visibility === 'public' ? 'public, max-age=300' : 'private, max-age=0', 'X-Robots-Tag': 'noindex' });
+ }
res.set('Cache-Control', 'private, max-age=0');
res.set('Cache-Control', 'private, max-age=0');
res.set('X-Robots-Tag', 'noindex');
res.set('X-Robots-Tag', 'noindex');
return res.redirect(302, plan.url);
return res.redirect(302, plan.url);
@@ -396,11 +396,19 @@ async function sha256Object(provider, key) {
@@ -396,11 +396,19 @@ async function sha256Object(provider, key) {
async function deleteObject(provider, key) {
async function deleteObject(provider, key) {
const client = clientFor(provider);
const client = clientFor(provider);
if (!client) return;
if (!client) return;
+ // A shielded provider's cached slices leave the shield with the object (placement/shield.js): the size decides
+ // how many slices to refresh, so it is read before the delete.
+ const shield = require('../placement/shield');
+ const head = shield.enabled(provider) ? await headObject(provider, key).catch(() => null) : null;
try {
try {
await client.send(new S3.DeleteObjectCommand({ Bucket: PROVIDER_ENV[provider].bucket, Key: key }));
await client.send(new S3.DeleteObjectCommand({ Bucket: PROVIDER_ENV[provider].bucket, Key: key }));
} catch (err) {
} catch (err) {
console.warn(`[VodStorage] Delete ${provider}:${key} failed:`, err.message);
console.warn(`[VodStorage] Delete ${provider}:${key} failed:`, err.message);
+ return;
}
}
+ if (head) shield.purge({ provider, key, size: head.size, presign: presignGet })
+ .then((r) => { if (r.failed) console.warn(`[VodStorage] shield purge ${provider}:${key}: ${r.failed} of ${r.slices} slices not refreshed`); })
+ .catch(() => { /* purge never throws */ });
}
}
/**
/**
new file mode 100644
new file mode 100644
@@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
+'use strict';
+// Origin shield (Media Fabric F1b): which reads nginx serves from its slice cache, and the purge of a deleted object.
+const assert = require('assert');
+
+(async () => {
+ const shield = require('../server/placement/shield');
+ const url = 'https://s3.us-west-004.backblazeb2.com/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc';
+
+ delete process.env.MEDIA_SHIELD;
+ assert.strictEqual(shield.target({ provider: 'b2', url }), null, 'off unless MEDIA_SHIELD names the provider');
+
+ process.env.MEDIA_SHIELD = 'b2';
+ assert.strictEqual(shield.target({ provider: 'b2', url, host: 'edge.openvibe.media' }), null, 'no edge host configured: off');
+ process.env.MEDIA_SHIELD_HOST = 'edge.openvibe.media';
+ assert.strictEqual(shield.target({ provider: 'b2', url, host: 'openvibe.media' }), null, 'never on the Cloudflare-proxied host');
+ const host = 'edge.openvibe.media';
+ assert.strictEqual(shield.target({ provider: 'b2', url, host }), '/_media_shield/b2/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc',
+ 'the path-style path and the signed query ride into the internal location unchanged');
+ assert.strictEqual(shield.target({ provider: 'r2', url, host }), null, 'R2 is not shielded in F1b');
+ assert.strictEqual(shield.target({ provider: 'b2', url, host, visibility: 'private' }), null, 'private bytes never enter the shared cache');
+ assert.strictEqual(shield.target({ provider: 'b2', url, host, sandbox: true }), null, 'sandbox bytes never enter it either');
+ assert.strictEqual(shield.target({ provider: 'b2', url: 'not a url', host }), null);
+ assert.strictEqual(shield.target({ provider: 'b2', url: 'https://s3.example.com/', host }), null, 'a URL without an object path is not shielded');
+
+ // send(): the route's headers plus the internal redirect, an empty 200.
+ const res = { h: {}, set(k, v) { this.h[k] = v; }, status(c) { this.code = c; return this; }, end() { this.ended = true; return this; } };
+ shield.send(res, '/_media_shield/b2/x', { 'Content-Type': 'video/mp4', 'Cache-Control': 'public, max-age=300', Skip: null });
+ assert.deepStrictEqual([res.code, res.ended, res.h['X-Accel-Redirect'], res.h['Content-Type'], 'Skip' in res.h], [200, true, '/_media_shield/b2/x', 'video/mp4', false]);
+
+ // purge(): one refresh request per 10 MB slice, loopback, with the public Host and each slice's exact range.
+ const calls = [];
+ const fetchImpl = async (u, opts) => { calls.push([u, opts.headers.Host, opts.headers.Range]); return { arrayBuffer: async () => new ArrayBuffer(0) }; };
+ const r = await shield.purge({ provider: 'b2', key: 'vods/a b.mp4', size: 25 * 1024 * 1024, presign: async () => url, fetchImpl });
+ assert.deepStrictEqual([r.purged, r.slices, r.failed], [3, 3, 0]);
+ assert.ok(calls.every(([u, h]) => u.startsWith('http://127.0.0.1:8479/_media_shield_refresh/b2/ov-vods/vods/a%20b.mp4?') && h === 'edge.openvibe.media'), JSON.stringify(calls[0]));
+ assert.deepStrictEqual(calls.map((c) => c[2]), ['bytes=0-10485759', 'bytes=10485760-20971519', 'bytes=20971520-31457279']);
+ assert.deepStrictEqual(await shield.purge({ provider: 'r2', key: 'k', size: 1, presign: async () => url, fetchImpl }), { purged: 0, skipped: true }, 'nothing to purge off the shield');
+
+ delete process.env.MEDIA_SHIELD; delete process.env.MEDIA_SHIELD_HOST;
+ console.log('placement-shield: all checks passed');
+ process.exit(0);
+})().catch((err) => { console.error(err); process.exit(1); });