openvibers/openvibe.media Commits
Origin shield (F1b) review fixes, still inert until MEDIA_SHIELD is set. The edge is known from X-Media-Shield-Host, which only the edge server block sets and openvibe.media clears (req.hostname honoured a client's X-Forwarded-Host), and routes check it before presigning. A URL signed for any B2 host but the one nginx proxies to keeps its 302 (a changed MEDIA_B2_ENDPOINT would have 403'd every slice); a test pins shield.js and the nginx snippet to the same host. Overwriting a shielded key (a repair, or a re-upload within ten minutes of a delete) refreshes its slices after the upload; purges run four slices at a time and send no dead Host header. nginx: the edge block gets openvibe.media's body size, timeouts and /metrics 404; the refresh listener is loopback-only (allow/deny); the shield location hides B2's Content-Disposition, Cache-Control, Expires, Set-Cookie and request ids and adds nosniff and noindex. The internal presign lives six hours. docs/media-fabric.md F1b decisions updated. npm test 74/75 (N-1 skipped as designed).
Changes
@@@ -16,6 +16,14 @@ server {ssl_certificate /etc/letsencrypt/live/edge.openvibe.media/fullchain.pem;ssl_certificate_key /etc/letsencrypt/live/edge.openvibe.media/privkey.pem;++ # As on openvibe.media: large uploads, long Node responses (DVR, recordings, SSE).++ client_max_body_size 4096m;++ proxy_read_timeout 3600s;++ proxy_send_timeout 3600s;++++ # Prometheus metrics are for loopback scrapers only (the app 404s proxied callers too).++ location = /metrics { return 404; }++location /_media_shield/b2/ {internal;access_log /var/log/nginx/media-shield.log media_shield;@@@ -31,15 +39,29 @@ server {proxy_cache_lock_age 10s;proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;proxy_cache_background_update on;++ # Node's answer owns the viewer-facing headers (Content-Type, Content-Disposition, Cache-Control survive the++ # X-Accel redirect): B2's copies, and its request ids, are dropped. The security headers Node sets do not survive++ # the redirect, so they are set here.++ proxy_hide_header Content-Disposition;++ proxy_hide_header Cache-Control;++ proxy_hide_header Expires;++ proxy_hide_header Set-Cookie;++ proxy_hide_header x-amz-request-id;++ proxy_hide_header x-amz-id-2;++ add_header X-Content-Type-Options nosniff always;++ add_header X-Robots-Tag noindex always;add_header X-Cache-Status $upstream_cache_status always;include snippets/media-shield-b2-upstream.conf;}-- # Everything else is Node, exactly as on openvibe.media (client-IP headers from $remote_addr only).++ # Everything else is Node, exactly as on openvibe.media (client-IP headers from $remote_addr only), plus the one++ # header that tells Node this is the edge: nginx sets it from its own server name, so a client cannot forge it.location / {proxy_pass http://127.0.0.1:4100;proxy_http_version 1.1;proxy_set_header Host $host;++ proxy_set_header X-Media-Shield-Host $server_name;++ proxy_set_header X-Forwarded-Host "";proxy_set_header X-Real-IP $remote_addr;proxy_set_header X-Forwarded-For $remote_addr;proxy_set_header CF-Connecting-IP $remote_addr;@@@ -1,5 +1,7 @@# Media Fabric F1b — how the shield reaches B2 (install as /etc/nginx/snippets/media-shield-b2-upstream.conf).--# B2 is addressed path-style, so every presigned URL is signed for this one host (SigV4 signs the Host header).++# B2 is addressed path-style, so every presigned URL is signed for this one host (SigV4 signs the Host header). It must++# equal server/placement/shield.js B2_UPSTREAM_HOST (test/placement-shield.test.js pins both) and the host of++# MEDIA_B2_ENDPOINT: a URL signed for any other host is never sent here (Node keeps its 302).proxy_set_header Range $slice_range;proxy_set_header If-Range "";proxy_set_header Host s3.us-west-004.backblazeb2.com;@@@ -14,6 +14,9 @@ server {server_name _;access_log /var/log/nginx/media-shield.log media_shield;location /_media_shield_refresh/b2/ {++ # Node calls this over real HTTP, so it cannot be `internal`: only loopback may force cache writes.++ allow 127.0.0.1;++ deny all;rewrite ^/_media_shield_refresh/b2/(.*)$ /$1 break;slice 10m;proxy_cache media_shield;@@@ -24,6 +24,10 @@ server {proxy_pass http://127.0.0.1:4100;proxy_http_version 1.1;proxy_set_header Host $host;++ # Only the edge host's block may say "edge" (placement/shield.js): cleared here so a client cannot send it++ # through Cloudflare; X-Forwarded-Host is cleared too (Express's req.hostname would honour it).++ proxy_set_header X-Media-Shield-Host "";++ proxy_set_header X-Forwarded-Host "";proxy_set_header X-Real-IP $remote_addr;proxy_set_header X-Forwarded-For $remote_addr;proxy_set_header CF-Connecting-IP $remote_addr;@@@ -194,14 +194,26 @@ metric. Budgets per class and provider with a forecast; `provider.cost.thresholduntil F2's engine can enforce the 70 %-of-port rule (an R2 read through the shield spends the same host bandwidth).- **A DNS-only edge host.** openvibe.media is behind Cloudflare's proxy and its terms for video are not verified, sothe shield answers only on `edge.openvibe.media` (grey cloud; `MEDIA_SHIELD_HOST`). Bytes that went viewer → B2 never-- start flowing through Cloudflare.++ start flowing through Cloudflare. Node knows the edge from `X-Media-Shield-Host`, which only the edge server block sets++ (from its own server name) and `openvibe.media` clears; never from Host or `X-Forwarded-Host`, which a client chooses.++- **One B2 host.** nginx proxies to one B2 endpoint and SigV4 signs the Host, so a URL signed for any other host (a++ changed `MEDIA_B2_ENDPOINT`) is never shielded and keeps its 302 (`shield.js` `B2_UPSTREAM_HOST` = the nginx snippet;++ a test pins both).- **Cache key:** `b2|<bucket>/<key>|<slice range>`: the object path and the 10 MB slice, never the signature. The-- shield's presign lives an hour (it stays inside nginx and a long response fetches later slices with it); the-- viewer-facing 302 keeps its short TTL.++ shield's presign lives six hours (it stays inside nginx and a long response fetches later slices with it; a single++ response longer than that gets an uncached 403 on its next slice); the viewer-facing 302 keeps its short TTL.++- **Headers:** Node's answer owns Content-Type, Content-Disposition and Cache-Control (they survive the X-Accel++ redirect; B2's copies are hidden); the shield location adds `nosniff` and `noindex` itself. Unlisted objects are served++ through the shared cache (anyone with the link may read them; `/o` answers `public, max-age=3600` for them), unlisted++ VODs answer `private, max-age=0`.- **Private and sandbox bytes never enter the shield.**- **Sizing:** one disk with recordings: `max_size=5g`, `min_free=20g` (the recording guardian warns at 15 GB), `inactive=24h`.--- **Purge:** deleting a shielded object refreshes each of its slices through a loopback-only listener-- (`127.0.0.1:8479`, `proxy_cache_bypass`), so the provider's 404 replaces the cached bytes at once.++- **Purge:** deleting a shielded object refreshes each of its slices (four at a time) through a loopback-only listener++ (`127.0.0.1:8479`, `allow 127.0.0.1; deny all`, `proxy_cache_bypass`), so the provider's 404 replaces the cached bytes++ at once. Overwriting a shielded key (a repair re-upload, or a re-upload within ten minutes of a delete) refreshes it++ after the upload, so the shield never serves the old bytes.- **Logs:** shield logs carry the path only: the signed query is a bearer token.- Config: `deploy/nginx/media-shield.http.conf` (conf.d), `media-shield-b2-upstream.conf` (snippets),-- `edge.openvibe.media.conf` (sites-enabled). Switch on with `MEDIA_SHIELD=b2` and `MEDIA_SHIELD_HOST` after nginx has them.++ `edge.openvibe.media.conf` (sites-enabled), and `openvibe.media.conf` clearing `X-Media-Shield-Host`. Switch on with++ `MEDIA_SHIELD=b2` and `MEDIA_SHIELD_HOST` after nginx has them, and check a real slice with `curl -I` on the edge++ (one Content-Type, Node's Content-Disposition, `X-Cache-Status`, no `x-amz-*`).@@@ -774,11 +774,11 @@ publicRouter.get('/:id', async (req, res) => {continue;}// The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through its-- // slice cache, with an hour-long internal presign that never reaches the viewer.++ // slice cache, with a long internal presign that never reaches the viewer (only on the DNS-only edge host).const shield = require('../placement/shield');-- if (shield.enabled(provider) && !sandbox && obj.visibility !== 'private') {++ if (shield.enabled(provider) && shield.onEdge(req) && !sandbox && obj.visibility !== 'private') {const inner = await require('../vod/vod-storage').presignGet(provider, loc.key, shield.SHIELD_TTL_SECONDS, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);-- const t = inner && shield.target({ provider, url: inner, visibility: obj.visibility, sandbox, host: req.hostname });++ const t = inner && shield.target({ provider, url: inner, visibility: obj.visibility, sandbox, edge: shield.edgeOf(req) });if (t) return shield.send(res, t, headers);}// The router already signed its first choice; a fallback copy is signed here.@@@ -6,7 +6,7 @@* `/_media_shield/<provider>/…` location, which proxies the presigned request through a slice cache (10 MB slices,* `proxy_cache_lock`): N viewers of the same object range cost one provider GET, the bytes come from the Media host's* prepaid bandwidth, and the provider URL never reaches the viewer. The cache key is the object path and the slice,-- * never the signature (deploy/nginx/openvibe.media.conf).++ * never the signature (deploy/nginx/media-shield.http.conf, edge.openvibe.media.conf, media-shield-b2-upstream.conf).** MEDIA_SHIELD=b2 providers whose reads go through the shield (comma-separated; empty = off). Set it only* once nginx has the locations: a Node that emits X-Accel to an nginx without them 404s.@@@ -15,33 +15,51 @@* T4): bytes that went viewer → B2 must not start flowing through Cloudflare. Unset = off.* MEDIA_SHIELD_REFRESH_URL the loopback-only purge listener (default http://127.0.0.1:8479).*++ * The edge is known from `X-Media-Shield-Host`, which only the edge server block sets (from its own $host) and every++ * other Media server block clears: never from Host or X-Forwarded-Host, which a client can choose (Express's++ * req.hostname honours X-Forwarded-Host under `trust proxy`). nginx proxies to ONE B2 host (B2_UPSTREAM_HOST, equal to++ * deploy/nginx/media-shield-b2-upstream.conf, a test pins it) and SigV4 signs the Host header, so a URL signed for any++ * other host (MEDIA_B2_ENDPOINT changed) is never shielded: it keeps its 302 instead of failing every slice with a 403.++ ** Rules: private and sandbox bytes never enter the shared cache; B2 only for now (path-style URLs, one signed host; R2-- * waits for F2's port-utilization guard); the shield's presign lives an hour (it stays inside nginx, and a long response-- * fetches later slices with it), the viewer-facing 302 keeps its short TTL.++ * waits for F2's port-utilization guard); the shield's presign lives six hours (it stays inside nginx, and a long++ * response fetches later slices with it), the viewer-facing 302 keeps its short TTL.*-- * target({ provider, url, visibility, sandbox }) → '/_media_shield/b2/<bucket>/<key>?<signed query>' | null++ * onEdge(req) → true on the edge host (check it before presigning anything)++ * target({ provider, url, visibility, sandbox, edge }) → '/_media_shield/b2/<bucket>/<key>?<signed query>' | null* send(res, target, headers) → the 200 + X-Accel-Redirect answer* await purge({ provider, key, size, presign }) → refreshes every cached slice of a deleted or replaced object*/'use strict';const SLICE_BYTES = 10 * 1024 * 1024; // nginx `slice 10m`--const SHIELD_TTL_SECONDS = 3600;++const SHIELD_TTL_SECONDS = 6 * 3600; // inside nginx only; covers a long response's later slices++const B2_UPSTREAM_HOST = 's3.us-west-004.backblazeb2.com'; // = deploy/nginx/media-shield-b2-upstream.conf++const UPSTREAM_HOSTS = { b2: B2_UPSTREAM_HOST };++const EDGE_HEADER = 'x-media-shield-host'; // set by the edge server block only (nginx)++const PURGE_CONCURRENCY = 4;function providers() {return String(process.env.MEDIA_SHIELD || '').split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);}function enabled(provider) { return providers().includes(String(provider || '').toLowerCase()); }--/** The internal location for one presigned URL, or null when this read must not go through the shield. `host` is the-- * request's host: the shield answers only on MEDIA_SHIELD_HOST (a DNS-only edge name), never through Cloudflare. */--function target({ provider, url, visibility = 'public', sandbox = false, host = null }) {++function edgeHost() { return String(process.env.MEDIA_SHIELD_HOST || '').trim().toLowerCase(); }++/** The edge name nginx vouched for on this request (X-Media-Shield-Host), or null. */++function edgeOf(req) { const v = req && typeof req.get === 'function' ? req.get(EDGE_HEADER) : null; return v ? String(v).trim().toLowerCase() : null; }++/** True when this request came in on the edge host: routes check it before presigning anything for the shield. */++function onEdge(req) { const want = edgeHost(); return !!want && edgeOf(req) === want; }++++/** The internal location for one presigned URL, or null when this read must not go through the shield. `edge` is++ * edgeOf(req): the shield answers only on MEDIA_SHIELD_HOST (a DNS-only edge name), never through Cloudflare. */++function target({ provider, url, visibility = 'public', sandbox = false, edge = null }) {if (!url || !enabled(provider)) return null;-- const edge = String(process.env.MEDIA_SHIELD_HOST || '').toLowerCase();-- if (!edge || String(host || '').toLowerCase() !== edge) return null;++ const want = edgeHost();++ if (!want || String(edge || '').toLowerCase() !== want) return null;if (visibility === 'private' || sandbox) return null; // never into a shared cachelet u; try { u = new URL(url); } catch { return null; }-- // Path-style URLs only: the pathname is /<bucket>/<key>, the host is the provider's one signed endpoint.++ // Path-style URLs for the one host nginx proxies to (the signature covers Host): /<bucket>/<key>.++ if (u.host.toLowerCase() !== UPSTREAM_HOSTS[String(provider).toLowerCase()]) return null;if (!u.pathname || u.pathname === '/') return null;return `/_media_shield/${String(provider).toLowerCase()}${u.pathname}${u.search}`;}@@@ -56,26 +74,31 @@ function send(res, targetPath, headers = {}) {/*** Refresh every cached slice of an object after it was deleted or replaced on a shielded provider: one request per-- * 10 MB slice to nginx's loopback-only refresh location, which bypasses the cache and stores the provider's answer-- * (a 404 for a deleted object: cached for a minute, then gone). Never throws; returns what it did.++ * 10 MB slice to nginx's loopback-only refresh listener (a few at a time), which bypasses the cache and stores the++ * provider's answer (new bytes for a replaced object; a 404 for a deleted one, cached for a minute). Never throws;++ * returns what it did. The listener is the only server on its port, so no Host header is needed (fetch drops it).*/--async function purge({ provider, key, size, presign, fetchImpl = globalThis.fetch, base = process.env.MEDIA_SHIELD_REFRESH_URL || 'http://127.0.0.1:8479', host = process.env.MEDIA_SHIELD_HOST || 'localhost' }) {-- if (!enabled(provider) || !key) return { purged: 0, skipped: true };++async function purge({ provider, key, size, presign, fetchImpl = globalThis.fetch, base = process.env.MEDIA_SHIELD_REFRESH_URL || 'http://127.0.0.1:8479', concurrency = PURGE_CONCURRENCY }) {++ if (!enabled(provider) || !key || !edgeHost()) return { purged: 0, skipped: true };let url; try { url = await presign(provider, key, 300); } catch (err) { return { purged: 0, error: err.message }; }-- const t = target({ provider, url, host: process.env.MEDIA_SHIELD_HOST });++ const t = target({ provider, url, edge: edgeHost() });if (!t) return { purged: 0, skipped: true };const refresh = t.replace('/_media_shield/', '/_media_shield_refresh/');const slices = Math.max(1, Math.ceil((Number(size) || SLICE_BYTES) / SLICE_BYTES));-- let purged = 0, failed = 0;-- for (let i = 0; i < slices; i++) {-- const range = `bytes=${i * SLICE_BYTES}-${(i + 1) * SLICE_BYTES - 1}`;-- try {-- const r = await fetchImpl(base + refresh, { headers: { Host: host, Range: range }, signal: AbortSignal.timeout(15000) });-- try { await r.arrayBuffer(); } catch { /* drained */ }-- purged++;-- } catch { failed++; }++ let purged = 0, failed = 0, next = 0;++ async function worker() {++ while (next < slices) {++ const i = next++;++ const range = `bytes=${i * SLICE_BYTES}-${(i + 1) * SLICE_BYTES - 1}`;++ try {++ const r = await fetchImpl(base + refresh, { headers: { Range: range }, signal: AbortSignal.timeout(15000) });++ try { await r.arrayBuffer(); } catch { /* drained */ }++ purged++;++ } catch { failed++; }++ }}++ await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, slices)) }, worker));return { purged, failed, slices };}--module.exports = { target, send, purge, enabled, providers, SLICE_BYTES, SHIELD_TTL_SECONDS };++module.exports = { target, send, purge, enabled, providers, onEdge, edgeOf, SLICE_BYTES, SHIELD_TTL_SECONDS, B2_UPSTREAM_HOST, EDGE_HEADER };@@@ -175,11 +175,12 @@ async function serveMediaRecord(kind, record, req, res) {const plan = await vodStorage.resolvePlayback(record, { session: router.sessionFor(req, `${kind}:${record.id}`) });if (plan?.kind === 'redirect') {// The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through-- // its slice cache with an hour-long internal presign; everything else keeps the short-lived 302.++ // its slice cache with a long internal presign (only on the DNS-only edge host); everything else keeps the++ // short-lived 302.const shield = require('../placement/shield');-- if (shield.enabled(plan.provider) && visibility !== 'private' && !record.is_recording) {++ if (shield.enabled(plan.provider) && shield.onEdge(req) && visibility !== 'private' && !record.is_recording) {const url = await vodStorage.presignGet(plan.provider, vodStorage.keyForVod(record), shield.SHIELD_TTL_SECONDS).catch(() => null);-- const t = url && shield.target({ provider: plan.provider, url, visibility, host: req.hostname });++ const t = url && shield.target({ provider: plan.provider, url, visibility, edge: shield.edgeOf(req) });if (t) return shield.send(res, t, { 'Cache-Control': visibility === 'public' ? 'public, max-age=300' : 'private, max-age=0', 'X-Robots-Tag': 'noindex' });}res.set('Cache-Control', 'private, max-age=0');@@@ -310,6 +310,10 @@ async function uploadFile(provider, key, filePath, contentType = 'video/webm') {if (!client) throw new Error(`Provider ${provider} not configured`);loadSdk();const size = fs.statSync(filePath).size;++ // A shielded key that already exists (a repair re-upload) or was deleted moments ago may have slices in the shield:++ // they are refreshed after the upload, or the shield would serve the old bytes for up to a day.++ const shield = require('../placement/shield');++ const before = shield.enabled(provider) ? await headObject(provider, key).catch(() => null) : null;// A hung multipart upload used to hold the sweep lock indefinitely (nothing else// could drain). Abort past the deadline so the sweep moves on to the next VOD;// leavePartsOnError cleans the partial multipart up on the bucket.@@@ -336,6 +340,12 @@ async function uploadFile(provider, key, filePath, contentType = 'video/webm') {} finally {clearTimeout(deadline);}++ const deletedSize = recentlyDeleted.get(`${provider}:${key}`);++ if (before || deletedSize != null) {++ recentlyDeleted.delete(`${provider}:${key}`);++ const r = await shield.purge({ provider, key, size: Math.max(size, (before && before.size) || 0, deletedSize || 0), presign: presignGet });++ if (r.failed) console.warn(`[VodStorage] shield refresh ${provider}:${key}: ${r.failed} of ${r.slices} slices not refreshed`);++ }// Verify size before anything destructive happensconst localSize = fs.statSync(filePath).size;const head = await headObject(provider, key);@@@ -393,6 +403,14 @@ async function sha256Object(provider, key) {return { sha256: hash.digest('hex'), size };}++// Shielded keys deleted in the last ten minutes, with their size: a re-upload of the same key refreshes the shield++// even though its HEAD finds nothing (the delete's own refresh may still be running and cache a 404 for a minute).++const recentlyDeleted = new Map();++function rememberDeleted(id, size) {++ recentlyDeleted.set(id, size);++ setTimeout(() => { if (recentlyDeleted.get(id) === size) recentlyDeleted.delete(id); }, 10 * 60e3).unref?.();++}++async function deleteObject(provider, key) {const client = clientFor(provider);if (!client) return;@@@ -406,6 +424,7 @@ async function deleteObject(provider, key) {console.warn(`[VodStorage] Delete ${provider}:${key} failed:`, err.message);return;}++ if (head) rememberDeleted(`${provider}:${key}`, head.size);if (head) shield.purge({ provider, key, size: head.size, presign: presignGet }).then((r) => { if (r.failed) console.warn(`[VodStorage] shield purge ${provider}:${key}: ${r.failed} of ${r.slices} slices not refreshed`); }).catch(() => { /* purge never throws */ });@@@ -10,30 +10,59 @@ const assert = require('assert');assert.strictEqual(shield.target({ provider: 'b2', url }), null, 'off unless MEDIA_SHIELD names the provider');process.env.MEDIA_SHIELD = 'b2';-- assert.strictEqual(shield.target({ provider: 'b2', url, host: 'edge.openvibe.media' }), null, 'no edge host configured: off');++ assert.strictEqual(shield.target({ provider: 'b2', url, edge: 'edge.openvibe.media' }), null, 'no edge host configured: off');process.env.MEDIA_SHIELD_HOST = 'edge.openvibe.media';-- assert.strictEqual(shield.target({ provider: 'b2', url, host: 'openvibe.media' }), null, 'never on the Cloudflare-proxied host');-- const host = 'edge.openvibe.media';-- assert.strictEqual(shield.target({ provider: 'b2', url, host }), '/_media_shield/b2/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc',++ assert.strictEqual(shield.target({ provider: 'b2', url, edge: null }), null, 'never on the Cloudflare-proxied host (nginx sends no edge header there)');++ const edge = 'edge.openvibe.media';++ assert.strictEqual(shield.target({ provider: 'b2', url, edge }), '/_media_shield/b2/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc','the path-style path and the signed query ride into the internal location unchanged');-- assert.strictEqual(shield.target({ provider: 'r2', url, host }), null, 'R2 is not shielded in F1b');-- assert.strictEqual(shield.target({ provider: 'b2', url, host, visibility: 'private' }), null, 'private bytes never enter the shared cache');-- assert.strictEqual(shield.target({ provider: 'b2', url, host, sandbox: true }), null, 'sandbox bytes never enter it either');-- assert.strictEqual(shield.target({ provider: 'b2', url: 'not a url', host }), null);-- assert.strictEqual(shield.target({ provider: 'b2', url: 'https://s3.example.com/', host }), null, 'a URL without an object path is not shielded');++ assert.strictEqual(shield.target({ provider: 'r2', url, edge }), null, 'R2 is not shielded in F1b');++ assert.strictEqual(shield.target({ provider: 'b2', url, edge, visibility: 'private' }), null, 'private bytes never enter the shared cache');++ assert.strictEqual(shield.target({ provider: 'b2', url, edge, sandbox: true }), null, 'sandbox bytes never enter it either');++ assert.strictEqual(shield.target({ provider: 'b2', url: 'not a url', edge }), null);++ assert.strictEqual(shield.target({ provider: 'b2', url: `https://${shield.B2_UPSTREAM_HOST}/`, edge }), null, 'a URL without an object path is not shielded');++ // SigV4 signs the Host: a URL signed for another B2 endpoint would 403 on every slice through nginx, so it keeps its 302.++ assert.strictEqual(shield.target({ provider: 'b2', url: url.replace('s3.us-west-004', 's3.eu-central-003'), edge }), null, 'another B2 host is never shielded');++++ // The edge comes from the header only the edge server block sets: never from Host or X-Forwarded-Host.++ const req = (h) => ({ get: (k) => h[k.toLowerCase()] });++ assert.strictEqual(shield.onEdge(req({ 'x-media-shield-host': 'edge.openvibe.media' })), true);++ assert.strictEqual(shield.onEdge(req({ host: 'edge.openvibe.media', 'x-forwarded-host': 'edge.openvibe.media' })), false, 'a client-chosen Host or X-Forwarded-Host is not the edge');++ assert.strictEqual(shield.onEdge(req({ 'x-media-shield-host': 'openvibe.media' })), false);++++ // The nginx side of the same rules (the config is what enforces them on the host).++ const fs = require('fs'), path = require('path');++ const conf = (f) => fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', f), 'utf8');++ const upstream = conf('media-shield-b2-upstream.conf');++ assert.ok(upstream.includes(`proxy_set_header Host ${shield.B2_UPSTREAM_HOST};`) && upstream.includes(`proxy_pass https://${shield.B2_UPSTREAM_HOST};`), 'nginx proxies to the host shield.js signs for');++ const edgeConf = conf('edge.openvibe.media.conf'), mainConf = conf('openvibe.media.conf');++ assert.match(edgeConf, /proxy_set_header X-Media-Shield-Host \$server_name;/, 'the edge block vouches for itself');++ assert.match(mainConf, /proxy_set_header X-Media-Shield-Host "";/, 'openvibe.media clears the header a client could send through Cloudflare');++ for (const c of [edgeConf, mainConf]) assert.match(c, /proxy_set_header X-Forwarded-Host "";/);++ for (const d of ['client_max_body_size 4096m;', 'proxy_read_timeout 3600s;', 'location = /metrics { return 404; }']) assert.ok(edgeConf.includes(d), `the edge block has ${d}`);++ assert.match(conf('media-shield.http.conf'), /allow 127\.0\.0\.1;\s*deny all;/, 'only loopback may refresh the cache');// send(): the route's headers plus the internal redirect, an empty 200.const res = { h: {}, set(k, v) { this.h[k] = v; }, status(c) { this.code = c; return this; }, end() { this.ended = true; return this; } };shield.send(res, '/_media_shield/b2/x', { 'Content-Type': 'video/mp4', 'Cache-Control': 'public, max-age=300', Skip: null });assert.deepStrictEqual([res.code, res.ended, res.h['X-Accel-Redirect'], res.h['Content-Type'], 'Skip' in res.h], [200, true, '/_media_shield/b2/x', 'video/mp4', false]);-- // purge(): one refresh request per 10 MB slice, loopback, with the public Host and each slice's exact range.++ // purge(): one refresh request per 10 MB slice to the loopback listener, each slice's exact range, a few at a time.const calls = [];-- const fetchImpl = async (u, opts) => { calls.push([u, opts.headers.Host, opts.headers.Range]); return { arrayBuffer: async () => new ArrayBuffer(0) }; };++ let inFlight = 0, peak = 0;++ const fetchImpl = async (u, opts) => {++ inFlight++; peak = Math.max(peak, inFlight); calls.push([u, opts.headers.Range]);++ await new Promise((r) => setTimeout(r, 5)); inFlight--;++ return { arrayBuffer: async () => new ArrayBuffer(0) };++ };const r = await shield.purge({ provider: 'b2', key: 'vods/a b.mp4', size: 25 * 1024 * 1024, presign: async () => url, fetchImpl });assert.deepStrictEqual([r.purged, r.slices, r.failed], [3, 3, 0]);-- assert.ok(calls.every(([u, h]) => u.startsWith('http://127.0.0.1:8479/_media_shield_refresh/b2/ov-vods/vods/a%20b.mp4?') && h === 'edge.openvibe.media'), JSON.stringify(calls[0]));-- assert.deepStrictEqual(calls.map((c) => c[2]), ['bytes=0-10485759', 'bytes=10485760-20971519', 'bytes=20971520-31457279']);++ assert.ok(calls.every(([u]) => u.startsWith('http://127.0.0.1:8479/_media_shield_refresh/b2/ov-vods/vods/a%20b.mp4?')), JSON.stringify(calls[0]));++ assert.deepStrictEqual(calls.map((c) => c[1]).sort(), ['bytes=0-10485759', 'bytes=10485760-20971519', 'bytes=20971520-31457279']);++ calls.length = 0; peak = 0;++ const big = await shield.purge({ provider: 'b2', key: 'vods/a b.mp4', size: 40 * 10 * 1024 * 1024, presign: async () => url, fetchImpl });++ assert.deepStrictEqual([big.purged, big.slices], [40, 40]);++ assert.ok(peak <= 4 && peak > 1, `a large object refreshes a few slices at a time (peak ${peak})`);assert.deepStrictEqual(await shield.purge({ provider: 'r2', key: 'k', size: 1, presign: async () => url, fetchImpl }), { purged: 0, skipped: true }, 'nothing to purge off the shield');delete process.env.MEDIA_SHIELD; delete process.env.MEDIA_SHIELD_HOST;
@@ -16,6 +16,14 @@ server {
@@ -16,6 +16,14 @@ server {
ssl_certificate /etc/letsencrypt/live/edge.openvibe.media/fullchain.pem;
ssl_certificate /etc/letsencrypt/live/edge.openvibe.media/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/edge.openvibe.media/privkey.pem;
ssl_certificate_key /etc/letsencrypt/live/edge.openvibe.media/privkey.pem;
+ # As on openvibe.media: large uploads, long Node responses (DVR, recordings, SSE).
+ client_max_body_size 4096m;
+ proxy_read_timeout 3600s;
+ proxy_send_timeout 3600s;
+
+ # Prometheus metrics are for loopback scrapers only (the app 404s proxied callers too).
+ location = /metrics { return 404; }
+
location /_media_shield/b2/ {
location /_media_shield/b2/ {
internal;
internal;
access_log /var/log/nginx/media-shield.log media_shield;
access_log /var/log/nginx/media-shield.log media_shield;
@@ -31,15 +39,29 @@ server {
@@ -31,15 +39,29 @@ server {
proxy_cache_lock_age 10s;
proxy_cache_lock_age 10s;
proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
proxy_cache_background_update on;
+ # Node's answer owns the viewer-facing headers (Content-Type, Content-Disposition, Cache-Control survive the
+ # X-Accel redirect): B2's copies, and its request ids, are dropped. The security headers Node sets do not survive
+ # the redirect, so they are set here.
+ proxy_hide_header Content-Disposition;
+ proxy_hide_header Cache-Control;
+ proxy_hide_header Expires;
+ proxy_hide_header Set-Cookie;
+ proxy_hide_header x-amz-request-id;
+ proxy_hide_header x-amz-id-2;
+ add_header X-Content-Type-Options nosniff always;
+ add_header X-Robots-Tag noindex always;
add_header X-Cache-Status $upstream_cache_status always;
add_header X-Cache-Status $upstream_cache_status always;
include snippets/media-shield-b2-upstream.conf;
include snippets/media-shield-b2-upstream.conf;
}
}
- # Everything else is Node, exactly as on openvibe.media (client-IP headers from $remote_addr only).
+ # Everything else is Node, exactly as on openvibe.media (client-IP headers from $remote_addr only), plus the one
+ # header that tells Node this is the edge: nginx sets it from its own server name, so a client cannot forge it.
location / {
location / {
proxy_pass http://127.0.0.1:4100;
proxy_pass http://127.0.0.1:4100;
proxy_http_version 1.1;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Host $host;
+ proxy_set_header X-Media-Shield-Host $server_name;
+ proxy_set_header X-Forwarded-Host "";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header CF-Connecting-IP $remote_addr;
proxy_set_header CF-Connecting-IP $remote_addr;
@@ -1,5 +1,7 @@
@@ -1,5 +1,7 @@
# Media Fabric F1b — how the shield reaches B2 (install as /etc/nginx/snippets/media-shield-b2-upstream.conf).
# Media Fabric F1b — how the shield reaches B2 (install as /etc/nginx/snippets/media-shield-b2-upstream.conf).
-# B2 is addressed path-style, so every presigned URL is signed for this one host (SigV4 signs the Host header).
+# B2 is addressed path-style, so every presigned URL is signed for this one host (SigV4 signs the Host header). It must
+# equal server/placement/shield.js B2_UPSTREAM_HOST (test/placement-shield.test.js pins both) and the host of
+# MEDIA_B2_ENDPOINT: a URL signed for any other host is never sent here (Node keeps its 302).
proxy_set_header Range $slice_range;
proxy_set_header Range $slice_range;
proxy_set_header If-Range "";
proxy_set_header If-Range "";
proxy_set_header Host s3.us-west-004.backblazeb2.com;
proxy_set_header Host s3.us-west-004.backblazeb2.com;
@@ -14,6 +14,9 @@ server {
@@ -14,6 +14,9 @@ server {
server_name _;
server_name _;
access_log /var/log/nginx/media-shield.log media_shield;
access_log /var/log/nginx/media-shield.log media_shield;
location /_media_shield_refresh/b2/ {
location /_media_shield_refresh/b2/ {
+ # Node calls this over real HTTP, so it cannot be `internal`: only loopback may force cache writes.
+ allow 127.0.0.1;
+ deny all;
rewrite ^/_media_shield_refresh/b2/(.*)$ /$1 break;
rewrite ^/_media_shield_refresh/b2/(.*)$ /$1 break;
slice 10m;
slice 10m;
proxy_cache media_shield;
proxy_cache media_shield;
@@ -24,6 +24,10 @@ server {
@@ -24,6 +24,10 @@ server {
proxy_pass http://127.0.0.1:4100;
proxy_pass http://127.0.0.1:4100;
proxy_http_version 1.1;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Host $host;
+ # Only the edge host's block may say "edge" (placement/shield.js): cleared here so a client cannot send it
+ # through Cloudflare; X-Forwarded-Host is cleared too (Express's req.hostname would honour it).
+ proxy_set_header X-Media-Shield-Host "";
+ proxy_set_header X-Forwarded-Host "";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header CF-Connecting-IP $remote_addr;
proxy_set_header CF-Connecting-IP $remote_addr;
@@ -194,14 +194,26 @@ metric. Budgets per class and provider with a forecast; `provider.cost.threshold
@@ -194,14 +194,26 @@ metric. Budgets per class and provider with a forecast; `provider.cost.threshold
until F2's engine can enforce the 70 %-of-port rule (an R2 read through the shield spends the same host bandwidth).
until F2's engine can enforce the 70 %-of-port rule (an R2 read through the shield spends the same host bandwidth).
- **A DNS-only edge host.** openvibe.media is behind Cloudflare's proxy and its terms for video are not verified, so
- **A DNS-only edge host.** openvibe.media is behind Cloudflare's proxy and its terms for video are not verified, so
the shield answers only on `edge.openvibe.media` (grey cloud; `MEDIA_SHIELD_HOST`). Bytes that went viewer → B2 never
the shield answers only on `edge.openvibe.media` (grey cloud; `MEDIA_SHIELD_HOST`). Bytes that went viewer → B2 never
- start flowing through Cloudflare.
+ start flowing through Cloudflare. Node knows the edge from `X-Media-Shield-Host`, which only the edge server block sets
+ (from its own server name) and `openvibe.media` clears; never from Host or `X-Forwarded-Host`, which a client chooses.
+- **One B2 host.** nginx proxies to one B2 endpoint and SigV4 signs the Host, so a URL signed for any other host (a
+ changed `MEDIA_B2_ENDPOINT`) is never shielded and keeps its 302 (`shield.js` `B2_UPSTREAM_HOST` = the nginx snippet;
+ a test pins both).
- **Cache key:** `b2|<bucket>/<key>|<slice range>`: the object path and the 10 MB slice, never the signature. The
- **Cache key:** `b2|<bucket>/<key>|<slice range>`: the object path and the 10 MB slice, never the signature. The
- shield's presign lives an hour (it stays inside nginx and a long response fetches later slices with it); the
- viewer-facing 302 keeps its short TTL.
+ shield's presign lives six hours (it stays inside nginx and a long response fetches later slices with it; a single
+ response longer than that gets an uncached 403 on its next slice); the viewer-facing 302 keeps its short TTL.
+- **Headers:** Node's answer owns Content-Type, Content-Disposition and Cache-Control (they survive the X-Accel
+ redirect; B2's copies are hidden); the shield location adds `nosniff` and `noindex` itself. Unlisted objects are served
+ through the shared cache (anyone with the link may read them; `/o` answers `public, max-age=3600` for them), unlisted
+ VODs answer `private, max-age=0`.
- **Private and sandbox bytes never enter the shield.**
- **Private and sandbox bytes never enter the shield.**
- **Sizing:** one disk with recordings: `max_size=5g`, `min_free=20g` (the recording guardian warns at 15 GB), `inactive=24h`.
- **Sizing:** one disk with recordings: `max_size=5g`, `min_free=20g` (the recording guardian warns at 15 GB), `inactive=24h`.
-- **Purge:** deleting a shielded object refreshes each of its slices through a loopback-only listener
- (`127.0.0.1:8479`, `proxy_cache_bypass`), so the provider's 404 replaces the cached bytes at once.
+- **Purge:** deleting a shielded object refreshes each of its slices (four at a time) through a loopback-only listener
+ (`127.0.0.1:8479`, `allow 127.0.0.1; deny all`, `proxy_cache_bypass`), so the provider's 404 replaces the cached bytes
+ at once. Overwriting a shielded key (a repair re-upload, or a re-upload within ten minutes of a delete) refreshes it
+ after the upload, so the shield never serves the old bytes.
- **Logs:** shield logs carry the path only: the signed query is a bearer token.
- **Logs:** shield logs carry the path only: the signed query is a bearer token.
- Config: `deploy/nginx/media-shield.http.conf` (conf.d), `media-shield-b2-upstream.conf` (snippets),
- Config: `deploy/nginx/media-shield.http.conf` (conf.d), `media-shield-b2-upstream.conf` (snippets),
- `edge.openvibe.media.conf` (sites-enabled). Switch on with `MEDIA_SHIELD=b2` and `MEDIA_SHIELD_HOST` after nginx has them.
+ `edge.openvibe.media.conf` (sites-enabled), and `openvibe.media.conf` clearing `X-Media-Shield-Host`. Switch on with
+ `MEDIA_SHIELD=b2` and `MEDIA_SHIELD_HOST` after nginx has them, and check a real slice with `curl -I` on the edge
+ (one Content-Type, Node's Content-Disposition, `X-Cache-Status`, no `x-amz-*`).
@@ -774,11 +774,11 @@ publicRouter.get('/:id', async (req, res) => {
@@ -774,11 +774,11 @@ publicRouter.get('/:id', async (req, res) => {
continue;
continue;
}
}
// The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through its
// The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through its
- // slice cache, with an hour-long internal presign that never reaches the viewer.
+ // slice cache, with a long internal presign that never reaches the viewer (only on the DNS-only edge host).
const shield = require('../placement/shield');
const shield = require('../placement/shield');
- if (shield.enabled(provider) && !sandbox && obj.visibility !== 'private') {
+ if (shield.enabled(provider) && shield.onEdge(req) && !sandbox && obj.visibility !== 'private') {
const inner = await require('../vod/vod-storage').presignGet(provider, loc.key, shield.SHIELD_TTL_SECONDS, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);
const inner = await require('../vod/vod-storage').presignGet(provider, loc.key, shield.SHIELD_TTL_SECONDS, { contentType: mime, contentDisposition: headers['Content-Disposition'] }).catch(() => null);
- const t = inner && shield.target({ provider, url: inner, visibility: obj.visibility, sandbox, host: req.hostname });
+ const t = inner && shield.target({ provider, url: inner, visibility: obj.visibility, sandbox, edge: shield.edgeOf(req) });
if (t) return shield.send(res, t, headers);
if (t) return shield.send(res, t, headers);
}
}
// The router already signed its first choice; a fallback copy is signed here.
// The router already signed its first choice; a fallback copy is signed here.
@@ -6,7 +6,7 @@
@@ -6,7 +6,7 @@
* `/_media_shield/<provider>/…` location, which proxies the presigned request through a slice cache (10 MB slices,
* `/_media_shield/<provider>/…` location, which proxies the presigned request through a slice cache (10 MB slices,
* `proxy_cache_lock`): N viewers of the same object range cost one provider GET, the bytes come from the Media host's
* `proxy_cache_lock`): N viewers of the same object range cost one provider GET, the bytes come from the Media host's
* prepaid bandwidth, and the provider URL never reaches the viewer. The cache key is the object path and the slice,
* prepaid bandwidth, and the provider URL never reaches the viewer. The cache key is the object path and the slice,
- * never the signature (deploy/nginx/openvibe.media.conf).
+ * never the signature (deploy/nginx/media-shield.http.conf, edge.openvibe.media.conf, media-shield-b2-upstream.conf).
*
*
* MEDIA_SHIELD=b2 providers whose reads go through the shield (comma-separated; empty = off). Set it only
* MEDIA_SHIELD=b2 providers whose reads go through the shield (comma-separated; empty = off). Set it only
* once nginx has the locations: a Node that emits X-Accel to an nginx without them 404s.
* once nginx has the locations: a Node that emits X-Accel to an nginx without them 404s.
@@ -15,33 +15,51 @@
@@ -15,33 +15,51 @@
* T4): bytes that went viewer → B2 must not start flowing through Cloudflare. Unset = off.
* T4): bytes that went viewer → B2 must not start flowing through Cloudflare. Unset = off.
* MEDIA_SHIELD_REFRESH_URL the loopback-only purge listener (default http://127.0.0.1:8479).
* MEDIA_SHIELD_REFRESH_URL the loopback-only purge listener (default http://127.0.0.1:8479).
*
*
+ * The edge is known from `X-Media-Shield-Host`, which only the edge server block sets (from its own $host) and every
+ * other Media server block clears: never from Host or X-Forwarded-Host, which a client can choose (Express's
+ * req.hostname honours X-Forwarded-Host under `trust proxy`). nginx proxies to ONE B2 host (B2_UPSTREAM_HOST, equal to
+ * deploy/nginx/media-shield-b2-upstream.conf, a test pins it) and SigV4 signs the Host header, so a URL signed for any
+ * other host (MEDIA_B2_ENDPOINT changed) is never shielded: it keeps its 302 instead of failing every slice with a 403.
+ *
* Rules: private and sandbox bytes never enter the shared cache; B2 only for now (path-style URLs, one signed host; R2
* Rules: private and sandbox bytes never enter the shared cache; B2 only for now (path-style URLs, one signed host; R2
- * waits for F2's port-utilization guard); the shield's presign lives an hour (it stays inside nginx, and a long response
- * fetches later slices with it), the viewer-facing 302 keeps its short TTL.
+ * waits for F2's port-utilization guard); the shield's presign lives six hours (it stays inside nginx, and a long
+ * response fetches later slices with it), the viewer-facing 302 keeps its short TTL.
*
*
- * target({ provider, url, visibility, sandbox }) → '/_media_shield/b2/<bucket>/<key>?<signed query>' | null
+ * onEdge(req) → true on the edge host (check it before presigning anything)
+ * target({ provider, url, visibility, sandbox, edge }) → '/_media_shield/b2/<bucket>/<key>?<signed query>' | null
* send(res, target, headers) → the 200 + X-Accel-Redirect answer
* send(res, target, headers) → the 200 + X-Accel-Redirect answer
* await purge({ provider, key, size, presign }) → refreshes every cached slice of a deleted or replaced object
* await purge({ provider, key, size, presign }) → refreshes every cached slice of a deleted or replaced object
*/
*/
'use strict';
'use strict';
const SLICE_BYTES = 10 * 1024 * 1024; // nginx `slice 10m`
const SLICE_BYTES = 10 * 1024 * 1024; // nginx `slice 10m`
-const SHIELD_TTL_SECONDS = 3600;
+const SHIELD_TTL_SECONDS = 6 * 3600; // inside nginx only; covers a long response's later slices
+const B2_UPSTREAM_HOST = 's3.us-west-004.backblazeb2.com'; // = deploy/nginx/media-shield-b2-upstream.conf
+const UPSTREAM_HOSTS = { b2: B2_UPSTREAM_HOST };
+const EDGE_HEADER = 'x-media-shield-host'; // set by the edge server block only (nginx)
+const PURGE_CONCURRENCY = 4;
function providers() {
function providers() {
return String(process.env.MEDIA_SHIELD || '').split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
return String(process.env.MEDIA_SHIELD || '').split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
}
}
function enabled(provider) { return providers().includes(String(provider || '').toLowerCase()); }
function enabled(provider) { return providers().includes(String(provider || '').toLowerCase()); }
-/** The internal location for one presigned URL, or null when this read must not go through the shield. `host` is the
- * request's host: the shield answers only on MEDIA_SHIELD_HOST (a DNS-only edge name), never through Cloudflare. */
-function target({ provider, url, visibility = 'public', sandbox = false, host = null }) {
+function edgeHost() { return String(process.env.MEDIA_SHIELD_HOST || '').trim().toLowerCase(); }
+/** The edge name nginx vouched for on this request (X-Media-Shield-Host), or null. */
+function edgeOf(req) { const v = req && typeof req.get === 'function' ? req.get(EDGE_HEADER) : null; return v ? String(v).trim().toLowerCase() : null; }
+/** True when this request came in on the edge host: routes check it before presigning anything for the shield. */
+function onEdge(req) { const want = edgeHost(); return !!want && edgeOf(req) === want; }
+
+/** The internal location for one presigned URL, or null when this read must not go through the shield. `edge` is
+ * edgeOf(req): the shield answers only on MEDIA_SHIELD_HOST (a DNS-only edge name), never through Cloudflare. */
+function target({ provider, url, visibility = 'public', sandbox = false, edge = null }) {
if (!url || !enabled(provider)) return null;
if (!url || !enabled(provider)) return null;
- const edge = String(process.env.MEDIA_SHIELD_HOST || '').toLowerCase();
- if (!edge || String(host || '').toLowerCase() !== edge) return null;
+ const want = edgeHost();
+ if (!want || String(edge || '').toLowerCase() !== want) return null;
if (visibility === 'private' || sandbox) return null; // never into a shared cache
if (visibility === 'private' || sandbox) return null; // never into a shared cache
let u; try { u = new URL(url); } catch { return null; }
let u; try { u = new URL(url); } catch { return null; }
- // Path-style URLs only: the pathname is /<bucket>/<key>, the host is the provider's one signed endpoint.
+ // Path-style URLs for the one host nginx proxies to (the signature covers Host): /<bucket>/<key>.
+ if (u.host.toLowerCase() !== UPSTREAM_HOSTS[String(provider).toLowerCase()]) return null;
if (!u.pathname || u.pathname === '/') return null;
if (!u.pathname || u.pathname === '/') return null;
return `/_media_shield/${String(provider).toLowerCase()}${u.pathname}${u.search}`;
return `/_media_shield/${String(provider).toLowerCase()}${u.pathname}${u.search}`;
}
}
@@ -56,26 +74,31 @@ function send(res, targetPath, headers = {}) {
@@ -56,26 +74,31 @@ function send(res, targetPath, headers = {}) {
/**
/**
* Refresh every cached slice of an object after it was deleted or replaced on a shielded provider: one request per
* Refresh every cached slice of an object after it was deleted or replaced on a shielded provider: one request per
- * 10 MB slice to nginx's loopback-only refresh location, which bypasses the cache and stores the provider's answer
- * (a 404 for a deleted object: cached for a minute, then gone). Never throws; returns what it did.
+ * 10 MB slice to nginx's loopback-only refresh listener (a few at a time), which bypasses the cache and stores the
+ * provider's answer (new bytes for a replaced object; a 404 for a deleted one, cached for a minute). Never throws;
+ * returns what it did. The listener is the only server on its port, so no Host header is needed (fetch drops it).
*/
*/
-async function purge({ provider, key, size, presign, fetchImpl = globalThis.fetch, base = process.env.MEDIA_SHIELD_REFRESH_URL || 'http://127.0.0.1:8479', host = process.env.MEDIA_SHIELD_HOST || 'localhost' }) {
- if (!enabled(provider) || !key) return { purged: 0, skipped: true };
+async function purge({ provider, key, size, presign, fetchImpl = globalThis.fetch, base = process.env.MEDIA_SHIELD_REFRESH_URL || 'http://127.0.0.1:8479', concurrency = PURGE_CONCURRENCY }) {
+ if (!enabled(provider) || !key || !edgeHost()) return { purged: 0, skipped: true };
let url; try { url = await presign(provider, key, 300); } catch (err) { return { purged: 0, error: err.message }; }
let url; try { url = await presign(provider, key, 300); } catch (err) { return { purged: 0, error: err.message }; }
- const t = target({ provider, url, host: process.env.MEDIA_SHIELD_HOST });
+ const t = target({ provider, url, edge: edgeHost() });
if (!t) return { purged: 0, skipped: true };
if (!t) return { purged: 0, skipped: true };
const refresh = t.replace('/_media_shield/', '/_media_shield_refresh/');
const refresh = t.replace('/_media_shield/', '/_media_shield_refresh/');
const slices = Math.max(1, Math.ceil((Number(size) || SLICE_BYTES) / SLICE_BYTES));
const slices = Math.max(1, Math.ceil((Number(size) || SLICE_BYTES) / SLICE_BYTES));
- let purged = 0, failed = 0;
- for (let i = 0; i < slices; i++) {
- const range = `bytes=${i * SLICE_BYTES}-${(i + 1) * SLICE_BYTES - 1}`;
- try {
- const r = await fetchImpl(base + refresh, { headers: { Host: host, Range: range }, signal: AbortSignal.timeout(15000) });
- try { await r.arrayBuffer(); } catch { /* drained */ }
- purged++;
- } catch { failed++; }
+ let purged = 0, failed = 0, next = 0;
+ async function worker() {
+ while (next < slices) {
+ const i = next++;
+ const range = `bytes=${i * SLICE_BYTES}-${(i + 1) * SLICE_BYTES - 1}`;
+ try {
+ const r = await fetchImpl(base + refresh, { headers: { Range: range }, signal: AbortSignal.timeout(15000) });
+ try { await r.arrayBuffer(); } catch { /* drained */ }
+ purged++;
+ } catch { failed++; }
+ }
}
}
+ await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, slices)) }, worker));
return { purged, failed, slices };
return { purged, failed, slices };
}
}
-module.exports = { target, send, purge, enabled, providers, SLICE_BYTES, SHIELD_TTL_SECONDS };
+module.exports = { target, send, purge, enabled, providers, onEdge, edgeOf, SLICE_BYTES, SHIELD_TTL_SECONDS, B2_UPSTREAM_HOST, EDGE_HEADER };
@@ -175,11 +175,12 @@ async function serveMediaRecord(kind, record, req, res) {
@@ -175,11 +175,12 @@ async function serveMediaRecord(kind, record, req, res) {
const plan = await vodStorage.resolvePlayback(record, { session: router.sessionFor(req, `${kind}:${record.id}`) });
const plan = await vodStorage.resolvePlayback(record, { session: router.sessionFor(req, `${kind}:${record.id}`) });
if (plan?.kind === 'redirect') {
if (plan?.kind === 'redirect') {
// The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through
// The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through
- // its slice cache with an hour-long internal presign; everything else keeps the short-lived 302.
+ // its slice cache with a long internal presign (only on the DNS-only edge host); everything else keeps the
+ // short-lived 302.
const shield = require('../placement/shield');
const shield = require('../placement/shield');
- if (shield.enabled(plan.provider) && visibility !== 'private' && !record.is_recording) {
+ if (shield.enabled(plan.provider) && shield.onEdge(req) && visibility !== 'private' && !record.is_recording) {
const url = await vodStorage.presignGet(plan.provider, vodStorage.keyForVod(record), shield.SHIELD_TTL_SECONDS).catch(() => null);
const url = await vodStorage.presignGet(plan.provider, vodStorage.keyForVod(record), shield.SHIELD_TTL_SECONDS).catch(() => null);
- const t = url && shield.target({ provider: plan.provider, url, visibility, host: req.hostname });
+ const t = url && shield.target({ provider: plan.provider, url, visibility, edge: shield.edgeOf(req) });
if (t) return shield.send(res, t, { 'Cache-Control': visibility === 'public' ? 'public, max-age=300' : 'private, max-age=0', 'X-Robots-Tag': 'noindex' });
if (t) return shield.send(res, t, { 'Cache-Control': visibility === 'public' ? 'public, max-age=300' : 'private, max-age=0', 'X-Robots-Tag': 'noindex' });
}
}
res.set('Cache-Control', 'private, max-age=0');
res.set('Cache-Control', 'private, max-age=0');
@@ -310,6 +310,10 @@ async function uploadFile(provider, key, filePath, contentType = 'video/webm') {
@@ -310,6 +310,10 @@ async function uploadFile(provider, key, filePath, contentType = 'video/webm') {
if (!client) throw new Error(`Provider ${provider} not configured`);
if (!client) throw new Error(`Provider ${provider} not configured`);
loadSdk();
loadSdk();
const size = fs.statSync(filePath).size;
const size = fs.statSync(filePath).size;
+ // A shielded key that already exists (a repair re-upload) or was deleted moments ago may have slices in the shield:
+ // they are refreshed after the upload, or the shield would serve the old bytes for up to a day.
+ const shield = require('../placement/shield');
+ const before = shield.enabled(provider) ? await headObject(provider, key).catch(() => null) : null;
// A hung multipart upload used to hold the sweep lock indefinitely (nothing else
// A hung multipart upload used to hold the sweep lock indefinitely (nothing else
// could drain). Abort past the deadline so the sweep moves on to the next VOD;
// could drain). Abort past the deadline so the sweep moves on to the next VOD;
// leavePartsOnError cleans the partial multipart up on the bucket.
// leavePartsOnError cleans the partial multipart up on the bucket.
@@ -336,6 +340,12 @@ async function uploadFile(provider, key, filePath, contentType = 'video/webm') {
@@ -336,6 +340,12 @@ async function uploadFile(provider, key, filePath, contentType = 'video/webm') {
} finally {
} finally {
clearTimeout(deadline);
clearTimeout(deadline);
}
}
+ const deletedSize = recentlyDeleted.get(`${provider}:${key}`);
+ if (before || deletedSize != null) {
+ recentlyDeleted.delete(`${provider}:${key}`);
+ const r = await shield.purge({ provider, key, size: Math.max(size, (before && before.size) || 0, deletedSize || 0), presign: presignGet });
+ if (r.failed) console.warn(`[VodStorage] shield refresh ${provider}:${key}: ${r.failed} of ${r.slices} slices not refreshed`);
+ }
// Verify size before anything destructive happens
// Verify size before anything destructive happens
const localSize = fs.statSync(filePath).size;
const localSize = fs.statSync(filePath).size;
const head = await headObject(provider, key);
const head = await headObject(provider, key);
@@ -393,6 +403,14 @@ async function sha256Object(provider, key) {
@@ -393,6 +403,14 @@ async function sha256Object(provider, key) {
return { sha256: hash.digest('hex'), size };
return { sha256: hash.digest('hex'), size };
}
}
+// Shielded keys deleted in the last ten minutes, with their size: a re-upload of the same key refreshes the shield
+// even though its HEAD finds nothing (the delete's own refresh may still be running and cache a 404 for a minute).
+const recentlyDeleted = new Map();
+function rememberDeleted(id, size) {
+ recentlyDeleted.set(id, size);
+ setTimeout(() => { if (recentlyDeleted.get(id) === size) recentlyDeleted.delete(id); }, 10 * 60e3).unref?.();
+}
+
async function deleteObject(provider, key) {
async function deleteObject(provider, key) {
const client = clientFor(provider);
const client = clientFor(provider);
if (!client) return;
if (!client) return;
@@ -406,6 +424,7 @@ async function deleteObject(provider, key) {
@@ -406,6 +424,7 @@ async function deleteObject(provider, key) {
console.warn(`[VodStorage] Delete ${provider}:${key} failed:`, err.message);
console.warn(`[VodStorage] Delete ${provider}:${key} failed:`, err.message);
return;
return;
}
}
+ if (head) rememberDeleted(`${provider}:${key}`, head.size);
if (head) shield.purge({ provider, key, size: head.size, presign: presignGet })
if (head) shield.purge({ provider, key, size: head.size, presign: presignGet })
.then((r) => { if (r.failed) console.warn(`[VodStorage] shield purge ${provider}:${key}: ${r.failed} of ${r.slices} slices not refreshed`); })
.then((r) => { if (r.failed) console.warn(`[VodStorage] shield purge ${provider}:${key}: ${r.failed} of ${r.slices} slices not refreshed`); })
.catch(() => { /* purge never throws */ });
.catch(() => { /* purge never throws */ });
@@ -10,30 +10,59 @@ const assert = require('assert');
@@ -10,30 +10,59 @@ const assert = require('assert');
assert.strictEqual(shield.target({ provider: 'b2', url }), null, 'off unless MEDIA_SHIELD names the provider');
assert.strictEqual(shield.target({ provider: 'b2', url }), null, 'off unless MEDIA_SHIELD names the provider');
process.env.MEDIA_SHIELD = 'b2';
process.env.MEDIA_SHIELD = 'b2';
- assert.strictEqual(shield.target({ provider: 'b2', url, host: 'edge.openvibe.media' }), null, 'no edge host configured: off');
+ assert.strictEqual(shield.target({ provider: 'b2', url, edge: 'edge.openvibe.media' }), null, 'no edge host configured: off');
process.env.MEDIA_SHIELD_HOST = 'edge.openvibe.media';
process.env.MEDIA_SHIELD_HOST = 'edge.openvibe.media';
- assert.strictEqual(shield.target({ provider: 'b2', url, host: 'openvibe.media' }), null, 'never on the Cloudflare-proxied host');
- const host = 'edge.openvibe.media';
- assert.strictEqual(shield.target({ provider: 'b2', url, host }), '/_media_shield/b2/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc',
+ assert.strictEqual(shield.target({ provider: 'b2', url, edge: null }), null, 'never on the Cloudflare-proxied host (nginx sends no edge header there)');
+ const edge = 'edge.openvibe.media';
+ assert.strictEqual(shield.target({ provider: 'b2', url, edge }), '/_media_shield/b2/ov-vods/vods/a%20b.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc',
'the path-style path and the signed query ride into the internal location unchanged');
'the path-style path and the signed query ride into the internal location unchanged');
- assert.strictEqual(shield.target({ provider: 'r2', url, host }), null, 'R2 is not shielded in F1b');
- assert.strictEqual(shield.target({ provider: 'b2', url, host, visibility: 'private' }), null, 'private bytes never enter the shared cache');
- assert.strictEqual(shield.target({ provider: 'b2', url, host, sandbox: true }), null, 'sandbox bytes never enter it either');
- assert.strictEqual(shield.target({ provider: 'b2', url: 'not a url', host }), null);
- assert.strictEqual(shield.target({ provider: 'b2', url: 'https://s3.example.com/', host }), null, 'a URL without an object path is not shielded');
+ assert.strictEqual(shield.target({ provider: 'r2', url, edge }), null, 'R2 is not shielded in F1b');
+ assert.strictEqual(shield.target({ provider: 'b2', url, edge, visibility: 'private' }), null, 'private bytes never enter the shared cache');
+ assert.strictEqual(shield.target({ provider: 'b2', url, edge, sandbox: true }), null, 'sandbox bytes never enter it either');
+ assert.strictEqual(shield.target({ provider: 'b2', url: 'not a url', edge }), null);
+ assert.strictEqual(shield.target({ provider: 'b2', url: `https://${shield.B2_UPSTREAM_HOST}/`, edge }), null, 'a URL without an object path is not shielded');
+ // SigV4 signs the Host: a URL signed for another B2 endpoint would 403 on every slice through nginx, so it keeps its 302.
+ assert.strictEqual(shield.target({ provider: 'b2', url: url.replace('s3.us-west-004', 's3.eu-central-003'), edge }), null, 'another B2 host is never shielded');
+
+ // The edge comes from the header only the edge server block sets: never from Host or X-Forwarded-Host.
+ const req = (h) => ({ get: (k) => h[k.toLowerCase()] });
+ assert.strictEqual(shield.onEdge(req({ 'x-media-shield-host': 'edge.openvibe.media' })), true);
+ assert.strictEqual(shield.onEdge(req({ host: 'edge.openvibe.media', 'x-forwarded-host': 'edge.openvibe.media' })), false, 'a client-chosen Host or X-Forwarded-Host is not the edge');
+ assert.strictEqual(shield.onEdge(req({ 'x-media-shield-host': 'openvibe.media' })), false);
+
+ // The nginx side of the same rules (the config is what enforces them on the host).
+ const fs = require('fs'), path = require('path');
+ const conf = (f) => fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', f), 'utf8');
+ const upstream = conf('media-shield-b2-upstream.conf');
+ assert.ok(upstream.includes(`proxy_set_header Host ${shield.B2_UPSTREAM_HOST};`) && upstream.includes(`proxy_pass https://${shield.B2_UPSTREAM_HOST};`), 'nginx proxies to the host shield.js signs for');
+ const edgeConf = conf('edge.openvibe.media.conf'), mainConf = conf('openvibe.media.conf');
+ assert.match(edgeConf, /proxy_set_header X-Media-Shield-Host \$server_name;/, 'the edge block vouches for itself');
+ assert.match(mainConf, /proxy_set_header X-Media-Shield-Host "";/, 'openvibe.media clears the header a client could send through Cloudflare');
+ for (const c of [edgeConf, mainConf]) assert.match(c, /proxy_set_header X-Forwarded-Host "";/);
+ for (const d of ['client_max_body_size 4096m;', 'proxy_read_timeout 3600s;', 'location = /metrics { return 404; }']) assert.ok(edgeConf.includes(d), `the edge block has ${d}`);
+ assert.match(conf('media-shield.http.conf'), /allow 127\.0\.0\.1;\s*deny all;/, 'only loopback may refresh the cache');
// send(): the route's headers plus the internal redirect, an empty 200.
// send(): the route's headers plus the internal redirect, an empty 200.
const res = { h: {}, set(k, v) { this.h[k] = v; }, status(c) { this.code = c; return this; }, end() { this.ended = true; return this; } };
const res = { h: {}, set(k, v) { this.h[k] = v; }, status(c) { this.code = c; return this; }, end() { this.ended = true; return this; } };
shield.send(res, '/_media_shield/b2/x', { 'Content-Type': 'video/mp4', 'Cache-Control': 'public, max-age=300', Skip: null });
shield.send(res, '/_media_shield/b2/x', { 'Content-Type': 'video/mp4', 'Cache-Control': 'public, max-age=300', Skip: null });
assert.deepStrictEqual([res.code, res.ended, res.h['X-Accel-Redirect'], res.h['Content-Type'], 'Skip' in res.h], [200, true, '/_media_shield/b2/x', 'video/mp4', false]);
assert.deepStrictEqual([res.code, res.ended, res.h['X-Accel-Redirect'], res.h['Content-Type'], 'Skip' in res.h], [200, true, '/_media_shield/b2/x', 'video/mp4', false]);
- // purge(): one refresh request per 10 MB slice, loopback, with the public Host and each slice's exact range.
+ // purge(): one refresh request per 10 MB slice to the loopback listener, each slice's exact range, a few at a time.
const calls = [];
const calls = [];
- const fetchImpl = async (u, opts) => { calls.push([u, opts.headers.Host, opts.headers.Range]); return { arrayBuffer: async () => new ArrayBuffer(0) }; };
+ let inFlight = 0, peak = 0;
+ const fetchImpl = async (u, opts) => {
+ inFlight++; peak = Math.max(peak, inFlight); calls.push([u, opts.headers.Range]);
+ await new Promise((r) => setTimeout(r, 5)); inFlight--;
+ return { arrayBuffer: async () => new ArrayBuffer(0) };
+ };
const r = await shield.purge({ provider: 'b2', key: 'vods/a b.mp4', size: 25 * 1024 * 1024, presign: async () => url, fetchImpl });
const r = await shield.purge({ provider: 'b2', key: 'vods/a b.mp4', size: 25 * 1024 * 1024, presign: async () => url, fetchImpl });
assert.deepStrictEqual([r.purged, r.slices, r.failed], [3, 3, 0]);
assert.deepStrictEqual([r.purged, r.slices, r.failed], [3, 3, 0]);
- assert.ok(calls.every(([u, h]) => u.startsWith('http://127.0.0.1:8479/_media_shield_refresh/b2/ov-vods/vods/a%20b.mp4?') && h === 'edge.openvibe.media'), JSON.stringify(calls[0]));
- assert.deepStrictEqual(calls.map((c) => c[2]), ['bytes=0-10485759', 'bytes=10485760-20971519', 'bytes=20971520-31457279']);
+ assert.ok(calls.every(([u]) => u.startsWith('http://127.0.0.1:8479/_media_shield_refresh/b2/ov-vods/vods/a%20b.mp4?')), JSON.stringify(calls[0]));
+ assert.deepStrictEqual(calls.map((c) => c[1]).sort(), ['bytes=0-10485759', 'bytes=10485760-20971519', 'bytes=20971520-31457279']);
+ calls.length = 0; peak = 0;
+ const big = await shield.purge({ provider: 'b2', key: 'vods/a b.mp4', size: 40 * 10 * 1024 * 1024, presign: async () => url, fetchImpl });
+ assert.deepStrictEqual([big.purged, big.slices], [40, 40]);
+ assert.ok(peak <= 4 && peak > 1, `a large object refreshes a few slices at a time (peak ${peak})`);
assert.deepStrictEqual(await shield.purge({ provider: 'r2', key: 'k', size: 1, presign: async () => url, fetchImpl }), { purged: 0, skipped: true }, 'nothing to purge off the shield');
assert.deepStrictEqual(await shield.purge({ provider: 'r2', key: 'k', size: 1, presign: async () => url, fetchImpl }), { purged: 0, skipped: true }, 'nothing to purge off the shield');
delete process.env.MEDIA_SHIELD; delete process.env.MEDIA_SHIELD_HOST;
delete process.env.MEDIA_SHIELD; delete process.env.MEDIA_SHIELD_HOST;