openvibers/openvibe.media Commits

?download on /v/:id and /c/:id: the bytes as a named attachment (disk, presigned B2/R2, a virtual clip remuxed to MP4 as it is sent) (#183)

OP OpenVibers committed 18 days ago · 1b1f934cac · 3 files, +217 −2

Changes

ddiff --git a/docs/object-model.md b/docs/object-model.md
iindex b117511..a4a7b22 100644
---- a/docs/object-model.md
++++ b/docs/object-model.md
@@@ -454,6 +454,14 @@ recorded, still processing, not available), is `noindex`, and a clip still being
instead of a 404. The sitemap lists only playable watch pages. The byte routes (`?raw=1`, `<video>`
requests, `/o/:id`) are unchanged: Live's DVR player reads a recording while it is being written.
++**Downloads.** `?download` on `/v/:id` and `/c/:id` answers the bytes as an attachment, so a browser saves them
++instead of playing them; `?name` names the file (control, quote and path characters dropped, at most 150 characters,
++the real extension added; `vod-<id>`/`clip-<id>` without it). A file on disk carries the header; a copy in B2/R2 is a
++presigned URL asking the store for it (`ResponseContentDisposition`, never the shield); a virtual clip is its playlist
++remuxed by ffmpeg (stream copy, fragmented MP4) as it is sent, two at once at most (503 with `Retry-After` beyond).
++Private items need the usual owner credentials or a signed URL (Live's `GET /api/vods/:id/download-url` hands one out).
++OpenVibe.Video's Download menu links here.
++
## Retention holds
`media_holds(object_id, kind, reason, created_by, created_at, released_at, released_by, note)`: one
ddiff --git a/server/public/routes.js b/server/public/routes.js
iindex 8f84af8..eb8697b 100644
---- a/server/public/routes.js
++++ b/server/public/routes.js
@@@ -127,6 +127,75 @@ function streamFileWithRange(req, res, filePath, extraHeaders = {}) {
}
}
++// ── Downloads (?download) ────────────────────────────────────
++// `?download` asks for a recording's or clip's bytes as an attachment, so a browser saves them instead of playing them
++// (OpenVibe.Video's Download menu, through Live's /api/vods/:id/download-url). `?name` names the file; the real extension
++// is added. A file on disk is sent with the header, a copy in B2/R2 is a presigned URL the store answers with it, and a
++// virtual clip (no file of its own, F3.5) is its playlist remuxed into one MP4 as it is sent.
++const REMUX_MAX = 2; // virtual clips remuxed at once; more wait (503, Retry-After)
++const REMUX_KILL_MS = 120000; // a clip is at most a minute or two: a remux that runs longer is stuck
++let _remuxing = 0;
++
++/** The file name ?name asks for: no control, quote or path characters, at most 150 characters. '' when none. */
++function downloadBase(name) {
++ return String(name || '').replace(/[\u0000-\u001f\u007f"\\/:*?<>|]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 150);
++}
++
++/** The Content-Disposition of a ?download request (null otherwise): attachment, an ASCII name and the UTF-8 one. */
++function downloadDisposition(req, kind, record, ext) {
++ if (!req.query || req.query.download == null) return null;
++ const file = `${downloadBase(req.query.name) || `${kind}-${record.id}`}${ext || ''}`;
++ const ascii = file.replace(/[^\x20-\x7e]/g, '_');
++ const utf8 = encodeURIComponent(file).replace(/['()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
++ return `attachment; filename="${ascii}"; filename*=UTF-8''${utf8}`;
++}
++
++/**
++ * A virtual clip as one MP4: ffmpeg copies its own playlist (/o/<object>/source/index.m3u8 over loopback, signed when
++ * the clip is closed) into a fragmented MP4 piped to the reader, so it starts at once and no file is kept. The headers
++ * go out with the first bytes: a remux that fails before them is a 502 the caller can show.
++ */
++function sendVirtualClipFile(req, res, record, closed, disposition) {
++ if (_remuxing >= REMUX_MAX) {
++ res.set('Retry-After', '5');
++ return res.status(503).json({ error: 'Other downloads are being prepared: try again in a few seconds' });
++ }
++ let query = '';
++ if (closed) {
++ const signed = new URL(require('../objects/signing').signedPlaylistUrl(record.object_id).url);
++ query = `?exp=${encodeURIComponent(signed.searchParams.get('exp'))}&sig=${encodeURIComponent(signed.searchParams.get('sig'))}`;
++ }
++ const src = `http://127.0.0.1:${config.port}/o/${encodeURIComponent(record.object_id)}/source/index.m3u8${query}`;
++ _remuxing++;
++ let released = false;
++ const release = () => { if (!released) { released = true; _remuxing--; } };
++ const ff = require('child_process').spawn('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-i', src,
++ '-map', '0:v:0?', '-map', '0:a:0?', '-c', 'copy', '-movflags', '+frag_keyframe+empty_moov+default_base_moof', '-f', 'mp4', 'pipe:1'],
++ { stdio: ['ignore', 'pipe', 'pipe'] });
++ let started = false;
++ let lastErr = '';
++ const timer = setTimeout(() => { try { ff.kill('SIGKILL'); } catch { /* */ } }, REMUX_KILL_MS);
++ ff.stderr.on('data', (d) => { lastErr = String(d).trim().split('\n').pop().slice(0, 200) || lastErr; });
++ ff.stdout.on('data', (chunk) => {
++ if (!started) {
++ started = true;
++ res.writeHead(200, { 'Content-Type': 'video/mp4', 'Content-Disposition': disposition, 'Cache-Control': 'private, max-age=0',
++ 'X-Robots-Tag': 'noindex', 'X-Content-Type-Options': 'nosniff' });
++ }
++ if (!res.write(chunk)) { ff.stdout.pause(); res.once('drain', () => ff.stdout.resume()); }
++ });
++ const end = (code) => {
++ clearTimeout(timer);
++ release();
++ if (started) return res.end();
++ if (code !== 0) console.warn(`[Public] clip ${record.id} download remux failed: ${lastErr || `exit ${code}`}`);
++ if (!res.headersSent) res.status(502).json({ error: 'The clip could not be prepared for download' });
++ };
++ ff.on('close', end);
++ ff.on('error', () => end(-1));
++ res.on('close', () => { if (ff.exitCode == null) { try { ff.kill('SIGKILL'); } catch { /* */ } } });
++}
++
/**
* Serve a vods/clips row: local file (Range + live-DVR sidecar) or 302
* presigned B2/R2 — the inherited playback logic.
@@@ -145,11 +214,12 @@ async function serveMediaRecord(kind, record, req, res) {
// to the master playlist, signed for a closed clip with the clip's own 'hls' token, which reaches only the segments
// inside the clip's window. Before the watch page: its player and download link name ?raw=1 bytes it does not have.
if (kind === 'clip' && !record.file_path && record.storage_provider === 'timeline') {
-- const config = require('../config');
if (drill.refuseBytes(res)) return;
if (!config.hls.enabled || !record.object_id || record.status !== 'ready') return res.status(404).json({ error: 'Media file unavailable' });
await trackUniqueView(kind, record.id, req, record.user_id);
const closed = visibility === 'private' || await db.isSandboxTenant(record.app_id);
++ const asFile = downloadDisposition(req, kind, record, '.mp4');
++ if (asFile) return sendVirtualClipFile(req, res, record, closed, asFile);
res.set({ 'Cache-Control': 'private, max-age=0', 'X-Robots-Tag': 'noindex' });
return res.redirect(302, closed ? require('../objects/signing').signedPlaylistUrl(record.object_id).url : `${config.publicUrl}/o/${record.object_id}/master.m3u8`);
}
@@@ -195,6 +265,13 @@ async function serveMediaRecord(kind, record, req, res) {
// The shield could not fetch this read (nginx handed it back): another copy, and no shield for this one.
const shieldFailed = require('../placement/shield').failedOf(req);
const plan = await vodStorage.resolvePlayback(record, { session: router.sessionFor(req, `${kind}:${record.id}`), exclude: shieldFailed ? [shieldFailed] : [] });
++ // A download: the store's own presigned URL, answered as an attachment (the shield's cache keeps no such header).
++ const key = plan?.kind === 'redirect' ? vodStorage.keyForVod(record) : null;
++ const asFile = key && downloadDisposition(req, kind, record, path.extname(key) || path.extname(record.file_path || ''));
++ if (asFile) {
++ const url = await vodStorage.presignGet(plan.provider, key, 900, { contentDisposition: asFile }).catch(() => null);
++ if (url) { res.set({ 'Cache-Control': 'private, max-age=0', 'X-Robots-Tag': 'noindex' }); return res.redirect(302, url); }
++ }
if (plan?.kind === 'redirect') {
// The origin shield (placement/shield.js): a public copy on a shielded provider is served by nginx through
// its slice cache with a long internal presign (only on the DNS-only edge host); everything else keeps the
@@@ -229,8 +306,9 @@ async function serveMediaRecord(kind, record, req, res) {
const cacheHeaders = (isLiveSeekable || isLiveMp4)
? { 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache' }
: {};
++ const asFile = downloadDisposition(req, kind, record, path.extname(filePath));
-- streamFileWithRange(req, res, filePath, { ...cacheHeaders, ...noindex });
++ streamFileWithRange(req, res, filePath, { ...cacheHeaders, ...noindex, ...(asFile ? { 'Content-Disposition': asFile } : {}) });
}
// ── VOD playback ─────────────────────────────────────────────
ddiff --git a/test/media-downloads.test.js b/test/media-downloads.test.js
nnew file mode 100644
iindex 0000000..5d37b3d
---- /dev/null
++++ b/test/media-downloads.test.js
@@@ -0,0 +1,129 @@
++'use strict';
++// ?download on /v/:id and /c/:id (server/public/routes.js): the bytes as an attachment, so a browser saves them instead
++// of playing them (OpenVibe.Video's Download menu, through Live's download-url).
++// - A file on disk: Content-Disposition attachment, named after ?name with the file's extension (path, quote and
++// control characters dropped; the UTF-8 name too); without ?download nothing changes; a Range still works.
++// - A copy in B2/R2: the store's own presigned URL, asked with that disposition (never the shield).
++// - A virtual clip (no file of its own): its playlist remuxed into one MP4 as it is sent, with real ffmpeg over loopback;
++// a playlist that cannot be read is a 502 with nothing sent; more than two at once wait (503).
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const http = require('http');
++const net = require('net');
++const { spawnSync } = require('child_process');
++
++const freePort = () => new Promise((resolve) => { const s = net.createServer(); s.listen(0, '127.0.0.1', () => { const p = s.address().port; s.close(() => resolve(p)); }); });
++
++(async () => {
++ if (spawnSync('ffmpeg', ['-version']).status !== 0) { console.log('media downloads: skipped (ffmpeg not found)'); process.exit(0); }
++ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-media-downloads-'));
++ const dir = (n) => { const d = path.join(tmp, n); fs.mkdirSync(d, { recursive: true }); return d; };
++ process.env.VOD_PATH = dir('vods');
++ process.env.CLIPS_PATH = dir('clips');
++ process.env.FILES_PATH = dir('files');
++ process.env.OBJECTS_PATH = dir('objects');
++ process.env.THUMBNAILS_PATH = dir('thumbnails');
++ process.env.PASTES_PATH = dir('pastes');
++ process.env.MEDIA_PUBLIC_URL = 'https://media.test';
++ process.env.MEDIA_HLS_ENABLED = '1';
++ const port = await freePort();
++ process.env.PORT = String(port); // the remux reads the clip's playlist from this service over loopback
++
++ const ffmpeg = (...args) => { const r = spawnSync('ffmpeg', ['-v', 'error', '-y', ...args], { encoding: 'utf8' }); assert.strictEqual(r.status, 0, r.stderr); };
++ const vodFile = path.join(process.env.VOD_PATH, 'vod-live-7-1.mp4');
++ ffmpeg('-f', 'lavfi', '-i', 'testsrc=duration=4:size=160x90:rate=10', '-f', 'lavfi', '-i', 'sine=duration=4', '-c:v', 'mpeg4', '-c:a', 'aac', vodFile);
++ // A clip's HLS (fMP4 segments, as Media's timeline serves them) for the virtual clip's playlist.
++ const hls = dir('hls');
++ ffmpeg('-f', 'lavfi', '-i', 'testsrc=duration=6:size=160x90:rate=10', '-f', 'lavfi', '-i', 'sine=duration=6', '-c:v', 'mpeg4', '-g', '10', '-c:a', 'aac',
++ '-f', 'hls', '-hls_time', '2', '-hls_playlist_type', 'vod', '-hls_segment_type', 'fmp4', '-hls_fmp4_init_filename', 'init.mp4',
++ '-hls_segment_filename', path.join(hls, '%06d.m4s'), path.join(hls, 'index.m3u8'));
++
++ const config = require('../server/config');
++ assert.strictEqual(config.port, port);
++ config.hls.enabled = true;
++ const db = require('../server/db/database');
++ await db.upsertApp({ app_id: 'live', api_key: 'live-key' });
++ await db.run(`INSERT INTO vods (id, app_id, user_id, title, file_path, is_public, visibility, is_recording) OVERRIDING SYSTEM VALUE
++ VALUES (7, 'live', 5, 'Seven', ?, 1, 'public', 0)`, [vodFile]);
++ await db.run(`INSERT INTO vods (id, app_id, user_id, title, file_path, is_public, visibility, is_recording, storage_provider, storage_key) OVERRIDING SYSTEM VALUE
++ VALUES (8, 'live', 5, 'In the cloud', 'vod-live-8-1.mp4', 1, 'public', 0, 'b2', 'vods/vod-live-8-1.mp4')`);
++ for (const [id, obj] of [[60, 'med_virtual_ok'], [61, 'med_virtual_bad']]) {
++ await db.run(`INSERT INTO clips (id, app_id, user_id, vod_id, title, status, visibility, is_public, storage_provider, object_id) OVERRIDING SYSTEM VALUE
++ VALUES (?, 'live', 5, 7, 'Virtual', 'ready', 'public', 1, 'timeline', ?)`, [id, obj]);
++ }
++
++ const vodStorage = require('../server/vod/vod-storage');
++ const presigned = [];
++ vodStorage.resolvePlayback = async () => ({ kind: 'redirect', url: 'https://b2.test/plain', provider: 'b2', fallbacks: [] });
++ vodStorage.presignGet = async (provider, key, ttl, overrides = {}) => { presigned.push({ provider, key, ttl, ...overrides }); return 'https://b2.test/signed'; };
++
++ const express = require('express');
++ const app = express();
++ // The clip's playlist and segments, as Media's /o/:id/source/ routes would answer them (only the good object).
++ let held = null;
++ app.get('/o/med_virtual_ok/source/:name', (req, res, next) => (held ? held.push(next) : next()), (req, res) => {
++ const f = path.join(hls, path.basename(req.params.name));
++ if (!fs.existsSync(f)) return res.status(404).end();
++ res.sendFile(f);
++ });
++ app.use('/', require('../server/public/routes'));
++ const server = http.createServer(app);
++ await new Promise((r) => server.listen(port, '127.0.0.1', r));
++ const base = `http://127.0.0.1:${port}`;
++ const get = async (p, headers = {}) => { const r = await fetch(base + p, { headers, redirect: 'manual' }); return { status: r.status, headers: r.headers, body: Buffer.from(await r.arrayBuffer()) }; };
++ const raw = { accept: 'video/*' }; // a player's request: the bytes, not the watch page
++
++ // A file on disk.
++ const plain = await get('/v/7', raw);
++ assert.strictEqual(plain.status, 200);
++ assert.strictEqual(plain.headers.get('content-disposition'), null, 'playback is unchanged');
++ const dl = await get(`/v/7?download=1&name=${encodeURIComponent('Seven — "best"/bits')}`, raw);
++ assert.strictEqual(dl.status, 200);
++ assert.strictEqual(dl.headers.get('content-disposition'), 'attachment; filename="Seven _ best bits.mp4"; filename*=UTF-8\'\'Seven%20%E2%80%94%20best%20bits.mp4');
++ assert.ok(dl.body.equals(fs.readFileSync(vodFile)), 'the whole file');
++ const ranged = await get('/v/7?download=1', { ...raw, range: 'bytes=0-99' });
++ assert.strictEqual(ranged.status, 206);
++ assert.strictEqual(ranged.body.length, 100);
++ assert.strictEqual(ranged.headers.get('content-disposition'), 'attachment; filename="vod-7.mp4"; filename*=UTF-8\'\'vod-7.mp4', 'no ?name: vod-<id>');
++ const navigated = await get('/v/7?download=1', { accept: 'text/html', 'sec-fetch-dest': 'document' });
++ assert.strictEqual(navigated.headers.get('content-disposition') !== null, true, 'a browser navigation with ?download gets the file, not the watch page');
++
++ // A copy in the cloud: presigned with the disposition, not the plain playback URL.
++ const cloud = await get('/v/8?download=1&name=Cloud', raw);
++ assert.strictEqual(cloud.status, 302);
++ assert.strictEqual(cloud.headers.get('location'), 'https://b2.test/signed');
++ assert.deepStrictEqual(presigned.pop(), { provider: 'b2', key: 'vods/vod-live-8-1.mp4', ttl: 900, contentDisposition: 'attachment; filename="Cloud.mp4"; filename*=UTF-8\'\'Cloud.mp4' });
++ const play = await get('/v/8', raw);
++ assert.strictEqual(play.headers.get('location'), 'https://b2.test/plain', 'playback keeps its own URL');
++
++ // A virtual clip: one MP4 remuxed from its playlist.
++ const clip = await get('/c/60?download=1&name=Virtual', raw);
++ assert.strictEqual(clip.status, 200, clip.body.toString().slice(0, 200));
++ assert.strictEqual(clip.headers.get('content-type'), 'video/mp4');
++ assert.strictEqual(clip.headers.get('content-disposition'), 'attachment; filename="Virtual.mp4"; filename*=UTF-8\'\'Virtual.mp4');
++ const out = path.join(tmp, 'virtual.mp4');
++ fs.writeFileSync(out, clip.body);
++ const probe = spawnSync('ffprobe', ['-v', 'error', '-show_entries', 'format=duration:stream=codec_type', '-of', 'compact', out], { encoding: 'utf8' }).stdout;
++ assert.ok(/codec_type=video/.test(probe) && /codec_type=audio/.test(probe), probe);
++ const dur = Number((/duration=([\d.]+)/.exec(probe) || [])[1]);
++ assert.ok(Math.abs(dur - 6) < 0.5, `the whole clip (${dur} s)`);
++ const bad = await get('/c/61?download=1', raw);
++ assert.strictEqual(bad.status, 502, 'a playlist that cannot be read');
++ assert.deepStrictEqual(JSON.parse(bad.body.toString()), { error: 'The clip could not be prepared for download' });
++
++ // Two at once at most: a third waits.
++ held = [];
++ const first = [get('/c/60?download=1', raw), get('/c/60?download=1', raw)];
++ while (held.length < 2) await new Promise((r) => setTimeout(r, 20));
++ const third = await get('/c/60?download=1', raw);
++ assert.strictEqual(third.status, 503);
++ assert.strictEqual(third.headers.get('retry-after'), '5');
++ const waiting = held; held = null; waiting.forEach((next) => next());
++ for (const r of await Promise.all(first)) assert.strictEqual(r.status, 200);
++
++ server.close();
++ console.log('media downloads: ok');
++ process.exit(0);
++})().catch((err) => { console.error(err); process.exit(1); });