openvibers/openvibe.tools Commits

YT: the length and size limits are enforced, and /api/info is capped and cached. maxDuration (YT_MAX_DURATION, default 3 h) was configured but never checked: /api/info now answers downloadable: false with the reason (too long, or a live stream that has not ended) and the page shows it instead of the formats; /api/download refuses such a video from the cached info before yt-dlp starts; yt-dlp gets --match-filter 'duration<=N & !is_live' and prints the length at pre_process, so a skipped download says why instead of 'Output file not found'. --max-filesize (YT_MAX_FILESIZE_MB, default 2048) is passed, and the progress line now carries downloaded bytes, so a part that grows past the cap is stopped. /api/info ran yt-dlp uncapped: at most YT_INFO_CONCURRENCY (3) lookups run at once, 20 wait, more get 503 + Retry-After; the same video id is looked up once at a time and kept 10 minutes. /api/health shows the limits and the info queue. apps/yt/test/limits.test.js uses a stand-in yt-dlp

OP OpenVibers committed 23 Sept 2026 · cbc4155c38 · 5 files, +246 −8


Changes

ddiff --git a/apps/yt/public/js/app.js b/apps/yt/public/js/app.js
iindex 3005c68..43f53d6 100644
---- a/apps/yt/public/js/app.js
++++ b/apps/yt/public/js/app.js
@@@ -159,6 +159,13 @@
if (!info || !info.id) { showError('No video found at that link'); return; }
videoInfo = info;
renderVideoCard(info);
++ // Over the length limit (or still live): show the video and say why, instead of a
++ // download that would fail.
++ if (info.downloadable === false) {
++ showError(info.reason || 'This video cannot be downloaded here.');
++ show(videoCard);
++ return;
++ }
selectedQuality = 'best';
renderFormats();
hideAll();
ddiff --git a/apps/yt/server/config.js b/apps/yt/server/config.js
iindex fe9aa95..42e3f03 100644
---- a/apps/yt/server/config.js
++++ b/apps/yt/server/config.js
@@@ -9,13 +9,24 @@ module.exports = {
// yt-dlp binary path (auto-detect or override)
ytdlpPath: process.env.YTDLP_PATH || 'yt-dlp',
-- // Download limits
++ // Download limits (enforced: /api/info says so, /api/download refuses, yt-dlp gets --match-filter
++ // and --max-filesize, and a download is stopped when its reported length or size passes them)
download: {
-- maxDuration: 3 * 60 * 60, // 3 hours max video length (seconds)
++ maxDuration: parseInt(process.env.YT_MAX_DURATION, 10) || 3 * 60 * 60, // seconds (3 hours)
++ maxFilesize: parseInt(process.env.YT_MAX_FILESIZE_MB, 10) || 2048, // MB per downloaded part (2 GB)
timeout: 10 * 60 * 1000, // 10 min download timeout (ms)
maxConcurrent: 5, // max concurrent downloads
},
++ // Video info (/api/info): yt-dlp runs at most this many at once, others wait in a short queue;
++ // answers are kept per video id for a while.
++ info: {
++ maxConcurrent: parseInt(process.env.YT_INFO_CONCURRENCY, 10) || 3,
++ maxQueued: 20,
++ cacheTtlMs: 10 * 60 * 1000,
++ cacheMax: 500,
++ },
++
// Ephemeral file retention (ms)
retention: {
fileTTL: 60 * 60 * 1000, // 1 hour
ddiff --git a/apps/yt/server/downloader.js b/apps/yt/server/downloader.js
iindex 5f62465..76fb505 100644
---- a/apps/yt/server/downloader.js
++++ b/apps/yt/server/downloader.js
@@@ -70,6 +70,76 @@ function sanitizeUrl(url) {
}
}
++/** The YouTube video id of a link (watch?v=, youtu.be/, /shorts/, /live/, /embed/), or null. */
++function videoId(url) {
++ try {
++ const u = new URL(url);
++ let id = u.hostname === 'youtu.be' ? u.pathname.slice(1).split('/')[0] : u.searchParams.get('v');
++ if (!id) { const m = /^\/(?:shorts|live|embed|v)\/([^/?#]+)/.exec(u.pathname); if (m) id = m[1]; }
++ return id && /^[\w-]{6,20}$/.test(id) ? id : null;
++ } catch { return null; }
++}
++
++const sizeLabel = () => (config.download.maxFilesize >= 1024 ? `${Math.round(config.download.maxFilesize / 102.4) / 10} GB` : `${config.download.maxFilesize} MB`);
++const fmtHours = (s) => (s % 3600 === 0 ? `${s / 3600} hour${s === 3600 ? '' : 's'}` : `${Math.round(s / 60)} minutes`);
++
++/** Why this video cannot be downloaded here (null when it can). */
++function limitReason(duration, isLive) {
++ const max = config.download.maxDuration;
++ if (isLive) return 'This is a live stream that has not ended; it can be saved once it is over.';
++ if (Number.isFinite(duration) && duration > max) return `This video is ${fmtHours(Math.round(duration))} long; downloads are limited to ${fmtHours(max)}.`;
++ return null;
++}
++
++// ── Info: at most N yt-dlp runs at once, answers cached per video id ─────
++const infoCache = new Map(); // id → { info, expires }
++const infoInflight = new Map(); // id → Promise
++let infoRunning = 0;
++const infoQueue = [];
++
++function acquireInfoSlot() {
++ if (infoRunning < config.info.maxConcurrent) { infoRunning++; return Promise.resolve(); }
++ if (infoQueue.length >= config.info.maxQueued) return Promise.reject(Object.assign(new Error('The server is busy looking up other videos. Try again in a moment.'), { status: 503 }));
++ return new Promise(resolve => infoQueue.push(resolve));
++}
++function releaseInfoSlot() {
++ const next = infoQueue.shift();
++ if (next) next(); else infoRunning = Math.max(0, infoRunning - 1);
++}
++
++function cachedInfo(id) {
++ const hit = id && infoCache.get(id);
++ if (!hit) return null;
++ if (hit.expires <= Date.now()) { infoCache.delete(id); return null; }
++ return hit.info;
++}
++
++/** getInfo with the concurrency cap, the per-id cache and one run per id at a time. */
++function getInfoLimited(url, { run = getInfo } = {}) {
++ const id = videoId(url);
++ const hit = cachedInfo(id);
++ if (hit) return Promise.resolve(hit);
++ if (id && infoInflight.has(id)) return infoInflight.get(id);
++ const p = (async () => {
++ await acquireInfoSlot();
++ try {
++ const info = await run(url);
++ if (id) {
++ infoCache.set(id, { info, expires: Date.now() + config.info.cacheTtlMs });
++ while (infoCache.size > config.info.cacheMax) infoCache.delete(infoCache.keys().next().value);
++ }
++ return info;
++ } finally {
++ releaseInfoSlot();
++ if (id) infoInflight.delete(id);
++ }
++ })();
++ if (id) infoInflight.set(id, p);
++ return p;
++}
++
++function infoStats() { return { running: infoRunning, queued: infoQueue.length, cached: infoCache.size }; }
++
// ── Quality Presets ──────────────────────────────────────────
const QUALITY_PRESETS = {
'best': { video: 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best', ext: 'mp4' },
@@@ -132,7 +202,11 @@ function getInfo(url) {
uploaderUrl: info.uploader_url,
viewCount: info.view_count,
uploadDate: info.upload_date,
++ isLive: !!info.is_live,
formats: getAvailableFormats(info),
++ downloadable: !limitReason(info.duration, info.is_live),
++ ...(limitReason(info.duration, info.is_live) && { reason: limitReason(info.duration, info.is_live) }),
++ limits: { maxDuration: config.download.maxDuration, maxFilesizeMB: config.download.maxFilesize },
});
} catch (e) {
reject(new Error('Failed to parse video info'));
@@@ -233,8 +307,23 @@ const cleanField = (v) => {
* knows them (85/15 otherwise) and the overall figure never decreases.
*/
function applyLine(entry, line) {
++ if (line.startsWith('OVD|')) {
++ // pre_process: the video's length and live state, before the match filter and any download.
++ const [, dur, live] = line.split('|');
++ entry.probed = true;
++ const d = parseFloat(dur);
++ if (Number.isFinite(d)) entry.duration = d;
++ if (live === 'True') entry.isLive = true;
++ const why = limitReason(entry.duration, entry.isLive);
++ if (why && !entry.limitError) entry.limitError = why;
++ return;
++ }
if (line.startsWith('OVP|')) {
-- const [, pctRaw, speed, eta] = line.split('|');
++ const [, pctRaw, speed, eta, bytesRaw] = line.split('|');
++ const bytes = parseFloat(bytesRaw);
++ if (Number.isFinite(bytes) && bytes > config.download.maxFilesize * 1024 * 1024 && !entry.limitError) {
++ entry.limitError = `This download is larger than ${sizeLabel()}, the limit here.`;
++ }
const pct = parseFloat(pctRaw);
if (!Number.isFinite(pct)) return;
if (entry._lastPct != null && pct < entry._lastPct - 40 && entry._part < entry._weights.length - 1) entry._part++;
@@@ -325,9 +414,15 @@ function startDownload(url, quality = 'best', opts = {}) {
'--no-simulate',
'--progress',
'--newline', // progress on new lines for parsing
++ // Limits: a video longer than maxDuration (or a live stream without a length) does not
++ // pass the filter, a part larger than maxFilesize is not downloaded; OVD reports the
++ // length before either happens so the error can say why.
++ '--match-filter', `duration<=${config.download.maxDuration} & !is_live`,
++ '--max-filesize', `${config.download.maxFilesize}M`,
++ '--print', 'pre_process:OVD|%(duration)s|%(is_live)s',
'--print', 'before_dl:OVM|%(requested_formats.:.filesize)j|%(requested_formats.:.filesize_approx)j|%(title)j',
'--print', 'post_process:OVX|pp',
-- '--progress-template', 'download:OVP|%(progress._percent_str)s|%(progress._speed_str)s|%(progress._eta_str)s',
++ '--progress-template', 'download:OVP|%(progress._percent_str)s|%(progress._speed_str)s|%(progress._eta_str)s|%(progress.downloaded_bytes)s',
...(preset.postprocess || []),
cleanUrl,
];
@@@ -362,6 +457,14 @@ function startDownload(url, quality = 'best', opts = {}) {
const lines = (stdoutRest + data.toString()).split('\n');
stdoutRest = lines.pop();
for (const line of lines) applyLine(entry, line.trim());
++ // Over a limit: stop now rather than after the whole file arrived.
++ if (entry.limitError && entry.status === 'downloading') {
++ entry.status = 'error';
++ entry.error = entry.limitError;
++ entry.finishedAt = Date.now();
++ try { proc.kill('SIGTERM'); } catch { /* gone */ }
++ setTimeout(() => { try { proc.kill('SIGKILL'); } catch { /* gone */ } }, 3000).unref();
++ }
});
let stderrTail = '';
@@@ -391,8 +494,12 @@ function startDownload(url, quality = 'best', opts = {}) {
const dir = path.resolve(config.downloadsDir);
const files = fs.readdirSync(dir).filter(f => f.startsWith(id) && !/\.(part|ytdl|temp)$/i.test(f));
if (files.length === 0) {
++ // yt-dlp exits 0 when --match-filter or --max-filesize skipped the video.
entry.status = 'error';
-- entry.error = 'Output file not found';
++ entry.error = entry.limitError
++ || (entry.probed && !Number.isFinite(entry.duration) ? 'This video has no known length (a live or upcoming stream); it can be saved once it has ended.' : null)
++ || `This video is over the limits here: at most ${fmtHours(config.download.maxDuration)} long and ${sizeLabel()} per file.`;
++ removePartials(id);
return;
}
@@@ -609,7 +716,7 @@ function getUpstream() { return upstream; }
function noteUpstream(state) { if (state === 'ok' || state === 'blocked') upstream = { state, checkedAt: Date.now(), detail: state === 'blocked' ? 'YouTube is refusing requests from this server address' : '' }; }
module.exports = {
-- getInfo, startDownload, cancelDownload, getStatus, getFile, removeFile,
++ getInfo, getInfoLimited, cachedInfo, videoId, limitReason, infoStats, startDownload, cancelDownload, getStatus, getFile, removeFile,
sanitizeTitle, downloadFilename, contentDisposition, applyLine,
cleanup, startCleanup, stopCleanup, getStats, isValidUrl, probeUpstream, startUpstreamProbe, getUpstream,
};
ddiff --git a/apps/yt/server/index.js b/apps/yt/server/index.js
iindex e992af6..075a872 100644
---- a/apps/yt/server/index.js
++++ b/apps/yt/server/index.js
@@@ -120,7 +120,7 @@ app.use(hostGuard({ knows: seo.knowsHost, aliasOf: seo.aliasOf }));
app.get('/api/health', (_req, res) => {
const stats = downloader.getStats();
const up = downloader.getUpstream();
-- res.set('Cache-Control', 'no-store').json({ status: 'ok', service: 'openvibe-yt', version: '1.0.0', stats, youtube: up.state, youtubeCheckedAt: up.checkedAt ? new Date(up.checkedAt).toISOString() : null });
++ res.set('Cache-Control', 'no-store').json({ status: 'ok', service: 'openvibe-yt', version: '1.0.0', stats, info: downloader.infoStats(), limits: { maxDuration: config.download.maxDuration, maxFilesizeMB: config.download.maxFilesize }, youtube: up.state, youtubeCheckedAt: up.checkedAt ? new Date(up.checkedAt).toISOString() : null });
});
// Get video info
@@@ -130,9 +130,12 @@ app.post('/api/info', async (req, res) => {
if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported (youtube.com, youtu.be)' });
try {
-- const info = await downloader.getInfo(url);
++ // At most config.info.maxConcurrent yt-dlp lookups at once; the same video is looked up once
++ // per ten minutes however many people paste it.
++ const info = await downloader.getInfoLimited(url);
res.json({ success: true, video: info });
} catch (err) {
++ if (err.status === 503) return res.status(503).set('Retry-After', '5').json({ error: err.message });
console.error('[Info] Error:', err.message);
res.status(422).json({ error: err.message });
}
@@@ -143,6 +146,9 @@ app.post('/api/download', downloadLimiter, async (req, res) => {
const { url, quality, title } = req.body;
if (!url) return res.status(400).json({ error: 'URL is required' });
if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported' });
++ // The length is known from the info lookup the page made first: refuse before yt-dlp starts.
++ const known = downloader.cachedInfo(downloader.videoId(url));
++ if (known && known.downloadable === false) return res.status(422).json({ error: known.reason });
try {
const { id } = await downloader.startDownload(url, quality || 'best', { title: typeof title === 'string' ? title : '' });
ddiff --git a/apps/yt/test/limits.test.js b/apps/yt/test/limits.test.js
nnew file mode 100644
iindex 0000000..ede8c37
---- /dev/null
++++ b/apps/yt/test/limits.test.js
@@@ -0,0 +1,107 @@
++'use strict';
++// YT limits: maxDuration and the file size cap are enforced (they were configured but never checked),
++// and /api/info runs at most N yt-dlp lookups at once with a per-video cache. yt-dlp is a stand-in
++// script here: nothing reaches YouTube.
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++
++process.env.YT_MAX_DURATION = '600'; // 10 minutes, before config is loaded
++process.env.YT_MAX_FILESIZE_MB = '1';
++process.env.YT_INFO_CONCURRENCY = '3';
++const downloader = require('../server/downloader');
++const { startApp } = require('../../_shared/test/spawn');
++
++const sleep = (ms) => new Promise(r => setTimeout(r, ms));
++
++(async () => {
++ // ── Video ids ──
++ const id = downloader.videoId;
++ assert.strictEqual(id('https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=10'), 'dQw4w9WgXcQ');
++ assert.strictEqual(id('https://youtu.be/dQw4w9WgXcQ?si=x'), 'dQw4w9WgXcQ');
++ assert.strictEqual(id('https://www.youtube.com/shorts/abcDEF12345'), 'abcDEF12345');
++ assert.strictEqual(id('https://www.youtube.com/'), null);
++
++ // ── Progress lines: over the length or the size limit sets the error before the file is done ──
++ let e = { _weights: [1], _part: 0, progress: null };
++ downloader.applyLine(e, 'OVD|601|False');
++ assert.match(e.limitError, /10 minutes long; downloads are limited to 10 minutes/);
++ e = { _weights: [1], _part: 0, progress: null };
++ downloader.applyLine(e, 'OVD|NA|True');
++ assert.match(e.limitError, /live stream/);
++ e = { _weights: [1], _part: 0, progress: null };
++ downloader.applyLine(e, 'OVD|120|False');
++ assert.strictEqual(e.limitError, undefined);
++ downloader.applyLine(e, `OVP| 50.0%|1MiB/s|00:01|${2 * 1024 * 1024}`);
++ assert.match(e.limitError, /larger than 1 MB/);
++ e = { _weights: [1], _part: 0, progress: null };
++ downloader.applyLine(e, 'OVP| 50.0%|1MiB/s|00:01'); // the old four-field line still parses
++ assert.strictEqual(e.progress, 50);
++
++ // ── Info: at most 3 yt-dlp runs at once, one per video id, cached ──
++ let running = 0, peak = 0, runs = 0;
++ const run = async (url) => { runs++; running++; peak = Math.max(peak, running); await sleep(30); running--; return { id: downloader.videoId(url), duration: 60, downloadable: true }; };
++ const urls = Array.from({ length: 8 }, (_, i) => `https://www.youtube.com/watch?v=video${String(i).padStart(6, '0')}`);
++ await Promise.all([...urls, ...urls].map(u => downloader.getInfoLimited(u, { run })));
++ assert.strictEqual(peak, 3, 'never more than three lookups at once');
++ assert.strictEqual(runs, 8, 'the same video is looked up once even when asked twice at the same time');
++ await downloader.getInfoLimited(urls[0], { run });
++ assert.strictEqual(runs, 8, 'cached');
++ // A full queue answers 503 instead of piling up processes.
++ const slow = () => new Promise(r => setTimeout(() => r({ duration: 1 }), 200));
++ const flood = Array.from({ length: 30 }, (_, i) => downloader.getInfoLimited(`https://youtu.be/flood${String(i).padStart(6, '0')}`, { run: slow }).then(() => 'ok', err => err.status));
++ const outcomes = await Promise.all(flood);
++ assert.strictEqual(outcomes.filter(o => o === 'ok').length, 23, '3 running + 20 queued');
++ assert.strictEqual(outcomes.filter(o => o === 503).length, 7);
++
++ // ── As a real process with a stand-in yt-dlp ──
++ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-yt-'));
++ const fake = path.join(data, 'yt-dlp');
++ fs.writeFileSync(fake, `#!${process.execPath}
++const args = process.argv.slice(2);
++const url = args[args.length - 1];
++const long = url.includes('LONGLONG123');
++if (args.includes('--dump-json')) {
++ process.stdout.write(JSON.stringify({ id: long ? 'LONGLONG123' : 'SHORTxx1234', title: 't', duration: long ? 4000 : 30, formats: [{ vcodec: 'avc1', acodec: 'mp4a' }] }));
++ process.exit(0);
++}
++const i = args.indexOf('--match-filter');
++require('fs').writeFileSync(${JSON.stringify(path.join(data, 'args.json'))}, JSON.stringify(args));
++process.stdout.write('OVD|' + (long ? 4000 : 30) + '|False\\n');
++process.exit(0); // the match filter skipped it: no file
++`, { mode: 0o755 });
++ const app = await startApp('yt', { DATA_DIR: data, DOWNLOADS_DIR: path.join(data, 'downloads'), YTDLP_PATH: fake, YT_MAX_DURATION: '600', YT_MAX_FILESIZE_MB: '100' });
++ try {
++ const post = (p, body) => fetch(`${app.base}${p}`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
++ let r = await post('/api/info', { url: 'https://www.youtube.com/watch?v=LONGLONG123' });
++ let body = await r.json();
++ assert.strictEqual(r.status, 200);
++ assert.strictEqual(body.video.downloadable, false);
++ assert.match(body.video.reason, /67 minutes long; downloads are limited to 10 minutes/);
++ assert.deepStrictEqual(body.video.limits, { maxDuration: 600, maxFilesizeMB: 100 });
++ r = await post('/api/download', { url: 'https://youtu.be/LONGLONG123', quality: 'best' });
++ assert.strictEqual(r.status, 422, 'refused from the cached info, before yt-dlp starts');
++ assert.match((await r.json()).error, /limited to 10 minutes/);
++ assert.ok(!fs.existsSync(path.join(data, 'args.json')), 'no download was started');
++
++ // Without cached info, yt-dlp gets the filter and the size cap, and the skip is explained.
++ r = await post('/api/download', { url: 'https://www.youtube.com/watch?v=SHORTxx1234', quality: 'best' });
++ body = await r.json();
++ assert.strictEqual(r.status, 200, JSON.stringify(body));
++ let st;
++ for (let i = 0; i < 50; i++) { st = await (await fetch(`${app.base}/api/status/${body.id}`)).json(); if (st.status !== 'downloading') break; await sleep(50); }
++ const args = JSON.parse(fs.readFileSync(path.join(data, 'args.json'), 'utf8'));
++ assert.strictEqual(args[args.indexOf('--match-filter') + 1], 'duration<=600 & !is_live');
++ assert.strictEqual(args[args.indexOf('--max-filesize') + 1], '100M');
++ assert.ok(args.includes('pre_process:OVD|%(duration)s|%(is_live)s'));
++ assert.strictEqual(st.status, 'error');
++ assert.match(st.error, /over the limits here: at most 10 minutes long and 100 MB per file/);
++ const health = await (await fetch(`${app.base}/api/health`)).json();
++ assert.deepStrictEqual(health.limits, { maxDuration: 600, maxFilesizeMB: 100 });
++ } finally {
++ await app.kill();
++ fs.rmSync(data, { recursive: true, force: true });
++ }
++ console.log('yt limits (duration, file size, info concurrency + cache): all checks passed');
++})().catch((err) => { console.error(err); process.exit(1); });