openvibers/openvibe.tools Commits
YT: the length and size limits are enforced, and /api/info is capped and cached. maxDuration (YT_MAX_DURATION, default 3 h) was configured but never checked: /api/info now answers downloadable: false with the reason (too long, or a live stream that has not ended) and the page shows it instead of the formats; /api/download refuses such a video from the cached info before yt-dlp starts; yt-dlp gets --match-filter 'duration<=N & !is_live' and prints the length at pre_process, so a skipped download says why instead of 'Output file not found'. --max-filesize (YT_MAX_FILESIZE_MB, default 2048) is passed, and the progress line now carries downloaded bytes, so a part that grows past the cap is stopped. /api/info ran yt-dlp uncapped: at most YT_INFO_CONCURRENCY (3) lookups run at once, 20 wait, more get 503 + Retry-After; the same video id is looked up once at a time and kept 10 minutes. /api/health shows the limits and the info queue. apps/yt/test/limits.test.js uses a stand-in yt-dlp
Changes
@@@ -159,6 +159,13 @@if (!info || !info.id) { showError('No video found at that link'); return; }videoInfo = info;renderVideoCard(info);++ // Over the length limit (or still live): show the video and say why, instead of a++ // download that would fail.++ if (info.downloadable === false) {++ showError(info.reason || 'This video cannot be downloaded here.');++ show(videoCard);++ return;++ }selectedQuality = 'best';renderFormats();hideAll();@@@ -9,13 +9,24 @@ module.exports = {// yt-dlp binary path (auto-detect or override)ytdlpPath: process.env.YTDLP_PATH || 'yt-dlp',-- // Download limits++ // Download limits (enforced: /api/info says so, /api/download refuses, yt-dlp gets --match-filter++ // and --max-filesize, and a download is stopped when its reported length or size passes them)download: {-- maxDuration: 3 * 60 * 60, // 3 hours max video length (seconds)++ maxDuration: parseInt(process.env.YT_MAX_DURATION, 10) || 3 * 60 * 60, // seconds (3 hours)++ maxFilesize: parseInt(process.env.YT_MAX_FILESIZE_MB, 10) || 2048, // MB per downloaded part (2 GB)timeout: 10 * 60 * 1000, // 10 min download timeout (ms)maxConcurrent: 5, // max concurrent downloads},++ // Video info (/api/info): yt-dlp runs at most this many at once, others wait in a short queue;++ // answers are kept per video id for a while.++ info: {++ maxConcurrent: parseInt(process.env.YT_INFO_CONCURRENCY, 10) || 3,++ maxQueued: 20,++ cacheTtlMs: 10 * 60 * 1000,++ cacheMax: 500,++ },++// Ephemeral file retention (ms)retention: {fileTTL: 60 * 60 * 1000, // 1 hour@@@ -70,6 +70,76 @@ function sanitizeUrl(url) {}}++/** The YouTube video id of a link (watch?v=, youtu.be/, /shorts/, /live/, /embed/), or null. */++function videoId(url) {++ try {++ const u = new URL(url);++ let id = u.hostname === 'youtu.be' ? u.pathname.slice(1).split('/')[0] : u.searchParams.get('v');++ if (!id) { const m = /^\/(?:shorts|live|embed|v)\/([^/?#]+)/.exec(u.pathname); if (m) id = m[1]; }++ return id && /^[\w-]{6,20}$/.test(id) ? id : null;++ } catch { return null; }++}++++const sizeLabel = () => (config.download.maxFilesize >= 1024 ? `${Math.round(config.download.maxFilesize / 102.4) / 10} GB` : `${config.download.maxFilesize} MB`);++const fmtHours = (s) => (s % 3600 === 0 ? `${s / 3600} hour${s === 3600 ? '' : 's'}` : `${Math.round(s / 60)} minutes`);++++/** Why this video cannot be downloaded here (null when it can). */++function limitReason(duration, isLive) {++ const max = config.download.maxDuration;++ if (isLive) return 'This is a live stream that has not ended; it can be saved once it is over.';++ if (Number.isFinite(duration) && duration > max) return `This video is ${fmtHours(Math.round(duration))} long; downloads are limited to ${fmtHours(max)}.`;++ return null;++}++++// ── Info: at most N yt-dlp runs at once, answers cached per video id ─────++const infoCache = new Map(); // id → { info, expires }++const infoInflight = new Map(); // id → Promise++let infoRunning = 0;++const infoQueue = [];++++function acquireInfoSlot() {++ if (infoRunning < config.info.maxConcurrent) { infoRunning++; return Promise.resolve(); }++ if (infoQueue.length >= config.info.maxQueued) return Promise.reject(Object.assign(new Error('The server is busy looking up other videos. Try again in a moment.'), { status: 503 }));++ return new Promise(resolve => infoQueue.push(resolve));++}++function releaseInfoSlot() {++ const next = infoQueue.shift();++ if (next) next(); else infoRunning = Math.max(0, infoRunning - 1);++}++++function cachedInfo(id) {++ const hit = id && infoCache.get(id);++ if (!hit) return null;++ if (hit.expires <= Date.now()) { infoCache.delete(id); return null; }++ return hit.info;++}++++/** getInfo with the concurrency cap, the per-id cache and one run per id at a time. */++function getInfoLimited(url, { run = getInfo } = {}) {++ const id = videoId(url);++ const hit = cachedInfo(id);++ if (hit) return Promise.resolve(hit);++ if (id && infoInflight.has(id)) return infoInflight.get(id);++ const p = (async () => {++ await acquireInfoSlot();++ try {++ const info = await run(url);++ if (id) {++ infoCache.set(id, { info, expires: Date.now() + config.info.cacheTtlMs });++ while (infoCache.size > config.info.cacheMax) infoCache.delete(infoCache.keys().next().value);++ }++ return info;++ } finally {++ releaseInfoSlot();++ if (id) infoInflight.delete(id);++ }++ })();++ if (id) infoInflight.set(id, p);++ return p;++}++++function infoStats() { return { running: infoRunning, queued: infoQueue.length, cached: infoCache.size }; }++// ── Quality Presets ──────────────────────────────────────────const QUALITY_PRESETS = {'best': { video: 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best', ext: 'mp4' },@@@ -132,7 +202,11 @@ function getInfo(url) {uploaderUrl: info.uploader_url,viewCount: info.view_count,uploadDate: info.upload_date,++ isLive: !!info.is_live,formats: getAvailableFormats(info),++ downloadable: !limitReason(info.duration, info.is_live),++ ...(limitReason(info.duration, info.is_live) && { reason: limitReason(info.duration, info.is_live) }),++ limits: { maxDuration: config.download.maxDuration, maxFilesizeMB: config.download.maxFilesize },});} catch (e) {reject(new Error('Failed to parse video info'));@@@ -233,8 +307,23 @@ const cleanField = (v) => {* knows them (85/15 otherwise) and the overall figure never decreases.*/function applyLine(entry, line) {++ if (line.startsWith('OVD|')) {++ // pre_process: the video's length and live state, before the match filter and any download.++ const [, dur, live] = line.split('|');++ entry.probed = true;++ const d = parseFloat(dur);++ if (Number.isFinite(d)) entry.duration = d;++ if (live === 'True') entry.isLive = true;++ const why = limitReason(entry.duration, entry.isLive);++ if (why && !entry.limitError) entry.limitError = why;++ return;++ }if (line.startsWith('OVP|')) {-- const [, pctRaw, speed, eta] = line.split('|');++ const [, pctRaw, speed, eta, bytesRaw] = line.split('|');++ const bytes = parseFloat(bytesRaw);++ if (Number.isFinite(bytes) && bytes > config.download.maxFilesize * 1024 * 1024 && !entry.limitError) {++ entry.limitError = `This download is larger than ${sizeLabel()}, the limit here.`;++ }const pct = parseFloat(pctRaw);if (!Number.isFinite(pct)) return;if (entry._lastPct != null && pct < entry._lastPct - 40 && entry._part < entry._weights.length - 1) entry._part++;@@@ -325,9 +414,15 @@ function startDownload(url, quality = 'best', opts = {}) {'--no-simulate','--progress','--newline', // progress on new lines for parsing++ // Limits: a video longer than maxDuration (or a live stream without a length) does not++ // pass the filter, a part larger than maxFilesize is not downloaded; OVD reports the++ // length before either happens so the error can say why.++ '--match-filter', `duration<=${config.download.maxDuration} & !is_live`,++ '--max-filesize', `${config.download.maxFilesize}M`,++ '--print', 'pre_process:OVD|%(duration)s|%(is_live)s','--print', 'before_dl:OVM|%(requested_formats.:.filesize)j|%(requested_formats.:.filesize_approx)j|%(title)j','--print', 'post_process:OVX|pp',-- '--progress-template', 'download:OVP|%(progress._percent_str)s|%(progress._speed_str)s|%(progress._eta_str)s',++ '--progress-template', 'download:OVP|%(progress._percent_str)s|%(progress._speed_str)s|%(progress._eta_str)s|%(progress.downloaded_bytes)s',...(preset.postprocess || []),cleanUrl,];@@@ -362,6 +457,14 @@ function startDownload(url, quality = 'best', opts = {}) {const lines = (stdoutRest + data.toString()).split('\n');stdoutRest = lines.pop();for (const line of lines) applyLine(entry, line.trim());++ // Over a limit: stop now rather than after the whole file arrived.++ if (entry.limitError && entry.status === 'downloading') {++ entry.status = 'error';++ entry.error = entry.limitError;++ entry.finishedAt = Date.now();++ try { proc.kill('SIGTERM'); } catch { /* gone */ }++ setTimeout(() => { try { proc.kill('SIGKILL'); } catch { /* gone */ } }, 3000).unref();++ }});let stderrTail = '';@@@ -391,8 +494,12 @@ function startDownload(url, quality = 'best', opts = {}) {const dir = path.resolve(config.downloadsDir);const files = fs.readdirSync(dir).filter(f => f.startsWith(id) && !/\.(part|ytdl|temp)$/i.test(f));if (files.length === 0) {++ // yt-dlp exits 0 when --match-filter or --max-filesize skipped the video.entry.status = 'error';-- entry.error = 'Output file not found';++ entry.error = entry.limitError++ || (entry.probed && !Number.isFinite(entry.duration) ? 'This video has no known length (a live or upcoming stream); it can be saved once it has ended.' : null)++ || `This video is over the limits here: at most ${fmtHours(config.download.maxDuration)} long and ${sizeLabel()} per file.`;++ removePartials(id);return;}@@@ -609,7 +716,7 @@ function getUpstream() { return upstream; }function noteUpstream(state) { if (state === 'ok' || state === 'blocked') upstream = { state, checkedAt: Date.now(), detail: state === 'blocked' ? 'YouTube is refusing requests from this server address' : '' }; }module.exports = {-- getInfo, startDownload, cancelDownload, getStatus, getFile, removeFile,++ getInfo, getInfoLimited, cachedInfo, videoId, limitReason, infoStats, startDownload, cancelDownload, getStatus, getFile, removeFile,sanitizeTitle, downloadFilename, contentDisposition, applyLine,cleanup, startCleanup, stopCleanup, getStats, isValidUrl, probeUpstream, startUpstreamProbe, getUpstream,};@@@ -120,7 +120,7 @@ app.use(hostGuard({ knows: seo.knowsHost, aliasOf: seo.aliasOf }));app.get('/api/health', (_req, res) => {const stats = downloader.getStats();const up = downloader.getUpstream();-- res.set('Cache-Control', 'no-store').json({ status: 'ok', service: 'openvibe-yt', version: '1.0.0', stats, youtube: up.state, youtubeCheckedAt: up.checkedAt ? new Date(up.checkedAt).toISOString() : null });++ res.set('Cache-Control', 'no-store').json({ status: 'ok', service: 'openvibe-yt', version: '1.0.0', stats, info: downloader.infoStats(), limits: { maxDuration: config.download.maxDuration, maxFilesizeMB: config.download.maxFilesize }, youtube: up.state, youtubeCheckedAt: up.checkedAt ? new Date(up.checkedAt).toISOString() : null });});// Get video info@@@ -130,9 +130,12 @@ app.post('/api/info', async (req, res) => {if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported (youtube.com, youtu.be)' });try {-- const info = await downloader.getInfo(url);++ // At most config.info.maxConcurrent yt-dlp lookups at once; the same video is looked up once++ // per ten minutes however many people paste it.++ const info = await downloader.getInfoLimited(url);res.json({ success: true, video: info });} catch (err) {++ if (err.status === 503) return res.status(503).set('Retry-After', '5').json({ error: err.message });console.error('[Info] Error:', err.message);res.status(422).json({ error: err.message });}@@@ -143,6 +146,9 @@ app.post('/api/download', downloadLimiter, async (req, res) => {const { url, quality, title } = req.body;if (!url) return res.status(400).json({ error: 'URL is required' });if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported' });++ // The length is known from the info lookup the page made first: refuse before yt-dlp starts.++ const known = downloader.cachedInfo(downloader.videoId(url));++ if (known && known.downloadable === false) return res.status(422).json({ error: known.reason });try {const { id } = await downloader.startDownload(url, quality || 'best', { title: typeof title === 'string' ? title : '' });nnew file mode 100644@@@ -0,0 +1,107 @@++'use strict';++// YT limits: maxDuration and the file size cap are enforced (they were configured but never checked),++// and /api/info runs at most N yt-dlp lookups at once with a per-video cache. yt-dlp is a stand-in++// script here: nothing reaches YouTube.++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++++process.env.YT_MAX_DURATION = '600'; // 10 minutes, before config is loaded++process.env.YT_MAX_FILESIZE_MB = '1';++process.env.YT_INFO_CONCURRENCY = '3';++const downloader = require('../server/downloader');++const { startApp } = require('../../_shared/test/spawn');++++const sleep = (ms) => new Promise(r => setTimeout(r, ms));++++(async () => {++ // ── Video ids ──++ const id = downloader.videoId;++ assert.strictEqual(id('https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=10'), 'dQw4w9WgXcQ');++ assert.strictEqual(id('https://youtu.be/dQw4w9WgXcQ?si=x'), 'dQw4w9WgXcQ');++ assert.strictEqual(id('https://www.youtube.com/shorts/abcDEF12345'), 'abcDEF12345');++ assert.strictEqual(id('https://www.youtube.com/'), null);++++ // ── Progress lines: over the length or the size limit sets the error before the file is done ──++ let e = { _weights: [1], _part: 0, progress: null };++ downloader.applyLine(e, 'OVD|601|False');++ assert.match(e.limitError, /10 minutes long; downloads are limited to 10 minutes/);++ e = { _weights: [1], _part: 0, progress: null };++ downloader.applyLine(e, 'OVD|NA|True');++ assert.match(e.limitError, /live stream/);++ e = { _weights: [1], _part: 0, progress: null };++ downloader.applyLine(e, 'OVD|120|False');++ assert.strictEqual(e.limitError, undefined);++ downloader.applyLine(e, `OVP| 50.0%|1MiB/s|00:01|${2 * 1024 * 1024}`);++ assert.match(e.limitError, /larger than 1 MB/);++ e = { _weights: [1], _part: 0, progress: null };++ downloader.applyLine(e, 'OVP| 50.0%|1MiB/s|00:01'); // the old four-field line still parses++ assert.strictEqual(e.progress, 50);++++ // ── Info: at most 3 yt-dlp runs at once, one per video id, cached ──++ let running = 0, peak = 0, runs = 0;++ const run = async (url) => { runs++; running++; peak = Math.max(peak, running); await sleep(30); running--; return { id: downloader.videoId(url), duration: 60, downloadable: true }; };++ const urls = Array.from({ length: 8 }, (_, i) => `https://www.youtube.com/watch?v=video${String(i).padStart(6, '0')}`);++ await Promise.all([...urls, ...urls].map(u => downloader.getInfoLimited(u, { run })));++ assert.strictEqual(peak, 3, 'never more than three lookups at once');++ assert.strictEqual(runs, 8, 'the same video is looked up once even when asked twice at the same time');++ await downloader.getInfoLimited(urls[0], { run });++ assert.strictEqual(runs, 8, 'cached');++ // A full queue answers 503 instead of piling up processes.++ const slow = () => new Promise(r => setTimeout(() => r({ duration: 1 }), 200));++ const flood = Array.from({ length: 30 }, (_, i) => downloader.getInfoLimited(`https://youtu.be/flood${String(i).padStart(6, '0')}`, { run: slow }).then(() => 'ok', err => err.status));++ const outcomes = await Promise.all(flood);++ assert.strictEqual(outcomes.filter(o => o === 'ok').length, 23, '3 running + 20 queued');++ assert.strictEqual(outcomes.filter(o => o === 503).length, 7);++++ // ── As a real process with a stand-in yt-dlp ──++ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-yt-'));++ const fake = path.join(data, 'yt-dlp');++ fs.writeFileSync(fake, `#!${process.execPath}++const args = process.argv.slice(2);++const url = args[args.length - 1];++const long = url.includes('LONGLONG123');++if (args.includes('--dump-json')) {++ process.stdout.write(JSON.stringify({ id: long ? 'LONGLONG123' : 'SHORTxx1234', title: 't', duration: long ? 4000 : 30, formats: [{ vcodec: 'avc1', acodec: 'mp4a' }] }));++ process.exit(0);++}++const i = args.indexOf('--match-filter');++require('fs').writeFileSync(${JSON.stringify(path.join(data, 'args.json'))}, JSON.stringify(args));++process.stdout.write('OVD|' + (long ? 4000 : 30) + '|False\\n');++process.exit(0); // the match filter skipped it: no file++`, { mode: 0o755 });++ const app = await startApp('yt', { DATA_DIR: data, DOWNLOADS_DIR: path.join(data, 'downloads'), YTDLP_PATH: fake, YT_MAX_DURATION: '600', YT_MAX_FILESIZE_MB: '100' });++ try {++ const post = (p, body) => fetch(`${app.base}${p}`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });++ let r = await post('/api/info', { url: 'https://www.youtube.com/watch?v=LONGLONG123' });++ let body = await r.json();++ assert.strictEqual(r.status, 200);++ assert.strictEqual(body.video.downloadable, false);++ assert.match(body.video.reason, /67 minutes long; downloads are limited to 10 minutes/);++ assert.deepStrictEqual(body.video.limits, { maxDuration: 600, maxFilesizeMB: 100 });++ r = await post('/api/download', { url: 'https://youtu.be/LONGLONG123', quality: 'best' });++ assert.strictEqual(r.status, 422, 'refused from the cached info, before yt-dlp starts');++ assert.match((await r.json()).error, /limited to 10 minutes/);++ assert.ok(!fs.existsSync(path.join(data, 'args.json')), 'no download was started');++++ // Without cached info, yt-dlp gets the filter and the size cap, and the skip is explained.++ r = await post('/api/download', { url: 'https://www.youtube.com/watch?v=SHORTxx1234', quality: 'best' });++ body = await r.json();++ assert.strictEqual(r.status, 200, JSON.stringify(body));++ let st;++ for (let i = 0; i < 50; i++) { st = await (await fetch(`${app.base}/api/status/${body.id}`)).json(); if (st.status !== 'downloading') break; await sleep(50); }++ const args = JSON.parse(fs.readFileSync(path.join(data, 'args.json'), 'utf8'));++ assert.strictEqual(args[args.indexOf('--match-filter') + 1], 'duration<=600 & !is_live');++ assert.strictEqual(args[args.indexOf('--max-filesize') + 1], '100M');++ assert.ok(args.includes('pre_process:OVD|%(duration)s|%(is_live)s'));++ assert.strictEqual(st.status, 'error');++ assert.match(st.error, /over the limits here: at most 10 minutes long and 100 MB per file/);++ const health = await (await fetch(`${app.base}/api/health`)).json();++ assert.deepStrictEqual(health.limits, { maxDuration: 600, maxFilesizeMB: 100 });++ } finally {++ await app.kill();++ fs.rmSync(data, { recursive: true, force: true });++ }++ console.log('yt limits (duration, file size, info concurrency + cache): all checks passed');++})().catch((err) => { console.error(err); process.exit(1); });
@@ -159,6 +159,13 @@
@@ -159,6 +159,13 @@
if (!info || !info.id) { showError('No video found at that link'); return; }
if (!info || !info.id) { showError('No video found at that link'); return; }
videoInfo = info;
videoInfo = info;
renderVideoCard(info);
renderVideoCard(info);
+ // Over the length limit (or still live): show the video and say why, instead of a
+ // download that would fail.
+ if (info.downloadable === false) {
+ showError(info.reason || 'This video cannot be downloaded here.');
+ show(videoCard);
+ return;
+ }
selectedQuality = 'best';
selectedQuality = 'best';
renderFormats();
renderFormats();
hideAll();
hideAll();
@@ -9,13 +9,24 @@ module.exports = {
@@ -9,13 +9,24 @@ module.exports = {
// yt-dlp binary path (auto-detect or override)
// yt-dlp binary path (auto-detect or override)
ytdlpPath: process.env.YTDLP_PATH || 'yt-dlp',
ytdlpPath: process.env.YTDLP_PATH || 'yt-dlp',
- // Download limits
+ // Download limits (enforced: /api/info says so, /api/download refuses, yt-dlp gets --match-filter
+ // and --max-filesize, and a download is stopped when its reported length or size passes them)
download: {
download: {
- maxDuration: 3 * 60 * 60, // 3 hours max video length (seconds)
+ maxDuration: parseInt(process.env.YT_MAX_DURATION, 10) || 3 * 60 * 60, // seconds (3 hours)
+ maxFilesize: parseInt(process.env.YT_MAX_FILESIZE_MB, 10) || 2048, // MB per downloaded part (2 GB)
timeout: 10 * 60 * 1000, // 10 min download timeout (ms)
timeout: 10 * 60 * 1000, // 10 min download timeout (ms)
maxConcurrent: 5, // max concurrent downloads
maxConcurrent: 5, // max concurrent downloads
},
},
+ // Video info (/api/info): yt-dlp runs at most this many at once, others wait in a short queue;
+ // answers are kept per video id for a while.
+ info: {
+ maxConcurrent: parseInt(process.env.YT_INFO_CONCURRENCY, 10) || 3,
+ maxQueued: 20,
+ cacheTtlMs: 10 * 60 * 1000,
+ cacheMax: 500,
+ },
+
// Ephemeral file retention (ms)
// Ephemeral file retention (ms)
retention: {
retention: {
fileTTL: 60 * 60 * 1000, // 1 hour
fileTTL: 60 * 60 * 1000, // 1 hour
@@ -70,6 +70,76 @@ function sanitizeUrl(url) {
@@ -70,6 +70,76 @@ function sanitizeUrl(url) {
}
}
}
}
+/** The YouTube video id of a link (watch?v=, youtu.be/, /shorts/, /live/, /embed/), or null. */
+function videoId(url) {
+ try {
+ const u = new URL(url);
+ let id = u.hostname === 'youtu.be' ? u.pathname.slice(1).split('/')[0] : u.searchParams.get('v');
+ if (!id) { const m = /^\/(?:shorts|live|embed|v)\/([^/?#]+)/.exec(u.pathname); if (m) id = m[1]; }
+ return id && /^[\w-]{6,20}$/.test(id) ? id : null;
+ } catch { return null; }
+}
+
+const sizeLabel = () => (config.download.maxFilesize >= 1024 ? `${Math.round(config.download.maxFilesize / 102.4) / 10} GB` : `${config.download.maxFilesize} MB`);
+const fmtHours = (s) => (s % 3600 === 0 ? `${s / 3600} hour${s === 3600 ? '' : 's'}` : `${Math.round(s / 60)} minutes`);
+
+/** Why this video cannot be downloaded here (null when it can). */
+function limitReason(duration, isLive) {
+ const max = config.download.maxDuration;
+ if (isLive) return 'This is a live stream that has not ended; it can be saved once it is over.';
+ if (Number.isFinite(duration) && duration > max) return `This video is ${fmtHours(Math.round(duration))} long; downloads are limited to ${fmtHours(max)}.`;
+ return null;
+}
+
+// ── Info: at most N yt-dlp runs at once, answers cached per video id ─────
+const infoCache = new Map(); // id → { info, expires }
+const infoInflight = new Map(); // id → Promise
+let infoRunning = 0;
+const infoQueue = [];
+
+function acquireInfoSlot() {
+ if (infoRunning < config.info.maxConcurrent) { infoRunning++; return Promise.resolve(); }
+ if (infoQueue.length >= config.info.maxQueued) return Promise.reject(Object.assign(new Error('The server is busy looking up other videos. Try again in a moment.'), { status: 503 }));
+ return new Promise(resolve => infoQueue.push(resolve));
+}
+function releaseInfoSlot() {
+ const next = infoQueue.shift();
+ if (next) next(); else infoRunning = Math.max(0, infoRunning - 1);
+}
+
+function cachedInfo(id) {
+ const hit = id && infoCache.get(id);
+ if (!hit) return null;
+ if (hit.expires <= Date.now()) { infoCache.delete(id); return null; }
+ return hit.info;
+}
+
+/** getInfo with the concurrency cap, the per-id cache and one run per id at a time. */
+function getInfoLimited(url, { run = getInfo } = {}) {
+ const id = videoId(url);
+ const hit = cachedInfo(id);
+ if (hit) return Promise.resolve(hit);
+ if (id && infoInflight.has(id)) return infoInflight.get(id);
+ const p = (async () => {
+ await acquireInfoSlot();
+ try {
+ const info = await run(url);
+ if (id) {
+ infoCache.set(id, { info, expires: Date.now() + config.info.cacheTtlMs });
+ while (infoCache.size > config.info.cacheMax) infoCache.delete(infoCache.keys().next().value);
+ }
+ return info;
+ } finally {
+ releaseInfoSlot();
+ if (id) infoInflight.delete(id);
+ }
+ })();
+ if (id) infoInflight.set(id, p);
+ return p;
+}
+
+function infoStats() { return { running: infoRunning, queued: infoQueue.length, cached: infoCache.size }; }
+
// ── Quality Presets ──────────────────────────────────────────
// ── Quality Presets ──────────────────────────────────────────
const QUALITY_PRESETS = {
const QUALITY_PRESETS = {
'best': { video: 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best', ext: 'mp4' },
'best': { video: 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best', ext: 'mp4' },
@@ -132,7 +202,11 @@ function getInfo(url) {
@@ -132,7 +202,11 @@ function getInfo(url) {
uploaderUrl: info.uploader_url,
uploaderUrl: info.uploader_url,
viewCount: info.view_count,
viewCount: info.view_count,
uploadDate: info.upload_date,
uploadDate: info.upload_date,
+ isLive: !!info.is_live,
formats: getAvailableFormats(info),
formats: getAvailableFormats(info),
+ downloadable: !limitReason(info.duration, info.is_live),
+ ...(limitReason(info.duration, info.is_live) && { reason: limitReason(info.duration, info.is_live) }),
+ limits: { maxDuration: config.download.maxDuration, maxFilesizeMB: config.download.maxFilesize },
});
});
} catch (e) {
} catch (e) {
reject(new Error('Failed to parse video info'));
reject(new Error('Failed to parse video info'));
@@ -233,8 +307,23 @@ const cleanField = (v) => {
@@ -233,8 +307,23 @@ const cleanField = (v) => {
* knows them (85/15 otherwise) and the overall figure never decreases.
* knows them (85/15 otherwise) and the overall figure never decreases.
*/
*/
function applyLine(entry, line) {
function applyLine(entry, line) {
+ if (line.startsWith('OVD|')) {
+ // pre_process: the video's length and live state, before the match filter and any download.
+ const [, dur, live] = line.split('|');
+ entry.probed = true;
+ const d = parseFloat(dur);
+ if (Number.isFinite(d)) entry.duration = d;
+ if (live === 'True') entry.isLive = true;
+ const why = limitReason(entry.duration, entry.isLive);
+ if (why && !entry.limitError) entry.limitError = why;
+ return;
+ }
if (line.startsWith('OVP|')) {
if (line.startsWith('OVP|')) {
- const [, pctRaw, speed, eta] = line.split('|');
+ const [, pctRaw, speed, eta, bytesRaw] = line.split('|');
+ const bytes = parseFloat(bytesRaw);
+ if (Number.isFinite(bytes) && bytes > config.download.maxFilesize * 1024 * 1024 && !entry.limitError) {
+ entry.limitError = `This download is larger than ${sizeLabel()}, the limit here.`;
+ }
const pct = parseFloat(pctRaw);
const pct = parseFloat(pctRaw);
if (!Number.isFinite(pct)) return;
if (!Number.isFinite(pct)) return;
if (entry._lastPct != null && pct < entry._lastPct - 40 && entry._part < entry._weights.length - 1) entry._part++;
if (entry._lastPct != null && pct < entry._lastPct - 40 && entry._part < entry._weights.length - 1) entry._part++;
@@ -325,9 +414,15 @@ function startDownload(url, quality = 'best', opts = {}) {
@@ -325,9 +414,15 @@ function startDownload(url, quality = 'best', opts = {}) {
'--no-simulate',
'--no-simulate',
'--progress',
'--progress',
'--newline', // progress on new lines for parsing
'--newline', // progress on new lines for parsing
+ // Limits: a video longer than maxDuration (or a live stream without a length) does not
+ // pass the filter, a part larger than maxFilesize is not downloaded; OVD reports the
+ // length before either happens so the error can say why.
+ '--match-filter', `duration<=${config.download.maxDuration} & !is_live`,
+ '--max-filesize', `${config.download.maxFilesize}M`,
+ '--print', 'pre_process:OVD|%(duration)s|%(is_live)s',
'--print', 'before_dl:OVM|%(requested_formats.:.filesize)j|%(requested_formats.:.filesize_approx)j|%(title)j',
'--print', 'before_dl:OVM|%(requested_formats.:.filesize)j|%(requested_formats.:.filesize_approx)j|%(title)j',
'--print', 'post_process:OVX|pp',
'--print', 'post_process:OVX|pp',
- '--progress-template', 'download:OVP|%(progress._percent_str)s|%(progress._speed_str)s|%(progress._eta_str)s',
+ '--progress-template', 'download:OVP|%(progress._percent_str)s|%(progress._speed_str)s|%(progress._eta_str)s|%(progress.downloaded_bytes)s',
...(preset.postprocess || []),
...(preset.postprocess || []),
cleanUrl,
cleanUrl,
];
];
@@ -362,6 +457,14 @@ function startDownload(url, quality = 'best', opts = {}) {
@@ -362,6 +457,14 @@ function startDownload(url, quality = 'best', opts = {}) {
const lines = (stdoutRest + data.toString()).split('\n');
const lines = (stdoutRest + data.toString()).split('\n');
stdoutRest = lines.pop();
stdoutRest = lines.pop();
for (const line of lines) applyLine(entry, line.trim());
for (const line of lines) applyLine(entry, line.trim());
+ // Over a limit: stop now rather than after the whole file arrived.
+ if (entry.limitError && entry.status === 'downloading') {
+ entry.status = 'error';
+ entry.error = entry.limitError;
+ entry.finishedAt = Date.now();
+ try { proc.kill('SIGTERM'); } catch { /* gone */ }
+ setTimeout(() => { try { proc.kill('SIGKILL'); } catch { /* gone */ } }, 3000).unref();
+ }
});
});
let stderrTail = '';
let stderrTail = '';
@@ -391,8 +494,12 @@ function startDownload(url, quality = 'best', opts = {}) {
@@ -391,8 +494,12 @@ function startDownload(url, quality = 'best', opts = {}) {
const dir = path.resolve(config.downloadsDir);
const dir = path.resolve(config.downloadsDir);
const files = fs.readdirSync(dir).filter(f => f.startsWith(id) && !/\.(part|ytdl|temp)$/i.test(f));
const files = fs.readdirSync(dir).filter(f => f.startsWith(id) && !/\.(part|ytdl|temp)$/i.test(f));
if (files.length === 0) {
if (files.length === 0) {
+ // yt-dlp exits 0 when --match-filter or --max-filesize skipped the video.
entry.status = 'error';
entry.status = 'error';
- entry.error = 'Output file not found';
+ entry.error = entry.limitError
+ || (entry.probed && !Number.isFinite(entry.duration) ? 'This video has no known length (a live or upcoming stream); it can be saved once it has ended.' : null)
+ || `This video is over the limits here: at most ${fmtHours(config.download.maxDuration)} long and ${sizeLabel()} per file.`;
+ removePartials(id);
return;
return;
}
}
@@ -609,7 +716,7 @@ function getUpstream() { return upstream; }
@@ -609,7 +716,7 @@ function getUpstream() { return upstream; }
function noteUpstream(state) { if (state === 'ok' || state === 'blocked') upstream = { state, checkedAt: Date.now(), detail: state === 'blocked' ? 'YouTube is refusing requests from this server address' : '' }; }
function noteUpstream(state) { if (state === 'ok' || state === 'blocked') upstream = { state, checkedAt: Date.now(), detail: state === 'blocked' ? 'YouTube is refusing requests from this server address' : '' }; }
module.exports = {
module.exports = {
- getInfo, startDownload, cancelDownload, getStatus, getFile, removeFile,
+ getInfo, getInfoLimited, cachedInfo, videoId, limitReason, infoStats, startDownload, cancelDownload, getStatus, getFile, removeFile,
sanitizeTitle, downloadFilename, contentDisposition, applyLine,
sanitizeTitle, downloadFilename, contentDisposition, applyLine,
cleanup, startCleanup, stopCleanup, getStats, isValidUrl, probeUpstream, startUpstreamProbe, getUpstream,
cleanup, startCleanup, stopCleanup, getStats, isValidUrl, probeUpstream, startUpstreamProbe, getUpstream,
};
};
@@ -120,7 +120,7 @@ app.use(hostGuard({ knows: seo.knowsHost, aliasOf: seo.aliasOf }));
@@ -120,7 +120,7 @@ app.use(hostGuard({ knows: seo.knowsHost, aliasOf: seo.aliasOf }));
app.get('/api/health', (_req, res) => {
app.get('/api/health', (_req, res) => {
const stats = downloader.getStats();
const stats = downloader.getStats();
const up = downloader.getUpstream();
const up = downloader.getUpstream();
- res.set('Cache-Control', 'no-store').json({ status: 'ok', service: 'openvibe-yt', version: '1.0.0', stats, youtube: up.state, youtubeCheckedAt: up.checkedAt ? new Date(up.checkedAt).toISOString() : null });
+ res.set('Cache-Control', 'no-store').json({ status: 'ok', service: 'openvibe-yt', version: '1.0.0', stats, info: downloader.infoStats(), limits: { maxDuration: config.download.maxDuration, maxFilesizeMB: config.download.maxFilesize }, youtube: up.state, youtubeCheckedAt: up.checkedAt ? new Date(up.checkedAt).toISOString() : null });
});
});
// Get video info
// Get video info
@@ -130,9 +130,12 @@ app.post('/api/info', async (req, res) => {
@@ -130,9 +130,12 @@ app.post('/api/info', async (req, res) => {
if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported (youtube.com, youtu.be)' });
if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported (youtube.com, youtu.be)' });
try {
try {
- const info = await downloader.getInfo(url);
+ // At most config.info.maxConcurrent yt-dlp lookups at once; the same video is looked up once
+ // per ten minutes however many people paste it.
+ const info = await downloader.getInfoLimited(url);
res.json({ success: true, video: info });
res.json({ success: true, video: info });
} catch (err) {
} catch (err) {
+ if (err.status === 503) return res.status(503).set('Retry-After', '5').json({ error: err.message });
console.error('[Info] Error:', err.message);
console.error('[Info] Error:', err.message);
res.status(422).json({ error: err.message });
res.status(422).json({ error: err.message });
}
}
@@ -143,6 +146,9 @@ app.post('/api/download', downloadLimiter, async (req, res) => {
@@ -143,6 +146,9 @@ app.post('/api/download', downloadLimiter, async (req, res) => {
const { url, quality, title } = req.body;
const { url, quality, title } = req.body;
if (!url) return res.status(400).json({ error: 'URL is required' });
if (!url) return res.status(400).json({ error: 'URL is required' });
if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported' });
if (!downloader.isValidUrl(url)) return res.status(400).json({ error: 'Only YouTube URLs are supported' });
+ // The length is known from the info lookup the page made first: refuse before yt-dlp starts.
+ const known = downloader.cachedInfo(downloader.videoId(url));
+ if (known && known.downloadable === false) return res.status(422).json({ error: known.reason });
try {
try {
const { id } = await downloader.startDownload(url, quality || 'best', { title: typeof title === 'string' ? title : '' });
const { id } = await downloader.startDownload(url, quality || 'best', { title: typeof title === 'string' ? title : '' });
new file mode 100644
new file mode 100644
@@ -0,0 +1,107 @@
@@ -0,0 +1,107 @@
+'use strict';
+// YT limits: maxDuration and the file size cap are enforced (they were configured but never checked),
+// and /api/info runs at most N yt-dlp lookups at once with a per-video cache. yt-dlp is a stand-in
+// script here: nothing reaches YouTube.
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+process.env.YT_MAX_DURATION = '600'; // 10 minutes, before config is loaded
+process.env.YT_MAX_FILESIZE_MB = '1';
+process.env.YT_INFO_CONCURRENCY = '3';
+const downloader = require('../server/downloader');
+const { startApp } = require('../../_shared/test/spawn');
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+(async () => {
+ // ── Video ids ──
+ const id = downloader.videoId;
+ assert.strictEqual(id('https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=10'), 'dQw4w9WgXcQ');
+ assert.strictEqual(id('https://youtu.be/dQw4w9WgXcQ?si=x'), 'dQw4w9WgXcQ');
+ assert.strictEqual(id('https://www.youtube.com/shorts/abcDEF12345'), 'abcDEF12345');
+ assert.strictEqual(id('https://www.youtube.com/'), null);
+
+ // ── Progress lines: over the length or the size limit sets the error before the file is done ──
+ let e = { _weights: [1], _part: 0, progress: null };
+ downloader.applyLine(e, 'OVD|601|False');
+ assert.match(e.limitError, /10 minutes long; downloads are limited to 10 minutes/);
+ e = { _weights: [1], _part: 0, progress: null };
+ downloader.applyLine(e, 'OVD|NA|True');
+ assert.match(e.limitError, /live stream/);
+ e = { _weights: [1], _part: 0, progress: null };
+ downloader.applyLine(e, 'OVD|120|False');
+ assert.strictEqual(e.limitError, undefined);
+ downloader.applyLine(e, `OVP| 50.0%|1MiB/s|00:01|${2 * 1024 * 1024}`);
+ assert.match(e.limitError, /larger than 1 MB/);
+ e = { _weights: [1], _part: 0, progress: null };
+ downloader.applyLine(e, 'OVP| 50.0%|1MiB/s|00:01'); // the old four-field line still parses
+ assert.strictEqual(e.progress, 50);
+
+ // ── Info: at most 3 yt-dlp runs at once, one per video id, cached ──
+ let running = 0, peak = 0, runs = 0;
+ const run = async (url) => { runs++; running++; peak = Math.max(peak, running); await sleep(30); running--; return { id: downloader.videoId(url), duration: 60, downloadable: true }; };
+ const urls = Array.from({ length: 8 }, (_, i) => `https://www.youtube.com/watch?v=video${String(i).padStart(6, '0')}`);
+ await Promise.all([...urls, ...urls].map(u => downloader.getInfoLimited(u, { run })));
+ assert.strictEqual(peak, 3, 'never more than three lookups at once');
+ assert.strictEqual(runs, 8, 'the same video is looked up once even when asked twice at the same time');
+ await downloader.getInfoLimited(urls[0], { run });
+ assert.strictEqual(runs, 8, 'cached');
+ // A full queue answers 503 instead of piling up processes.
+ const slow = () => new Promise(r => setTimeout(() => r({ duration: 1 }), 200));
+ const flood = Array.from({ length: 30 }, (_, i) => downloader.getInfoLimited(`https://youtu.be/flood${String(i).padStart(6, '0')}`, { run: slow }).then(() => 'ok', err => err.status));
+ const outcomes = await Promise.all(flood);
+ assert.strictEqual(outcomes.filter(o => o === 'ok').length, 23, '3 running + 20 queued');
+ assert.strictEqual(outcomes.filter(o => o === 503).length, 7);
+
+ // ── As a real process with a stand-in yt-dlp ──
+ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-yt-'));
+ const fake = path.join(data, 'yt-dlp');
+ fs.writeFileSync(fake, `#!${process.execPath}
+const args = process.argv.slice(2);
+const url = args[args.length - 1];
+const long = url.includes('LONGLONG123');
+if (args.includes('--dump-json')) {
+ process.stdout.write(JSON.stringify({ id: long ? 'LONGLONG123' : 'SHORTxx1234', title: 't', duration: long ? 4000 : 30, formats: [{ vcodec: 'avc1', acodec: 'mp4a' }] }));
+ process.exit(0);
+}
+const i = args.indexOf('--match-filter');
+require('fs').writeFileSync(${JSON.stringify(path.join(data, 'args.json'))}, JSON.stringify(args));
+process.stdout.write('OVD|' + (long ? 4000 : 30) + '|False\\n');
+process.exit(0); // the match filter skipped it: no file
+`, { mode: 0o755 });
+ const app = await startApp('yt', { DATA_DIR: data, DOWNLOADS_DIR: path.join(data, 'downloads'), YTDLP_PATH: fake, YT_MAX_DURATION: '600', YT_MAX_FILESIZE_MB: '100' });
+ try {
+ const post = (p, body) => fetch(`${app.base}${p}`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
+ let r = await post('/api/info', { url: 'https://www.youtube.com/watch?v=LONGLONG123' });
+ let body = await r.json();
+ assert.strictEqual(r.status, 200);
+ assert.strictEqual(body.video.downloadable, false);
+ assert.match(body.video.reason, /67 minutes long; downloads are limited to 10 minutes/);
+ assert.deepStrictEqual(body.video.limits, { maxDuration: 600, maxFilesizeMB: 100 });
+ r = await post('/api/download', { url: 'https://youtu.be/LONGLONG123', quality: 'best' });
+ assert.strictEqual(r.status, 422, 'refused from the cached info, before yt-dlp starts');
+ assert.match((await r.json()).error, /limited to 10 minutes/);
+ assert.ok(!fs.existsSync(path.join(data, 'args.json')), 'no download was started');
+
+ // Without cached info, yt-dlp gets the filter and the size cap, and the skip is explained.
+ r = await post('/api/download', { url: 'https://www.youtube.com/watch?v=SHORTxx1234', quality: 'best' });
+ body = await r.json();
+ assert.strictEqual(r.status, 200, JSON.stringify(body));
+ let st;
+ for (let i = 0; i < 50; i++) { st = await (await fetch(`${app.base}/api/status/${body.id}`)).json(); if (st.status !== 'downloading') break; await sleep(50); }
+ const args = JSON.parse(fs.readFileSync(path.join(data, 'args.json'), 'utf8'));
+ assert.strictEqual(args[args.indexOf('--match-filter') + 1], 'duration<=600 & !is_live');
+ assert.strictEqual(args[args.indexOf('--max-filesize') + 1], '100M');
+ assert.ok(args.includes('pre_process:OVD|%(duration)s|%(is_live)s'));
+ assert.strictEqual(st.status, 'error');
+ assert.match(st.error, /over the limits here: at most 10 minutes long and 100 MB per file/);
+ const health = await (await fetch(`${app.base}/api/health`)).json();
+ assert.deepStrictEqual(health.limits, { maxDuration: 600, maxFilesizeMB: 100 });
+ } finally {
+ await app.kill();
+ fs.rmSync(data, { recursive: true, force: true });
+ }
+ console.log('yt limits (duration, file size, info concurrency + cache): all checks passed');
+})().catch((err) => { console.error(err); process.exit(1); });