openvibers/openvibe.tools Commits
One upload runtime (roadmap WS-L task 3): apps/_shared/upload.js createUploads({ multer, config, storage, maxFiles, rejectHint }) replaces img's, audio's and docs' own multer middleware (memory for img, disk for audio and docs, 5 and 50 files); disk names are random with a sanitised extension everywhere (audio kept any extension before); the per-app copies are deleted and a test keeps them from coming back
Changes
nnew file mode 100644@@@ -0,0 +1,77 @@++'use strict';++/**++ * The shared upload factory (roadmap WS-L task 3): memory and disk storage, the declared-type filter++ * (octet-stream let through for the guard's byte check), size and count limits with the { error }++ * answers the apps' pages read, sanitised disk names; and no satellite keeps its own copy.++ */++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const http = require('http');++const { dep } = require('./deps');++const { createUploads } = require('../upload');++++const express = dep('express');++const multer = dep('multer');++++const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-uploads-'));++const config = { upload: { maxFileSize: 1024, allowedMimes: ['image/png', 'text/plain'] }, uploadsDir: dir };++++async function serve(app) {++ const server = await new Promise((r) => { const s = http.createServer(app).listen(0, '127.0.0.1', () => r(s)); });++ return { base: `http://127.0.0.1:${server.address().port}`, close: () => server.close() };++}++const form = (fields) => { const f = new FormData(); for (const [name, body, type, file] of fields) f.append(name, new Blob([body], { type }), file); return f; };++++(async () => {++ assert.throws(() => createUploads({ config }), /needs multer/);++ assert.throws(() => createUploads({ multer }), /needs the app config/);++++ const mem = createUploads({ multer, config, storage: 'memory' });++ const disk = createUploads({ multer, config, storage: 'disk', maxFiles: 2, rejectHint: 'Upload a PNG.' });++ const app = express();++ app.post('/mem', mem.uploadSingle, (req, res) => res.json({ size: req.file.size, buffer: Buffer.isBuffer(req.file.buffer) }));++ app.post('/disk', disk.uploadSingle, (req, res) => res.json({ name: path.basename(req.file.path), inDir: path.dirname(req.file.path) === path.resolve(dir) }));++ app.post('/many', disk.uploadMultiple, (req, res) => res.json({ n: req.files.length }));++ app.post('/any', disk.uploadAny, (req, res) => res.json({ n: Object.values(req.files).flat().length }));++ const s = await serve(app);++ const post = async (p, f) => { const r = await fetch(s.base + p, { method: 'POST', body: f }); return { status: r.status, body: await r.json() }; };++ try {++ let r = await post('/mem', form([['file', 'hello', 'text/plain', 'a.txt']]));++ assert.deepStrictEqual([r.status, r.body], [200, { size: 5, buffer: true }], 'memory storage keeps the bytes in req.file.buffer');++ r = await post('/mem', form([['file', 'x', 'application/octet-stream', 'x.bin']]));++ assert.strictEqual(r.status, 200, 'octet-stream is let through for the guard to sniff');++ r = await post('/mem', form([['file', 'x', 'application/zip', 'x.zip']]));++ assert.deepStrictEqual([r.status, r.body.error], [400, 'Unsupported file type: application/zip. Accepted: image/png, text/plain']);++ r = await post('/mem', form([['file', 'x'.repeat(2048), 'text/plain', 'big.txt']]));++ assert.deepStrictEqual([r.status, r.body.error], [413, 'File too large. Maximum 0MB.']);++ r = await post('/mem', form([['other', 'x', 'text/plain', 'a.txt']]));++ assert.strictEqual(r.status, 400);++++ r = await post('/disk', form([['file', 'hi', 'text/plain', '../../evil name.TXT;rm']]));++ assert.strictEqual(r.status, 200);++ assert.match(r.body.name, /^[0-9a-f]{32}\.bin$/, 'a disk name is random, and a strange extension becomes .bin');++ assert.strictEqual(r.body.inDir, true, 'it lands in config.uploadsDir');++ r = await post('/disk', form([['file', 'hi', 'image/png', 'Pic.PNG']]));++ assert.match(r.body.name, /^[0-9a-f]{32}\.png$/);++ r = await post('/disk', form([['file', 'x', 'application/zip', 'x.zip']]));++ assert.strictEqual(r.body.error, 'Unsupported file type: application/zip. Upload a PNG.', 'the app\'s own hint');++++ r = await post('/many', form([['files', 'a', 'text/plain', 'a.txt'], ['files', 'b', 'text/plain', 'b.txt']]));++ assert.deepStrictEqual([r.status, r.body.n], [200, 2]);++ r = await post('/many', form([['files', 'a', 'text/plain', 'a.txt'], ['files', 'b', 'text/plain', 'b.txt'], ['files', 'c', 'text/plain', 'c.txt']]));++ assert.deepStrictEqual([r.status, r.body.error], [400, 'Too many files. Maximum 2 files per request.']);++ r = await post('/any', form([['file', 'a', 'text/plain', 'a.txt']]));++ assert.deepStrictEqual([r.status, r.body.n], [200, 1]);++ } finally { s.close(); }++++ // One runtime: no satellite keeps its own upload middleware.++ const apps = path.join(__dirname, '..', '..');++ for (const a of fs.readdirSync(apps)) {++ if (a.startsWith('_') || !fs.statSync(path.join(apps, a)).isDirectory()) continue;++ assert.ok(!fs.existsSync(path.join(apps, a, 'server', 'middleware', 'upload.js')), `${a} has its own upload middleware again`);++ }++ fs.rmSync(dir, { recursive: true, force: true });++ console.log('upload (shared runtime): all checks passed');++})().catch((e) => { console.error(e); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,93 @@++'use strict';++// ═══════════════════════════════════════════════════════════════++// Uploads for the satellites (roadmap WS-L task 3: one runtime, no per-app copies).++//++// const { uploadSingle, uploadMultiple, uploadAny } = createUploads({ multer: require('multer'), config, storage: 'disk', maxFiles: 50 });++//++// multer the app's own (apps/_shared has no node_modules; its modules take packages as arguments)++//++// storage 'memory' req.file.buffer: small files only (images)++// storage 'disk' files land in config.uploadsDir with a random name and a sanitised extension, so++// large or many files never sit in memory while they arrive; the routes read and++// delete them, the retention sweep removes leftovers, job submits move them++// maxFiles per request, for uploadMultiple / uploadAny ('files' field)++// rejectHint the end of the unsupported-type message (default: the accepted types)++//++// The declared type must be one of config.upload.allowedMimes, or application/octet-stream (API++// clients, openvibe-sdk): the guard checks the bytes of every upload against the tool's accepted++// types (guard/sniff.js). Answers are the { error } JSON the apps' pages already read.++// ═══════════════════════════════════════════════════════════════++++const path = require('path');++const fs = require('fs');++const crypto = require('crypto');++++function createUploads({ multer, config, storage = 'memory', maxFiles = 1, rejectHint = null } = {}) {++ if (typeof multer !== 'function') throw new TypeError('createUploads needs multer (the app\'s require(\'multer\'))');++ if (!config || !config.upload) throw new TypeError('createUploads needs the app config (upload.maxFileSize, upload.allowedMimes)');++ const allowed = new Set(config.upload.allowedMimes || []);++ const maxMb = Math.round(config.upload.maxFileSize / 1024 / 1024);++ let store;++ if (storage === 'disk') {++ const dir = path.resolve(config.uploadsDir || 'data/uploads');++ fs.mkdirSync(dir, { recursive: true });++ store = multer.diskStorage({++ destination: (_req, _file, cb) => cb(null, dir),++ filename: (_req, file, cb) => {++ const ext = (path.extname(file.originalname || '').toLowerCase().match(/^\.[a-z0-9]{1,8}$/) || ['.bin'])[0];++ cb(null, `${crypto.randomBytes(16).toString('hex')}${ext}`);++ },++ });++ } else {++ store = multer.memoryStorage();++ }++ const upload = multer({++ storage: store,++ limits: { fileSize: config.upload.maxFileSize, files: Math.max(1, maxFiles) },++ fileFilter(_req, file, cb) {++ if (allowed.has(file.mimetype) || file.mimetype === 'application/octet-stream') return cb(null, true);++ cb(new Error(`Unsupported file type: ${file.mimetype}. ${rejectHint || `Accepted: ${[...allowed].join(', ')}`}`));++ },++ });++ const tooLarge = (per) => `File too large. Maximum ${maxMb}MB${per ? ' per file' : ''}.`;++++ /** One file in 'file' → req.file ({ buffer } in memory, { path } on disk, plus originalname, mimetype, size). */++ function uploadSingle(req, res, next) {++ upload.single('file')(req, res, (err) => {++ if (err) return res.status(err.code === 'LIMIT_FILE_SIZE' ? 413 : 400).json({ error: err.code === 'LIMIT_FILE_SIZE' ? tooLarge(false) : err.message });++ if (!req.file) return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });++ next();++ });++ }++++ /** Up to maxFiles in 'files' → req.files (an array). */++ function uploadMultiple(req, res, next) {++ upload.array('files', maxFiles)(req, res, (err) => {++ if (err) {++ if (err.code === 'LIMIT_FILE_SIZE') return res.status(413).json({ error: tooLarge(true) });++ if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') return res.status(400).json({ error: `Too many files. Maximum ${maxFiles} files per request.` });++ return res.status(400).json({ error: err.message });++ }++ if (!req.files || req.files.length === 0) return res.status(400).json({ error: 'No files uploaded. Send files in the "files" field.' });++ next();++ });++ }++++ /** One file in 'file' or up to maxFiles in 'files' (the job endpoint takes either) → req.files { file?, files? }. */++ function uploadAny(req, res, next) {++ upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: maxFiles }])(req, res, (err) => {++ if (err) {++ if (err.code === 'LIMIT_FILE_SIZE') return res.status(413).json({ error: tooLarge(true) });++ if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') return res.status(400).json({ error: `Send one file in "file" or up to ${maxFiles} in "files".` });++ return res.status(400).json({ error: err.message });++ }++ const n = req.files ? Object.values(req.files).flat().length : 0;++ if (!n) return res.status(400).json({ error: 'No files uploaded. Send a file in the "file" field.' });++ next();++ });++ }++++ return { uploadSingle, uploadMultiple, uploadAny };++}++++module.exports = { createUploads };@@@ -18,7 +18,8 @@ const config = require('./config');const { resolveContext, DOMAIN_MAP } = require('./domain-map');const { getTool, listTools } = require('./tools');const { readMetadata } = require('./tools/metadata');--const { uploadSingle, uploadMultiple, uploadAny } = require('./middleware/upload');++// Uploads from the shared runtime (apps/_shared/upload.js, WS-L task 3).++const { uploadSingle, uploadMultiple, uploadAny } = require('../../_shared/upload').createUploads({ multer: require('multer'), config, storage: 'disk', maxFiles: 5, rejectHint: 'Upload an audio or video file.' });const retention = require('./retention/manager');const { probe, getDuration, cleanTmp } = require('./tools/ffmpeg-helper');const { buildOptions, describe, defineJobs, runSync, jobContext, limitsFor } = require('./process');ddeleted file mode 100644@@@ -1,102 +0,0 @@--'use strict';----// ═══════════════════════════════════════════════════════════════--// Audio.OpenVibe — Multer Upload Middleware--// Disk storage (audio files can be large) with size + mime validation.--// ═══════════════════════════════════════════════════════════════----const multer = require('multer');--const path = require('path');--const fs = require('fs');--const crypto = require('crypto');--const config = require('../config');----const ALLOWED_MIMES = new Set(config.upload.allowedMimes);----// Use disk storage for audio — memory storage is risky for 100MB files--const uploadsDir = path.resolve(config.uploadsDir);--if (!fs.existsSync(uploadsDir)) fs.mkdirSync(uploadsDir, { recursive: true });----const storage = multer.diskStorage({-- destination: (_req, _file, cb) => cb(null, uploadsDir),-- filename: (_req, file, cb) => {-- const id = crypto.randomBytes(16).toString('hex');-- const ext = path.extname(file.originalname) || '.bin';-- cb(null, `${id}${ext}`);-- },--});----const upload = multer({-- storage,-- limits: {-- fileSize: config.upload.maxFileSize,-- files: 5, // Allow up to 5 for merge tool-- },-- fileFilter(_req, file, cb) {-- // Be lenient — some browsers send application/octet-stream for audio-- if (ALLOWED_MIMES.has(file.mimetype) || file.mimetype === 'application/octet-stream') {-- cb(null, true);-- } else {-- cb(new Error(`Unsupported file type: ${file.mimetype}. Upload an audio or video file.`));-- }-- },--});----/**-- * Single file upload on 'file' field.-- */--function uploadSingle(req, res, next) {-- upload.single('file')(req, res, (err) => {-- if (err) {-- if (err.code === 'LIMIT_FILE_SIZE') {-- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB.` });-- }-- return res.status(400).json({ error: err.message });-- }-- if (!req.file) {-- return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });-- }-- next();-- });--}----/**-- * Multiple file upload on 'files' field (for merge).-- */--function uploadMultiple(req, res, next) {-- upload.array('files', 5)(req, res, (err) => {-- if (err) {-- if (err.code === 'LIMIT_FILE_SIZE') {-- return res.status(413).json({ error: `Files too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB each.` });-- }-- return res.status(400).json({ error: err.message });-- }-- if (!req.files || req.files.length < 2) {-- return res.status(400).json({ error: 'Upload at least 2 files to merge.' });-- }-- next();-- });--}----/**-- * One file in 'file' or up to 5 in 'files' (the job endpoint takes either shape; merge sends 'files').-- * Attaches req.files as { file?: [...], files?: [...] }.-- */--function uploadAny(req, res, next) {-- upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: 5 }])(req, res, (err) => {-- if (err) {-- if (err.code === 'LIMIT_FILE_SIZE') {-- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });-- }-- if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') {-- return res.status(400).json({ error: 'Send one file in "file" or up to 5 in "files".' });-- }-- return res.status(400).json({ error: err.message });-- }-- const n = req.files ? Object.values(req.files).flat().length : 0;-- if (!n) return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });-- next();-- });--}----module.exports = { uploadSingle, uploadMultiple, uploadAny };@@@ -12,7 +12,7 @@ const { tmpFile, probe, getAudioStream, getDuration, getMime } = require('./ffmpconst { FORMAT_CONFIG } = require('./convert');const MIN_FILES = 2;--const MAX_FILES = 5; // the upload limit (middleware/upload.js)++const MAX_FILES = 5; // the upload limit (createUploads maxFiles in server/index.js)const MAX_TOTAL_SECONDS = 3 * 60 * 60; // three hours of joined audioconst LOSSY = new Set(['mp3', 'ogg', 'm4a', 'aac', 'opus', 'wma', 'ac3', 'webm']);@@@ -18,7 +18,8 @@ const rateLimit = require('express-rate-limit');const config = require('./config');const { resolveContext, DOMAIN_MAP } = require('./domain-map');const { getTool, listTools, BINARIES } = require('./tools');--const { uploadSingle, uploadMultiple, uploadAny } = require('./middleware/upload');++// Uploads from the shared runtime (apps/_shared/upload.js, WS-L task 3).++const { uploadSingle, uploadMultiple, uploadAny } = require('../../_shared/upload').createUploads({ multer: require('multer'), config, storage: 'disk', maxFiles: 50 });const retention = require('./retention/manager');const { buildOptions, defineJobs, runTool } = require('./process');const { poolFromEnv } = require('../../_shared/jobs/pool');ddeleted file mode 100644@@@ -1,109 +0,0 @@--'use strict';----// ═══════════════════════════════════════════════════════════════--// Docs.OpenVibe — Multer Upload Middleware--// Disk storage (up to 50 files of 100 MB for a merge must never sit in memory while they arrive) with--// size + declared-type validation; the guard then checks the bytes (apps/_shared/guard/sniff.js).--// Files land in config.uploadsDir with a random name; the routes read them and delete them, and the--// retention sweep removes anything older than retention.tempMaxAge. Job submits move them into the job.--// ═══════════════════════════════════════════════════════════════----const multer = require('multer');--const path = require('path');--const fs = require('fs');--const crypto = require('crypto');--const config = require('../config');----const ALLOWED_MIMES = new Set(config.upload.allowedMimes);----const uploadsDir = path.resolve(config.uploadsDir);--fs.mkdirSync(uploadsDir, { recursive: true });----const storage = multer.diskStorage({-- destination: (_req, _file, cb) => cb(null, uploadsDir),-- filename: (_req, file, cb) => {-- const ext = (path.extname(file.originalname || '').toLowerCase().match(/^\.[a-z0-9]{1,8}$/) || ['.bin'])[0];-- cb(null, `${crypto.randomBytes(16).toString('hex')}${ext}`);-- },--});----const upload = multer({-- storage,-- limits: {-- fileSize: config.upload.maxFileSize,-- files: 50, // Allow up to 50 files for merge-- },-- fileFilter(_req, file, cb) {-- // A part without a type of its own (application/octet-stream: API clients, openvibe-sdk) is let-- // through: the guard checks the bytes of every upload against the tool's accepted types.-- if (ALLOWED_MIMES.has(file.mimetype) || file.mimetype === 'application/octet-stream') {-- cb(null, true);-- } else {-- cb(new Error(`Unsupported file type: ${file.mimetype}. Accepted: ${config.upload.allowedMimes.join(', ')}`));-- }-- },--});----/**-- * Single file upload on 'file' field.-- * Attaches req.file with { path, originalname, mimetype, size }.-- */--function uploadSingle(req, res, next) {-- upload.single('file')(req, res, (err) => {-- if (err) {-- if (err.code === 'LIMIT_FILE_SIZE') {-- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB.` });-- }-- return res.status(400).json({ error: err.message });-- }-- if (!req.file) {-- return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });-- }-- next();-- });--}----/**-- * Multiple file upload on 'files' field (for merge, img2pdf, etc.).-- * Attaches req.files as an array of { path, originalname, mimetype, size }.-- */--function uploadMultiple(req, res, next) {-- upload.array('files', 50)(req, res, (err) => {-- if (err) {-- if (err.code === 'LIMIT_FILE_SIZE') {-- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });-- }-- if (err.code === 'LIMIT_FILE_COUNT') {-- return res.status(400).json({ error: 'Too many files. Maximum 50 files per request.' });-- }-- return res.status(400).json({ error: err.message });-- }-- if (!req.files || req.files.length === 0) {-- return res.status(400).json({ error: 'No files uploaded. Send files in the "files" field.' });-- }-- next();-- });--}----/**-- * One file in 'file' or several in 'files' (the job endpoint takes either shape).-- * Attaches req.files as { file?: [...], files?: [...] }.-- */--function uploadAny(req, res, next) {-- upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: 50 }])(req, res, (err) => {-- if (err) {-- if (err.code === 'LIMIT_FILE_SIZE') {-- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });-- }-- if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') {-- return res.status(400).json({ error: 'Send one file in "file" or up to 50 in "files".' });-- }-- return res.status(400).json({ error: err.message });-- }-- const n = req.files ? Object.values(req.files).flat().length : 0;-- if (!n) return res.status(400).json({ error: 'No files uploaded. Send a file in the "file" field.' });-- next();-- });--}----module.exports = { uploadSingle, uploadMultiple, uploadAny };@@@ -17,7 +17,8 @@ const rateLimit = require('express-rate-limit');const config = require('./config');const { resolveContext, DOMAIN_MAP } = require('./domain-map');const { getTool, listTools } = require('./tools');--const { uploadSingle } = require('./middleware/upload');++// Uploads from the shared runtime (apps/_shared/upload.js, WS-L task 3).++const { uploadSingle } = require('../../_shared/upload').createUploads({ multer: require('multer'), config, storage: 'memory', maxFiles: 1 });const retention = require('./retention/manager');const { buildOptions, describe, runProcess, defineJobs } = require('./process');const { poolFromEnv } = require('../../_shared/jobs/pool');ddeleted file mode 100644@@@ -1,51 +0,0 @@--'use strict';----// ═══════════════════════════════════════════════════════════════--// Img.OpenVibe — Multer Upload Middleware--// Memory storage with file size + mime validation.--// ═══════════════════════════════════════════════════════════════----const multer = require('multer');--const config = require('../config');----const ALLOWED_MIMES = new Set(config.upload.allowedMimes);----const storage = multer.memoryStorage();----const upload = multer({-- storage,-- limits: {-- fileSize: config.upload.maxFileSize,-- files: 1,-- },-- fileFilter(_req, file, cb) {-- // A part without a type of its own (application/octet-stream: API clients, openvibe-sdk) is let-- // through: the guard checks the bytes of every upload against the tool's accepted types.-- if (ALLOWED_MIMES.has(file.mimetype) || file.mimetype === 'application/octet-stream') {-- cb(null, true);-- } else {-- cb(new Error(`Unsupported file type: ${file.mimetype}. Accepted: ${config.upload.allowedMimes.join(', ')}`));-- }-- },--});----/**-- * Express middleware for single file upload on 'file' field.-- * Attaches req.file with { buffer, originalname, mimetype, size }.-- */--function uploadSingle(req, res, next) {-- upload.single('file')(req, res, (err) => {-- if (err) {-- if (err.code === 'LIMIT_FILE_SIZE') {-- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB.` });-- }-- return res.status(400).json({ error: err.message });-- }-- if (!req.file) {-- return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });-- }-- next();-- });--}----module.exports = { uploadSingle };
new file mode 100644
new file mode 100644
@@ -0,0 +1,77 @@
@@ -0,0 +1,77 @@
+'use strict';
+/**
+ * The shared upload factory (roadmap WS-L task 3): memory and disk storage, the declared-type filter
+ * (octet-stream let through for the guard's byte check), size and count limits with the { error }
+ * answers the apps' pages read, sanitised disk names; and no satellite keeps its own copy.
+ */
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const http = require('http');
+const { dep } = require('./deps');
+const { createUploads } = require('../upload');
+
+const express = dep('express');
+const multer = dep('multer');
+
+const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-uploads-'));
+const config = { upload: { maxFileSize: 1024, allowedMimes: ['image/png', 'text/plain'] }, uploadsDir: dir };
+
+async function serve(app) {
+ const server = await new Promise((r) => { const s = http.createServer(app).listen(0, '127.0.0.1', () => r(s)); });
+ return { base: `http://127.0.0.1:${server.address().port}`, close: () => server.close() };
+}
+const form = (fields) => { const f = new FormData(); for (const [name, body, type, file] of fields) f.append(name, new Blob([body], { type }), file); return f; };
+
+(async () => {
+ assert.throws(() => createUploads({ config }), /needs multer/);
+ assert.throws(() => createUploads({ multer }), /needs the app config/);
+
+ const mem = createUploads({ multer, config, storage: 'memory' });
+ const disk = createUploads({ multer, config, storage: 'disk', maxFiles: 2, rejectHint: 'Upload a PNG.' });
+ const app = express();
+ app.post('/mem', mem.uploadSingle, (req, res) => res.json({ size: req.file.size, buffer: Buffer.isBuffer(req.file.buffer) }));
+ app.post('/disk', disk.uploadSingle, (req, res) => res.json({ name: path.basename(req.file.path), inDir: path.dirname(req.file.path) === path.resolve(dir) }));
+ app.post('/many', disk.uploadMultiple, (req, res) => res.json({ n: req.files.length }));
+ app.post('/any', disk.uploadAny, (req, res) => res.json({ n: Object.values(req.files).flat().length }));
+ const s = await serve(app);
+ const post = async (p, f) => { const r = await fetch(s.base + p, { method: 'POST', body: f }); return { status: r.status, body: await r.json() }; };
+ try {
+ let r = await post('/mem', form([['file', 'hello', 'text/plain', 'a.txt']]));
+ assert.deepStrictEqual([r.status, r.body], [200, { size: 5, buffer: true }], 'memory storage keeps the bytes in req.file.buffer');
+ r = await post('/mem', form([['file', 'x', 'application/octet-stream', 'x.bin']]));
+ assert.strictEqual(r.status, 200, 'octet-stream is let through for the guard to sniff');
+ r = await post('/mem', form([['file', 'x', 'application/zip', 'x.zip']]));
+ assert.deepStrictEqual([r.status, r.body.error], [400, 'Unsupported file type: application/zip. Accepted: image/png, text/plain']);
+ r = await post('/mem', form([['file', 'x'.repeat(2048), 'text/plain', 'big.txt']]));
+ assert.deepStrictEqual([r.status, r.body.error], [413, 'File too large. Maximum 0MB.']);
+ r = await post('/mem', form([['other', 'x', 'text/plain', 'a.txt']]));
+ assert.strictEqual(r.status, 400);
+
+ r = await post('/disk', form([['file', 'hi', 'text/plain', '../../evil name.TXT;rm']]));
+ assert.strictEqual(r.status, 200);
+ assert.match(r.body.name, /^[0-9a-f]{32}\.bin$/, 'a disk name is random, and a strange extension becomes .bin');
+ assert.strictEqual(r.body.inDir, true, 'it lands in config.uploadsDir');
+ r = await post('/disk', form([['file', 'hi', 'image/png', 'Pic.PNG']]));
+ assert.match(r.body.name, /^[0-9a-f]{32}\.png$/);
+ r = await post('/disk', form([['file', 'x', 'application/zip', 'x.zip']]));
+ assert.strictEqual(r.body.error, 'Unsupported file type: application/zip. Upload a PNG.', 'the app\'s own hint');
+
+ r = await post('/many', form([['files', 'a', 'text/plain', 'a.txt'], ['files', 'b', 'text/plain', 'b.txt']]));
+ assert.deepStrictEqual([r.status, r.body.n], [200, 2]);
+ r = await post('/many', form([['files', 'a', 'text/plain', 'a.txt'], ['files', 'b', 'text/plain', 'b.txt'], ['files', 'c', 'text/plain', 'c.txt']]));
+ assert.deepStrictEqual([r.status, r.body.error], [400, 'Too many files. Maximum 2 files per request.']);
+ r = await post('/any', form([['file', 'a', 'text/plain', 'a.txt']]));
+ assert.deepStrictEqual([r.status, r.body.n], [200, 1]);
+ } finally { s.close(); }
+
+ // One runtime: no satellite keeps its own upload middleware.
+ const apps = path.join(__dirname, '..', '..');
+ for (const a of fs.readdirSync(apps)) {
+ if (a.startsWith('_') || !fs.statSync(path.join(apps, a)).isDirectory()) continue;
+ assert.ok(!fs.existsSync(path.join(apps, a, 'server', 'middleware', 'upload.js')), `${a} has its own upload middleware again`);
+ }
+ fs.rmSync(dir, { recursive: true, force: true });
+ console.log('upload (shared runtime): all checks passed');
+})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,93 @@
@@ -0,0 +1,93 @@
+'use strict';
+// ═══════════════════════════════════════════════════════════════
+// Uploads for the satellites (roadmap WS-L task 3: one runtime, no per-app copies).
+//
+// const { uploadSingle, uploadMultiple, uploadAny } = createUploads({ multer: require('multer'), config, storage: 'disk', maxFiles: 50 });
+//
+// multer the app's own (apps/_shared has no node_modules; its modules take packages as arguments)
+//
+// storage 'memory' req.file.buffer: small files only (images)
+// storage 'disk' files land in config.uploadsDir with a random name and a sanitised extension, so
+// large or many files never sit in memory while they arrive; the routes read and
+// delete them, the retention sweep removes leftovers, job submits move them
+// maxFiles per request, for uploadMultiple / uploadAny ('files' field)
+// rejectHint the end of the unsupported-type message (default: the accepted types)
+//
+// The declared type must be one of config.upload.allowedMimes, or application/octet-stream (API
+// clients, openvibe-sdk): the guard checks the bytes of every upload against the tool's accepted
+// types (guard/sniff.js). Answers are the { error } JSON the apps' pages already read.
+// ═══════════════════════════════════════════════════════════════
+
+const path = require('path');
+const fs = require('fs');
+const crypto = require('crypto');
+
+function createUploads({ multer, config, storage = 'memory', maxFiles = 1, rejectHint = null } = {}) {
+ if (typeof multer !== 'function') throw new TypeError('createUploads needs multer (the app\'s require(\'multer\'))');
+ if (!config || !config.upload) throw new TypeError('createUploads needs the app config (upload.maxFileSize, upload.allowedMimes)');
+ const allowed = new Set(config.upload.allowedMimes || []);
+ const maxMb = Math.round(config.upload.maxFileSize / 1024 / 1024);
+ let store;
+ if (storage === 'disk') {
+ const dir = path.resolve(config.uploadsDir || 'data/uploads');
+ fs.mkdirSync(dir, { recursive: true });
+ store = multer.diskStorage({
+ destination: (_req, _file, cb) => cb(null, dir),
+ filename: (_req, file, cb) => {
+ const ext = (path.extname(file.originalname || '').toLowerCase().match(/^\.[a-z0-9]{1,8}$/) || ['.bin'])[0];
+ cb(null, `${crypto.randomBytes(16).toString('hex')}${ext}`);
+ },
+ });
+ } else {
+ store = multer.memoryStorage();
+ }
+ const upload = multer({
+ storage: store,
+ limits: { fileSize: config.upload.maxFileSize, files: Math.max(1, maxFiles) },
+ fileFilter(_req, file, cb) {
+ if (allowed.has(file.mimetype) || file.mimetype === 'application/octet-stream') return cb(null, true);
+ cb(new Error(`Unsupported file type: ${file.mimetype}. ${rejectHint || `Accepted: ${[...allowed].join(', ')}`}`));
+ },
+ });
+ const tooLarge = (per) => `File too large. Maximum ${maxMb}MB${per ? ' per file' : ''}.`;
+
+ /** One file in 'file' → req.file ({ buffer } in memory, { path } on disk, plus originalname, mimetype, size). */
+ function uploadSingle(req, res, next) {
+ upload.single('file')(req, res, (err) => {
+ if (err) return res.status(err.code === 'LIMIT_FILE_SIZE' ? 413 : 400).json({ error: err.code === 'LIMIT_FILE_SIZE' ? tooLarge(false) : err.message });
+ if (!req.file) return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });
+ next();
+ });
+ }
+
+ /** Up to maxFiles in 'files' → req.files (an array). */
+ function uploadMultiple(req, res, next) {
+ upload.array('files', maxFiles)(req, res, (err) => {
+ if (err) {
+ if (err.code === 'LIMIT_FILE_SIZE') return res.status(413).json({ error: tooLarge(true) });
+ if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') return res.status(400).json({ error: `Too many files. Maximum ${maxFiles} files per request.` });
+ return res.status(400).json({ error: err.message });
+ }
+ if (!req.files || req.files.length === 0) return res.status(400).json({ error: 'No files uploaded. Send files in the "files" field.' });
+ next();
+ });
+ }
+
+ /** One file in 'file' or up to maxFiles in 'files' (the job endpoint takes either) → req.files { file?, files? }. */
+ function uploadAny(req, res, next) {
+ upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: maxFiles }])(req, res, (err) => {
+ if (err) {
+ if (err.code === 'LIMIT_FILE_SIZE') return res.status(413).json({ error: tooLarge(true) });
+ if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') return res.status(400).json({ error: `Send one file in "file" or up to ${maxFiles} in "files".` });
+ return res.status(400).json({ error: err.message });
+ }
+ const n = req.files ? Object.values(req.files).flat().length : 0;
+ if (!n) return res.status(400).json({ error: 'No files uploaded. Send a file in the "file" field.' });
+ next();
+ });
+ }
+
+ return { uploadSingle, uploadMultiple, uploadAny };
+}
+
+module.exports = { createUploads };
@@ -18,7 +18,8 @@ const config = require('./config');
@@ -18,7 +18,8 @@ const config = require('./config');
const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { getTool, listTools } = require('./tools');
const { getTool, listTools } = require('./tools');
const { readMetadata } = require('./tools/metadata');
const { readMetadata } = require('./tools/metadata');
-const { uploadSingle, uploadMultiple, uploadAny } = require('./middleware/upload');
+// Uploads from the shared runtime (apps/_shared/upload.js, WS-L task 3).
+const { uploadSingle, uploadMultiple, uploadAny } = require('../../_shared/upload').createUploads({ multer: require('multer'), config, storage: 'disk', maxFiles: 5, rejectHint: 'Upload an audio or video file.' });
const retention = require('./retention/manager');
const retention = require('./retention/manager');
const { probe, getDuration, cleanTmp } = require('./tools/ffmpeg-helper');
const { probe, getDuration, cleanTmp } = require('./tools/ffmpeg-helper');
const { buildOptions, describe, defineJobs, runSync, jobContext, limitsFor } = require('./process');
const { buildOptions, describe, defineJobs, runSync, jobContext, limitsFor } = require('./process');
deleted file mode 100644
deleted file mode 100644
@@ -1,102 +0,0 @@
@@ -1,102 +0,0 @@
-'use strict';
-
-// ═══════════════════════════════════════════════════════════════
-// Audio.OpenVibe — Multer Upload Middleware
-// Disk storage (audio files can be large) with size + mime validation.
-// ═══════════════════════════════════════════════════════════════
-
-const multer = require('multer');
-const path = require('path');
-const fs = require('fs');
-const crypto = require('crypto');
-const config = require('../config');
-
-const ALLOWED_MIMES = new Set(config.upload.allowedMimes);
-
-// Use disk storage for audio — memory storage is risky for 100MB files
-const uploadsDir = path.resolve(config.uploadsDir);
-if (!fs.existsSync(uploadsDir)) fs.mkdirSync(uploadsDir, { recursive: true });
-
-const storage = multer.diskStorage({
- destination: (_req, _file, cb) => cb(null, uploadsDir),
- filename: (_req, file, cb) => {
- const id = crypto.randomBytes(16).toString('hex');
- const ext = path.extname(file.originalname) || '.bin';
- cb(null, `${id}${ext}`);
- },
-});
-
-const upload = multer({
- storage,
- limits: {
- fileSize: config.upload.maxFileSize,
- files: 5, // Allow up to 5 for merge tool
- },
- fileFilter(_req, file, cb) {
- // Be lenient — some browsers send application/octet-stream for audio
- if (ALLOWED_MIMES.has(file.mimetype) || file.mimetype === 'application/octet-stream') {
- cb(null, true);
- } else {
- cb(new Error(`Unsupported file type: ${file.mimetype}. Upload an audio or video file.`));
- }
- },
-});
-
-/**
- * Single file upload on 'file' field.
- */
-function uploadSingle(req, res, next) {
- upload.single('file')(req, res, (err) => {
- if (err) {
- if (err.code === 'LIMIT_FILE_SIZE') {
- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB.` });
- }
- return res.status(400).json({ error: err.message });
- }
- if (!req.file) {
- return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });
- }
- next();
- });
-}
-
-/**
- * Multiple file upload on 'files' field (for merge).
- */
-function uploadMultiple(req, res, next) {
- upload.array('files', 5)(req, res, (err) => {
- if (err) {
- if (err.code === 'LIMIT_FILE_SIZE') {
- return res.status(413).json({ error: `Files too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB each.` });
- }
- return res.status(400).json({ error: err.message });
- }
- if (!req.files || req.files.length < 2) {
- return res.status(400).json({ error: 'Upload at least 2 files to merge.' });
- }
- next();
- });
-}
-
-/**
- * One file in 'file' or up to 5 in 'files' (the job endpoint takes either shape; merge sends 'files').
- * Attaches req.files as { file?: [...], files?: [...] }.
- */
-function uploadAny(req, res, next) {
- upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: 5 }])(req, res, (err) => {
- if (err) {
- if (err.code === 'LIMIT_FILE_SIZE') {
- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });
- }
- if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') {
- return res.status(400).json({ error: 'Send one file in "file" or up to 5 in "files".' });
- }
- return res.status(400).json({ error: err.message });
- }
- const n = req.files ? Object.values(req.files).flat().length : 0;
- if (!n) return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });
- next();
- });
-}
-
-module.exports = { uploadSingle, uploadMultiple, uploadAny };
@@ -12,7 +12,7 @@ const { tmpFile, probe, getAudioStream, getDuration, getMime } = require('./ffmp
@@ -12,7 +12,7 @@ const { tmpFile, probe, getAudioStream, getDuration, getMime } = require('./ffmp
const { FORMAT_CONFIG } = require('./convert');
const { FORMAT_CONFIG } = require('./convert');
const MIN_FILES = 2;
const MIN_FILES = 2;
-const MAX_FILES = 5; // the upload limit (middleware/upload.js)
+const MAX_FILES = 5; // the upload limit (createUploads maxFiles in server/index.js)
const MAX_TOTAL_SECONDS = 3 * 60 * 60; // three hours of joined audio
const MAX_TOTAL_SECONDS = 3 * 60 * 60; // three hours of joined audio
const LOSSY = new Set(['mp3', 'ogg', 'm4a', 'aac', 'opus', 'wma', 'ac3', 'webm']);
const LOSSY = new Set(['mp3', 'ogg', 'm4a', 'aac', 'opus', 'wma', 'ac3', 'webm']);
@@ -18,7 +18,8 @@ const rateLimit = require('express-rate-limit');
@@ -18,7 +18,8 @@ const rateLimit = require('express-rate-limit');
const config = require('./config');
const config = require('./config');
const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { getTool, listTools, BINARIES } = require('./tools');
const { getTool, listTools, BINARIES } = require('./tools');
-const { uploadSingle, uploadMultiple, uploadAny } = require('./middleware/upload');
+// Uploads from the shared runtime (apps/_shared/upload.js, WS-L task 3).
+const { uploadSingle, uploadMultiple, uploadAny } = require('../../_shared/upload').createUploads({ multer: require('multer'), config, storage: 'disk', maxFiles: 50 });
const retention = require('./retention/manager');
const retention = require('./retention/manager');
const { buildOptions, defineJobs, runTool } = require('./process');
const { buildOptions, defineJobs, runTool } = require('./process');
const { poolFromEnv } = require('../../_shared/jobs/pool');
const { poolFromEnv } = require('../../_shared/jobs/pool');
deleted file mode 100644
deleted file mode 100644
@@ -1,109 +0,0 @@
@@ -1,109 +0,0 @@
-'use strict';
-
-// ═══════════════════════════════════════════════════════════════
-// Docs.OpenVibe — Multer Upload Middleware
-// Disk storage (up to 50 files of 100 MB for a merge must never sit in memory while they arrive) with
-// size + declared-type validation; the guard then checks the bytes (apps/_shared/guard/sniff.js).
-// Files land in config.uploadsDir with a random name; the routes read them and delete them, and the
-// retention sweep removes anything older than retention.tempMaxAge. Job submits move them into the job.
-// ═══════════════════════════════════════════════════════════════
-
-const multer = require('multer');
-const path = require('path');
-const fs = require('fs');
-const crypto = require('crypto');
-const config = require('../config');
-
-const ALLOWED_MIMES = new Set(config.upload.allowedMimes);
-
-const uploadsDir = path.resolve(config.uploadsDir);
-fs.mkdirSync(uploadsDir, { recursive: true });
-
-const storage = multer.diskStorage({
- destination: (_req, _file, cb) => cb(null, uploadsDir),
- filename: (_req, file, cb) => {
- const ext = (path.extname(file.originalname || '').toLowerCase().match(/^\.[a-z0-9]{1,8}$/) || ['.bin'])[0];
- cb(null, `${crypto.randomBytes(16).toString('hex')}${ext}`);
- },
-});
-
-const upload = multer({
- storage,
- limits: {
- fileSize: config.upload.maxFileSize,
- files: 50, // Allow up to 50 files for merge
- },
- fileFilter(_req, file, cb) {
- // A part without a type of its own (application/octet-stream: API clients, openvibe-sdk) is let
- // through: the guard checks the bytes of every upload against the tool's accepted types.
- if (ALLOWED_MIMES.has(file.mimetype) || file.mimetype === 'application/octet-stream') {
- cb(null, true);
- } else {
- cb(new Error(`Unsupported file type: ${file.mimetype}. Accepted: ${config.upload.allowedMimes.join(', ')}`));
- }
- },
-});
-
-/**
- * Single file upload on 'file' field.
- * Attaches req.file with { path, originalname, mimetype, size }.
- */
-function uploadSingle(req, res, next) {
- upload.single('file')(req, res, (err) => {
- if (err) {
- if (err.code === 'LIMIT_FILE_SIZE') {
- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB.` });
- }
- return res.status(400).json({ error: err.message });
- }
- if (!req.file) {
- return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });
- }
- next();
- });
-}
-
-/**
- * Multiple file upload on 'files' field (for merge, img2pdf, etc.).
- * Attaches req.files as an array of { path, originalname, mimetype, size }.
- */
-function uploadMultiple(req, res, next) {
- upload.array('files', 50)(req, res, (err) => {
- if (err) {
- if (err.code === 'LIMIT_FILE_SIZE') {
- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });
- }
- if (err.code === 'LIMIT_FILE_COUNT') {
- return res.status(400).json({ error: 'Too many files. Maximum 50 files per request.' });
- }
- return res.status(400).json({ error: err.message });
- }
- if (!req.files || req.files.length === 0) {
- return res.status(400).json({ error: 'No files uploaded. Send files in the "files" field.' });
- }
- next();
- });
-}
-
-/**
- * One file in 'file' or several in 'files' (the job endpoint takes either shape).
- * Attaches req.files as { file?: [...], files?: [...] }.
- */
-function uploadAny(req, res, next) {
- upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: 50 }])(req, res, (err) => {
- if (err) {
- if (err.code === 'LIMIT_FILE_SIZE') {
- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });
- }
- if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') {
- return res.status(400).json({ error: 'Send one file in "file" or up to 50 in "files".' });
- }
- return res.status(400).json({ error: err.message });
- }
- const n = req.files ? Object.values(req.files).flat().length : 0;
- if (!n) return res.status(400).json({ error: 'No files uploaded. Send a file in the "file" field.' });
- next();
- });
-}
-
-module.exports = { uploadSingle, uploadMultiple, uploadAny };
@@ -17,7 +17,8 @@ const rateLimit = require('express-rate-limit');
@@ -17,7 +17,8 @@ const rateLimit = require('express-rate-limit');
const config = require('./config');
const config = require('./config');
const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { getTool, listTools } = require('./tools');
const { getTool, listTools } = require('./tools');
-const { uploadSingle } = require('./middleware/upload');
+// Uploads from the shared runtime (apps/_shared/upload.js, WS-L task 3).
+const { uploadSingle } = require('../../_shared/upload').createUploads({ multer: require('multer'), config, storage: 'memory', maxFiles: 1 });
const retention = require('./retention/manager');
const retention = require('./retention/manager');
const { buildOptions, describe, runProcess, defineJobs } = require('./process');
const { buildOptions, describe, runProcess, defineJobs } = require('./process');
const { poolFromEnv } = require('../../_shared/jobs/pool');
const { poolFromEnv } = require('../../_shared/jobs/pool');
deleted file mode 100644
deleted file mode 100644
@@ -1,51 +0,0 @@
@@ -1,51 +0,0 @@
-'use strict';
-
-// ═══════════════════════════════════════════════════════════════
-// Img.OpenVibe — Multer Upload Middleware
-// Memory storage with file size + mime validation.
-// ═══════════════════════════════════════════════════════════════
-
-const multer = require('multer');
-const config = require('../config');
-
-const ALLOWED_MIMES = new Set(config.upload.allowedMimes);
-
-const storage = multer.memoryStorage();
-
-const upload = multer({
- storage,
- limits: {
- fileSize: config.upload.maxFileSize,
- files: 1,
- },
- fileFilter(_req, file, cb) {
- // A part without a type of its own (application/octet-stream: API clients, openvibe-sdk) is let
- // through: the guard checks the bytes of every upload against the tool's accepted types.
- if (ALLOWED_MIMES.has(file.mimetype) || file.mimetype === 'application/octet-stream') {
- cb(null, true);
- } else {
- cb(new Error(`Unsupported file type: ${file.mimetype}. Accepted: ${config.upload.allowedMimes.join(', ')}`));
- }
- },
-});
-
-/**
- * Express middleware for single file upload on 'file' field.
- * Attaches req.file with { buffer, originalname, mimetype, size }.
- */
-function uploadSingle(req, res, next) {
- upload.single('file')(req, res, (err) => {
- if (err) {
- if (err.code === 'LIMIT_FILE_SIZE') {
- return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB.` });
- }
- return res.status(400).json({ error: err.message });
- }
- if (!req.file) {
- return res.status(400).json({ error: 'No file uploaded. Send a file in the "file" field.' });
- }
- next();
- });
-}
-
-module.exports = { uploadSingle };