openvibers/openvibe.tools Commits
Durable jobs on Img, Audio and Docs: /api/v1/jobs with SSE progress, cancel, reattach, idempotency keys and owner scoping; results can be Media objects
Changes
nnew file mode 100644@@@ -0,0 +1,22 @@++name: ci++on:++ push:++ branches: [main]++ pull_request:++jobs:++ test:++ runs-on: ubuntu-latest++ steps:++ - uses: actions/checkout@v4++ - uses: actions/setup-node@v4++ with:++ node-version: 22.22.1 # production runs Node 22++ - name: ffmpeg (audio tests)++ run: sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg++ - run: npm run install:all++ - run: npm test++ - name: contracts check (satellites that use openvibe-contracts)++ run: |++ for app in img audio docs; do++ (cd apps/$app && npx openvibe-contracts-check --service tools --src server --src ../_shared)++ done@@@ -73,6 +73,50 @@ function canonicalUrlFor(req, fallbackHost, pathname) {return `https://${canonicalHostFor(req, fallbackHost)}${p}`;}++/**++ * Which host of the satellite's own domain map a request is for. Through the gateway the tool id++ * decides (a descriptive host, a mirror or a custom domain is not in the satellite's map, but++ * `<tool>.openvibe.tools` is); otherwise the Host header.++ * @param {(host: string) => boolean} has is this hostname in the satellite's map?++ */++function ownHost(req, has) {++ const info = req.ovHost || hostRole(req);++ const byTool = info.tool ? `${info.tool}.openvibe.tools` : '';++ if (byTool && has(byTool)) return byTool;++ return requestHost(req);++}++++/**++ * Swap `https://<fromHost>` for `https://<toHost>` inside <head> — for static pages whose canonical,++ * og:url and JSON-LD were written for one host (maps, food).++ */++function restampHead(html, fromHost, toHost) {++ if (!fromHost || !toHost || fromHost === toHost) return html;++ const end = html.search(/<\/head>/i);++ if (end < 0) return html;++ const from = new RegExp(`https://${fromHost.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=[/"'])`, 'g');++ return html.slice(0, end).replace(from, `https://${toHost}`) + html.slice(end);++}++++/**++ * Handler for a static page written for one host: the canonical, og:url and JSON-LD in its <head>++ * are rewritten to the canonical host of this request (the gateway's X-OV-Canonical-Host, or++ * `ownHost` itself when reached directly). The file is read once.++ */++function stampedPage(file, ownHostName) {++ let html = null;++ const cache = new Map();++ return function sendStampedPage(req, res) {++ if (html === null) html = require('fs').readFileSync(file, 'utf8');++ const canonical = canonicalHostFor(req, ownHostName);++ if (!cache.has(canonical)) { if (cache.size > 50) cache.clear(); cache.set(canonical, restampHead(html, ownHostName, canonical)); }++ res.setHeader('Content-Type', 'text/html; charset=utf-8');++ res.setHeader('Cache-Control', 'no-cache');++ res.setHeader('Vary', 'X-OV-Canonical-Host');++ return res.send(cache.get(canonical));++ };++}++/*** Express middleware. Sets `req.ovHost` and turns away hosts this satellite does not serve.*@@@ -117,4 +161,4 @@ function hostGuard(opts) {};}--module.exports = { hostRole, hostGuard, canonicalHostFor, canonicalUrlFor, requestHost, isLocalHost, cleanHost, TOOLS_HOME };++module.exports = { hostRole, hostGuard, canonicalHostFor, canonicalUrlFor, ownHost, restampHead, stampedPage, requestHost, isLocalHost, cleanHost, TOOLS_HOME };nnew file mode 100644@@@ -0,0 +1,116 @@++/* OpenVibe Tools — browser helper for /api/v1/jobs (served at /js/ov-jobs.js by every satellite that runs jobs).++ *++ * const job = await OVJobs.submit({ type: 'img.process', input: { tool: 'convert', format: 'webp' }, files: [file] });++ * OVJobs.remember(job.id); // ?job=<id> in the address + sessionStorage: a reload reattaches++ * OVJobs.watch(job.id, { onUpdate(job) {…}, onDone(job) {…} });++ * const pending = OVJobs.recall(); // on load: the job to reattach to, if any++ *++ * Progress arrives over SSE (EventSource resumes with Last-Event-ID by itself); if the stream cannot be++ * kept open the helper polls GET /api/v1/jobs/:id instead. Errors are problem+json; submit() throws an++ * Error whose message is the problem's detail.++ */++(function () {++ 'use strict';++ var TERMINAL = { succeeded: 1, failed: 1, cancelled: 1 };++ var EVENTS = ['job.queued', 'job.running', 'job.progress', 'job.cancel_requested', 'job.succeeded', 'job.failed', 'job.cancelled'];++ var KEY = 'ov-job:' + location.host + location.pathname;++++ function newKey() {++ if (window.crypto && crypto.randomUUID) return crypto.randomUUID();++ return String(Date.now()) + '-' + Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2);++ }++ function problemError(res, body) {++ var err = new Error((body && (body.detail || body.error || body.title)) || ('Server error (' + res.status + ')'));++ err.status = res.status; err.problem = body || null;++ return err;++ }++ function readJson(res) { return res.json().catch(function () { return null; }); }++++ function submit(o) {++ var fd = new FormData();++ fd.append('type', o.type);++ fd.append('input', JSON.stringify(o.input || {}));++ var files = o.files || [];++ var field = o.fileField || (files.length > 1 ? 'files' : 'file');++ for (var i = 0; i < files.length; i++) fd.append(field, files[i]);++ return fetch('/api/v1/jobs', {++ method: 'POST', body: fd, credentials: 'same-origin',++ headers: { 'Idempotency-Key': o.idempotencyKey || newKey(), Accept: 'application/json' },++ }).then(function (res) {++ return readJson(res).then(function (body) { if (!res.ok) throw problemError(res, body); return body; });++ });++ }++++ function get(id) {++ return fetch('/api/v1/jobs/' + encodeURIComponent(id), { credentials: 'same-origin', headers: { Accept: 'application/json' } })++ .then(function (res) { return readJson(res).then(function (body) { if (!res.ok) throw problemError(res, body); return body; }); });++ }++++ function cancel(id) {++ return fetch('/api/v1/jobs/' + encodeURIComponent(id), { method: 'DELETE', credentials: 'same-origin', headers: { Accept: 'application/json' } })++ .then(function (res) { return readJson(res).then(function (body) { if (!res.ok) throw problemError(res, body); return body; }); });++ }++++ /** Follow a job until it ends. → { close() } */++ function watch(id, h) {++ h = h || {};++ var done = false, es = null, timer = null, failures = 0;++ function finish(job) {++ if (done) return; done = true;++ if (es) es.close();++ if (timer) clearTimeout(timer);++ if (h.onDone) h.onDone(job);++ }++ function update(job) {++ if (done || !job) return;++ if (h.onUpdate) h.onUpdate(job);++ if (TERMINAL[job.state]) finish(job);++ }++ function poll() {++ if (done) return;++ get(id).then(function (job) { failures = 0; update(job); if (!done) timer = setTimeout(poll, 1500); })++ .catch(function (err) {++ if (err.status === 404) { done = true; if (h.onGone) h.onGone(err); return; }++ failures++; timer = setTimeout(poll, Math.min(15000, 1500 * failures));++ });++ }++ if (!window.EventSource) { poll(); return { close: function () { done = true; if (timer) clearTimeout(timer); } }; }++ es = new EventSource('/api/v1/jobs/' + encodeURIComponent(id) + '/events', { withCredentials: true });++ EVENTS.forEach(function (name) {++ es.addEventListener(name, function (e) { try { update(JSON.parse(e.data)); } catch (_) { /* ignore a bad frame */ } });++ });++ es.onerror = function () {++ // CLOSED: the server said 204 (nothing new — the job has ended) or refused; ask once, then poll.++ if (!done && es.readyState === 2) { es = null; poll(); }++ };++ return { close: function () { done = true; if (es) es.close(); if (timer) clearTimeout(timer); } };++ }++++ function remember(id) {++ try { sessionStorage.setItem(KEY, id); } catch (_) { /* storage may be off */ }++ try { var u = new URL(location.href); u.searchParams.set('job', id); history.replaceState(history.state, '', u.toString()); } catch (_) { /* old browser */ }++ }++ function recall() {++ var fromUrl = null;++ try { fromUrl = new URL(location.href).searchParams.get('job'); } catch (_) { /* old browser */ }++ if (fromUrl && /^job_[0-9A-Z]{26}$/.test(fromUrl)) return fromUrl;++ try { var s = sessionStorage.getItem(KEY); if (s && /^job_[0-9A-Z]{26}$/.test(s)) return s; } catch (_) { /* storage may be off */ }++ return null;++ }++ function forget() {++ try { sessionStorage.removeItem(KEY); } catch (_) { /* storage may be off */ }++ try { var u = new URL(location.href); if (u.searchParams.has('job')) { u.searchParams.delete('job'); history.replaceState(history.state, '', u.toString()); } } catch (_) { /* old browser */ }++ }++++ function fileUrl(job, n, opts) {++ var f = job && job.result && job.result.files && job.result.files[n || 0];++ if (!f) return null;++ return f.url + (opts && opts.inline ? '?inline=1' : '');++ }++ /** Milliseconds until a finished job's files expire (null while it is still running). */++ function expiresIn(job) {++ return job && job.expires_at ? Math.max(0, Date.parse(job.expires_at) - Date.now()) : null;++ }++++ window.OVJobs = { submit: submit, get: get, cancel: cancel, watch: watch, remember: remember, recall: recall, forget: forget, fileUrl: fileUrl, expiresIn: expiresIn, TERMINAL: TERMINAL };++})();nnew file mode 100644@@@ -0,0 +1,261 @@++'use strict';++// ═══════════════════════════════════════════════════════════════++// Jobs over HTTP — the same routes on every satellite that runs jobs.++//++// POST /api/v1/jobs submit (JSON { type, input, idempotency_key } or multipart:++// type, input (JSON text), file/files; Idempotency-Key header)++// → 202 + Location (new) | 200 + Idempotent-Replayed: true++// GET /api/v1/jobs/:id the job (reattach by id)++// DELETE /api/v1/jobs/:id cancel → 200 cancelled | 202 cancel requested (running)++// GET /api/v1/jobs/:id/events SSE: job.queued|running|progress|cancel_requested|succeeded|failed|cancelled;++// every event has an id; reconnecting with Last-Event-ID replays++// only what came after it; 204 once there is nothing left to say++// GET /api/v1/jobs/:id/files/:n a result file (attachment; ?inline=1 for previews)++// GET /js/ov-jobs.js the browser helper (submit, watch, reattach)++//++// Owner scoping: a job is visible only to whoever created it —++// a Network user (user:usr_…, from the ov_token subject_id), a service/app principal presenting a++// Network client-credentials token for audience openvibe.tools with tools.job.* capabilities, or,++// for everyone else, this browser's jobs session cookie. Anyone else gets the same 404 as for a++// job that does not exist. Errors are RFC 9457 problem+json.++// ═══════════════════════════════════════════════════════════════++++const fs = require('fs');++const path = require('path');++const crypto = require('crypto');++const { Readable } = require('stream');++++const ID_RE = /^job_[0-9A-HJKMNP-TV-Z]{26}$/;++const SESSION_RE = /^[A-Za-z0-9_-]{32,64}$/;++const TERMINAL = new Set(['succeeded', 'failed', 'cancelled']);++const CLIENT_JS = path.join(__dirname, 'client.js');++const CAPS = { create: 'tools.job.create', read: 'tools.job.read', cancel: 'tools.job.cancel' };++++const b64json = (part) => { try { return JSON.parse(Buffer.from(String(part).replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8')); } catch { return null; } };++++function contentDisposition(kind, name) {++ const ascii = String(name || 'result').replace(/[^\x20-\x7e]/g, '_').replace(/["\\]/g, '_');++ return `${kind}; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(String(name || 'result'))}`;++}++++/**++ * Who is asking. → { owner, kind: 'principal'|'user'|'session', claims? } | { error: [status, code, detail] } | null++ *++ * @param {object} o++ * @param {object} o.contracts++ * @param {() => string|null} o.getPublicKey the Network public key PEM the app already loads++ * @param {string} o.issuer the Network issuer URL++ * @param {string} [o.audience='openvibe.tools']++ * @param {string} [o.cookieName='ov_tools_jobs']++ * @param {boolean} [o.secureCookie]++ */++function createOwnerResolver(o) {++ const audience = o.audience || 'openvibe.tools';++ const cookieName = o.cookieName || 'ov_tools_jobs';++ const { contracts } = o;++++ function principalCapability(claims, cap) {++ const c = contracts.capabilities.check(claims, cap);++ if (c.allowed) return null;++ // The tools.job.* manifests are proposed for the next contracts release; until the installed++ // version knows them, an explicit grant in the token (which only Network can sign) is enough.++ if (c.code === 'capability.unknown' && contracts.capabilities.grants(claims.cap, cap)) return null;++ return [403, c.code === 'capability.unknown' ? 'capability.denied' : c.code, `${cap} not granted`];++ }++++ return function resolveOwner(req, res, { create = false, action = 'read' } = {}) {++ const auth = String(req.headers.authorization || '');++ if (auth.startsWith('Bearer ')) {++ const token = auth.slice(7).trim();++ const parts = token.split('.');++ const claims = parts.length === 3 ? b64json(parts[1]) : null;++ const looksLikePrincipal = claims && (Array.isArray(claims.cap) || /^(svc|app|mod):/.test(String(claims.sub || '')));++ if (looksLikePrincipal) {++ const r = contracts.serviceAuth.verifyServiceToken(token, { publicKey: o.getPublicKey(), issuer: o.issuer, audience });++ if (!r.ok) return { error: [401, r.code, r.reason] };++ const denied = principalCapability(r.claims, CAPS[action] || CAPS.read);++ if (denied) return { error: denied };++ return { owner: r.claims.sub, kind: 'principal', claims: r.claims };++ }++ }++ const sid = req.user && req.user.subject_id;++ if (sid && contracts.ids.isSubjectId('user', sid)) return { owner: `user:${sid}`, kind: 'user' };++++ let session = req.cookies && req.cookies[cookieName];++ if (!SESSION_RE.test(String(session || ''))) {++ if (!create) return null;++ session = crypto.randomBytes(24).toString('base64url');++ res.cookie(cookieName, session, {++ httpOnly: true, sameSite: 'lax', path: '/', maxAge: 7 * 24 * 60 * 60 * 1000,++ secure: o.secureCookie != null ? o.secureCookie : process.env.NODE_ENV === 'production',++ });++ }++ // Only a hash of the cookie is stored, so the database cannot be used to take over a session.++ return { owner: `session:${crypto.createHash('sha256').update(session).digest('hex').slice(0, 40)}`, kind: 'session' };++ };++}++++/**++ * Mount the job routes on an Express app.++ *++ * @param {object} app++ * @param {object} o++ * @param {object} o.system createJobSystem(...)++ * @param {object} o.contracts++ * @param {function} o.resolveOwner createOwnerResolver(...)++ * @param {function} [o.receive] multipart middleware that sets req.file / req.files (the app's multer)++ * @param {function[]} [o.limiters] middleware run before accepting a submit (rate limits)++ * @param {(req, who) => number} [o.ttlMs] how long a finished job is kept++ * @param {(req, input) => object} [o.defaults] fills input from the host (e.g. the format a png.* host implies)++ */++function mountJobRoutes(app, o) {++ const { system, contracts } = o;++ const send = (res, status, code, detail, extra) => contracts.http.sendProblem(res, status, code, { detail, extra, ctx: res.req && res.req.ov });++ const ttlMs = o.ttlMs || ((_req, who) => (who.kind === 'session' ? 60 * 60 * 1000 : 24 * 60 * 60 * 1000));++ const noStore = (res) => res.set('Cache-Control', 'no-store');++++ function who(req, res, opts) {++ const w = o.resolveOwner(req, res, opts);++ if (w && w.error) { send(res, ...w.error); return null; }++ return w || { owner: null };++ }++ /** The job, if it exists AND belongs to the caller; otherwise the same 404 either way. */++ function load(req, res, action) {++ const w = who(req, res, { action });++ if (!w) return null;++ const id = String(req.params.id || '');++ const row = ID_RE.test(id) ? system.get(id) : null;++ if (!row || !w.owner || row.owner !== w.owner) { send(res, 404, 'tools.job.not_found', 'No such job'); return null; }++ return row;++ }++++ app.get('/js/ov-jobs.js', (_req, res) => {++ res.set({ 'Content-Type': 'application/javascript; charset=utf-8', 'Cache-Control': 'no-cache' });++ res.sendFile(CLIENT_JS);++ });++++ const receive = o.receive || ((_req, _res, next) => next());++ // The apps' upload middleware answers { error } (their legacy shape); on this API those become problem+json.++ const multipartOnly = (req, res, next) => {++ if (!req.is('multipart/form-data')) return next();++ const json = res.json.bind(res);++ res.json = (body) => {++ res.json = json;++ if (res.statusCode >= 400 && body && body.error && !body.code) return send(res, res.statusCode, res.statusCode === 413 ? 'tools.job.too_large' : 'tools.job.invalid', String(body.error));++ return json(body);++ };++ receive(req, res, (err) => { res.json = json; next(err); });++ };++++ app.post('/api/v1/jobs', ...(o.limiters || []), multipartOnly, async (req, res) => {++ noStore(res);++ const w = who(req, res, { create: true, action: 'create' });++ if (!w) return;++ const b = req.body || {};++ let input = b.input;++ if (typeof input === 'string') {++ try { input = input.trim() ? JSON.parse(input) : {}; } catch { return send(res, 400, 'tools.job.invalid', 'input must be JSON'); }++ }++ if (input == null) input = {};++ if (typeof input !== 'object' || Array.isArray(input)) return send(res, 400, 'tools.job.invalid', 'input must be a JSON object');++ if (o.defaults) input = o.defaults(req, input) || input;++ const list = Array.isArray(req.files) ? req.files : req.files && typeof req.files === 'object' ? Object.values(req.files).flat() : req.file ? [req.file] : [];++ const files = list.map(f => ({ path: f.path, buffer: f.path ? undefined : f.buffer, name: f.originalname, mime: f.mimetype, size: f.size }));++ const key = req.headers['idempotency-key'] != null ? String(req.headers['idempotency-key']) : (b.idempotency_key != null ? String(b.idempotency_key) : null);++ try {++ const { job, replayed } = await system.submit({ owner: w.owner, type: String(b.type || ''), input, files, idempotencyKey: key, ttlMs: ttlMs(req, w) });++ res.set('Location', `/api/v1/jobs/${job.id}`);++ if (replayed) res.set('Idempotent-Replayed', 'true');++ return res.status(replayed ? 200 : 202).json(system.view(job));++ } catch (err) {++ if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);++ console.error('[Jobs] submit failed:', err.message);++ return send(res, 500, 'tools.job.submit_failed', 'The job could not be accepted');++ }++ });++++ app.get('/api/v1/jobs/:id', (req, res) => {++ noStore(res);++ const row = load(req, res, 'read');++ if (row) res.json(system.view(row));++ });++++ app.delete('/api/v1/jobs/:id', (req, res) => {++ noStore(res);++ const row = load(req, res, 'cancel');++ if (!row) return;++ if (row.state === 'succeeded' || row.state === 'failed') return send(res, 409, 'tools.job.already_finished', `The job already ${row.state}`, { state: row.state });++ const { job } = system.cancel(row.id);++ return res.status(job.state === 'cancelled' ? 200 : 202).json(system.view(job));++ });++++ app.get('/api/v1/jobs/:id/events', (req, res) => {++ const row = load(req, res, 'read');++ if (!row) return;++ const raw = req.headers['last-event-id'] != null ? req.headers['last-event-id'] : req.query.last_event_id;++ const after = Math.max(0, parseInt(raw, 10) || 0);++ const pending = system.eventsAfter(row.id, after);++ // Finished and nothing new since the client's last event: 204 tells EventSource to stop reconnecting.++ if (TERMINAL.has(row.state) && !pending.length) return res.status(204).set('Cache-Control', 'no-store').end();++++ res.writeHead(200, {++ 'Content-Type': 'text/event-stream; charset=utf-8',++ 'Cache-Control': 'no-store, no-transform',++ Connection: 'keep-alive',++ 'X-Accel-Buffering': 'no',++ });++ res.write('retry: 3000\n\n');++ let last = after, closed = false, replaying = true;++ const queue = [];++ const close = () => { if (closed) return; closed = true; unsubscribe(); offStop(); clearInterval(keepAlive); res.end(); };++ // On shutdown the stream ends; EventSource reconnects to the next process with Last-Event-ID.++ const offStop = system.onStop(close);++ const write = (e) => {++ if (closed || e.seq <= last) return;++ last = e.seq;++ res.write(`id: ${e.seq}\nevent: ${e.event}\ndata: ${JSON.stringify(e.data)}\n\n`);++ if (TERMINAL.has(e.data.state) && e.event !== 'job.progress') setImmediate(close);++ };++ // Subscribe before replaying so nothing emitted in between is lost; the seq check drops duplicates.++ const unsubscribe = system.subscribe(row.id, (e) => (replaying ? queue.push(e) : write(e)));++ for (const e of system.eventsAfter(row.id, after)) write(e);++ replaying = false;++ queue.forEach(write);++ const keepAlive = setInterval(() => { if (!closed) res.write(': keep-alive\n\n'); }, 15000);++ if (keepAlive.unref) keepAlive.unref();++ req.on('close', close);++ });++++ app.get('/api/v1/jobs/:id/files/:n', async (req, res) => {++ const row = load(req, res, 'read');++ if (!row) return;++ if (row.state !== 'succeeded') return send(res, 409, 'tools.job.not_ready', `The job is ${row.state}`);++ const f = system.resultFile(row, parseInt(req.params.n, 10));++ if (!f) return send(res, 404, 'tools.job.file_not_found', 'No such result file');++ const inline = ['1', 'true'].includes(String(req.query.inline || ''));++ res.set('Cache-Control', 'private, no-store');++ res.set('X-Content-Type-Options', 'nosniff');++ if (f.storage === 'media' && f.media && system.media) {++ try {++ const signed = await system.media.downloadUrl(f.media.media_id);++ // Previews load straight from Media (range requests, no bytes through this server);++ // downloads come through here so the browser saves them under the right name.++ if (inline) return res.redirect(302, signed.url);++ const up = await fetch(signed.internal_url, { signal: AbortSignal.timeout(120_000) });++ if (!up.ok || !up.body) return send(res, 502, 'tools.job.media_unavailable', 'OpenVibe.Media could not return the file');++ res.set({ 'Content-Type': f.mime, 'Content-Disposition': contentDisposition('attachment', f.name) });++ const len = up.headers.get('content-length');++ if (len) res.set('Content-Length', len);++ return Readable.fromWeb(up.body).on('error', () => res.destroy()).pipe(res);++ } catch (err) {++ console.error('[Jobs] media download failed:', err.message);++ return send(res, 502, 'tools.job.media_unavailable', 'OpenVibe.Media could not return the file');++ }++ }++ if (!f.path || !fs.existsSync(f.path)) return send(res, 410, 'tools.job.file_gone', 'The result file has expired');++ res.set({ 'Content-Type': f.mime, 'Content-Disposition': contentDisposition(inline ? 'inline' : 'attachment', f.name) });++ return res.sendFile(f.path);++ });++}++++module.exports = { mountJobRoutes, createOwnerResolver, contentDisposition, CAPS };nnew file mode 100644@@@ -0,0 +1,95 @@++'use strict';++// ═══════════════════════════════════════════════════════════════++// Tools job runtime (roadmap Wave 11). Shared by the satellites the way apps/_shared always is:++// required by relative path, no dependencies of its own — each app passes its better-sqlite3 and++// openvibe-contracts. See system.js (lifecycle), http.js (routes), media.js (results in Media),++// client.js (browser helper).++//++// Environment (one /etc/openvibe/tools.env for every unit):++// TOOLS_JOBS_CONCURRENCY jobs running at once per satellite (default 2)++// TOOLS_JOBS_CONCURRENCY_<APP> per-satellite override, e.g. TOOLS_JOBS_CONCURRENCY_AUDIO=1++// TOOLS_JOBS_MAX_ACTIVE unfinished jobs per owner (default 10)++// TOOLS_JOB_RESULTS local (default) | media — where result files go++// TOOLS_MEDIA_NAMESPACE Media namespace for results (default tools)++// OV_MEDIA_INTERNAL_URL Media on this host (default http://127.0.0.1:4100; MEDIA_URL is read as a fallback)++// OV_MEDIA_URL Media's public origin, for the CSP of previews (default https://openvibe.media)++// OV_NETWORK_INTERNAL_URL token endpoint host (default http://127.0.0.1:4000)++// OV_OAUTH_CLIENT_ID / OV_OAUTH_CLIENT_SECRET the tools client (client_credentials for Media)++// ═══════════════════════════════════════════════════════════════++++const fs = require('fs');++const path = require('path');++const { createJobSystem, JobError } = require('./system');++const { mountJobRoutes, createOwnerResolver, contentDisposition, CAPS } = require('./http');++const { createMediaResults } = require('./media');++++function envInt(name, fallback) {++ const v = parseInt(process.env[name], 10);++ return Number.isFinite(v) && v >= 0 ? v : fallback;++}++++/** Result storage from the environment → { media, mode, reason } (media is null for local). */++function mediaFromEnv(contracts, env = process.env) {++ const want = String(env.TOOLS_JOB_RESULTS || 'local').toLowerCase();++ if (want !== 'media') return { media: null, mode: 'local', reason: 'TOOLS_JOB_RESULTS is not "media"' };++ const secret = env.OV_OAUTH_CLIENT_SECRET || '';++ if (!secret) return { media: null, mode: 'local', reason: 'TOOLS_JOB_RESULTS=media but OV_OAUTH_CLIENT_SECRET is not set' };++ const network = (env.OV_NETWORK_INTERNAL_URL || 'http://127.0.0.1:4000').replace(/\/+$/, '');++ const tokens = contracts.serviceAuth.createTokenClient({++ tokenUrl: `${network}/oauth/token`,++ clientId: env.OV_OAUTH_CLIENT_ID || 'tools',++ clientSecret: secret,++ audience: 'openvibe.media',++ scope: 'media.object.upload media.object.read',++ });++ const media = createMediaResults({++ internalUrl: env.OV_MEDIA_INTERNAL_URL || env.MEDIA_URL || 'http://127.0.0.1:4100',++ namespace: env.TOOLS_MEDIA_NAMESPACE || 'tools',++ tokens,++ });++ return { media, mode: 'media', reason: null };++}++++/** Media's public origin (previews are 302s to it, so the CSP must allow it). */++function mediaOrigin(env = process.env) {++ try { return new URL(env.OV_MEDIA_URL || env.MEDIA_PUBLIC_URL || 'https://openvibe.media').origin; } catch { return 'https://openvibe.media'; }++}++++/**++ * One call per satellite: open <dataDir>/jobs.db, build the system, let the app define its job++ * types, mount the routes and start (boot recovery happens here).++ *++ * @param {object} o++ * @param {object} o.app Express app++ * @param {string} o.service 'img' | 'audio' | 'docs'++ * @param {string} o.dataDir++ * @param {Function} o.Database require('better-sqlite3')++ * @param {object} o.contracts require('openvibe-contracts')++ * @param {() => string|null} o.getPublicKey++ * @param {string} o.issuer++ * @param {(system) => void} o.define registers the app's job types++ * @param {Function} [o.receive] multipart middleware (the app's multer)++ * @param {Function[]} [o.limiters]++ * @param {Function} [o.defaults] (req, input) → input with host defaults++ */++function setupJobs(o) {++ fs.mkdirSync(o.dataDir, { recursive: true });++ const db = new o.Database(path.join(o.dataDir, 'jobs.db'));++ const results = mediaFromEnv(o.contracts);++ if (results.reason && String(process.env.TOOLS_JOB_RESULTS || '').toLowerCase() === 'media') console.warn(`[Jobs] ${o.service}: results stay local — ${results.reason}`);++ const system = createJobSystem({++ db, contracts: o.contracts, service: o.service, dataDir: o.dataDir,++ concurrency: envInt(`TOOLS_JOBS_CONCURRENCY_${o.service.toUpperCase()}`, envInt('TOOLS_JOBS_CONCURRENCY', 2)),++ maxActivePerOwner: envInt('TOOLS_JOBS_MAX_ACTIVE', 10) || 10,++ media: results.media,++ });++ o.define(system);++ const resolveOwner = createOwnerResolver({ contracts: o.contracts, getPublicKey: o.getPublicKey, issuer: o.issuer });++ mountJobRoutes(o.app, { system, contracts: o.contracts, resolveOwner, receive: o.receive, limiters: o.limiters, defaults: o.defaults });++ system.start();++ system.db = db;++ system.close = () => { system.stop(); try { db.close(); } catch { /* already closed */ } };++ return system;++}++++module.exports = { setupJobs, createJobSystem, mountJobRoutes, createOwnerResolver, createMediaResults, mediaFromEnv, mediaOrigin, contentDisposition, JobError, CAPS };nnew file mode 100644@@@ -0,0 +1,115 @@++'use strict';++// ═══════════════════════════════════════════════════════════════++// Job results as OpenVibe.Media objects (Media object API v2, roadmap Wave 4).++//++// POST {media}/api/v2/tools/objects init → { id: med_…, upload: { url, token } }++// PUT {upload.url} the bytes (upload token in the URL)++// POST {media}/api/v2/tools/objects/:id/complete verify sha256 → lifecycle ready++// GET {media}/api/v2/tools/objects/:id/download?format=json → short-lived signed URL++// DELETE {media}/api/v2/tools/objects/:id when the job is pruned++//++// Auth: a Network client-credentials token (audience openvibe.media) granting++// media.object.upload and media.object.read for namespace "tools". The owner goes in++// X-OV-Subject when the job belongs to a signed-in person (user:usr_…); guest and session++// results are service-owned objects in the tools namespace. Objects are private.++//++// Calls go to the internal Media URL; Media's upload URL (built on its public origin) is++// rewritten onto it, so bytes never leave the host.++// ═══════════════════════════════════════════════════════════════++++const fs = require('fs');++const { Readable } = require('stream');++++/**++ * @param {object} o++ * @param {string} o.internalUrl e.g. http://127.0.0.1:4100++ * @param {string} [o.namespace='tools']++ * @param {object} o.tokens contracts.serviceAuth.createTokenClient(...) (audience openvibe.media)++ * @param {function} [o.fetchImpl]++ * @param {number} [o.timeoutMs=120000]++ */++function createMediaResults(o) {++ const base = String(o.internalUrl || '').replace(/\/+$/, '');++ if (!base) throw new TypeError('createMediaResults needs internalUrl');++ const ns = encodeURIComponent(o.namespace || 'tools');++ const fetchImpl = o.fetchImpl || globalThis.fetch;++ const timeoutMs = o.timeoutMs || 120_000;++ const api = `${base}/api/v2/${ns}/objects`;++++ async function call(url, init = {}, retried = false) {++ const headers = { Accept: 'application/json', ...(init.headers || {}), ...(await o.tokens.authHeaders()) };++ const res = await fetchImpl(url, { ...init, headers, signal: AbortSignal.timeout(timeoutMs) });++ if (res.status === 401 && !retried && (init.body == null || typeof init.body === 'string')) { o.tokens.invalidate(); return call(url, init, true); }++ return res;++ }++ async function json(res, what) {++ const body = await res.json().catch(() => null);++ if (!res.ok) {++ const err = new Error(`Media ${what} ${res.status}: ${(body && (body.detail || body.error || body.code)) || 'no body'}`);++ err.status = res.status;++ err.code = body && body.code;++ throw err;++ }++ return body;++ }++ /** Media's upload URL is on its public origin; send the bytes to the internal one. */++ function internal(url) {++ const u = new URL(url);++ const b = new URL(base);++ u.protocol = b.protocol; u.host = b.host;++ return u.toString();++ }++ const subjectHeader = (owner) => {++ const m = /^user:(usr_[0-9A-HJKMNP-TV-Z]{26})$/.exec(String(owner || ''));++ return m ? { 'X-OV-Subject': m[1] } : {};++ };++++ /** Store one file → MediaRef { media_id, role, namespace, size_bytes, content_hash, mime_type }. */++ async function upload({ path: file, name, mime, size, sha256, owner, jobId, service, type }) {++ const init = await json(await call(api, {++ method: 'POST',++ headers: { 'Content-Type': 'application/json', ...subjectHeader(owner) },++ body: JSON.stringify({++ kind: 'file', visibility: 'private', size_bytes: size, mime_type: mime, filename: name, content_hash: sha256,++ metadata: { source: 'openvibe.tools', service, job_id: jobId, job_type: type },++ }),++ }), 'init');++ const id = init && init.id;++ if (!id || !init.upload || !init.upload.url) throw new Error('Media init answered without an upload URL');++ try {++ const put = await fetchImpl(internal(init.upload.url), {++ method: 'PUT',++ headers: { 'Content-Type': mime || 'application/octet-stream', 'Content-Length': String(size) },++ body: Readable.toWeb(fs.createReadStream(file)),++ duplex: 'half',++ signal: AbortSignal.timeout(timeoutMs),++ });++ await json(put, 'content');++ const done = await json(await call(`${api}/${encodeURIComponent(id)}/complete`, {++ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ content_hash: sha256 }),++ }), 'complete');++ return { media_id: id, role: 'output', namespace: o.namespace || 'tools', size_bytes: size, content_hash: sha256, mime_type: mime, status: done.lifecycle_status || 'ready' };++ } catch (err) {++ await remove(id).catch(() => {});++ throw err;++ }++ }++++ /** A short-lived signed URL for the private object → { url, expires_at }. */++ async function downloadUrl(mediaId, ttlSec = 300) {++ const body = await json(await call(`${api}/${encodeURIComponent(mediaId)}/download?format=json&ttl=${ttlSec}`), 'download');++ if (!body || !body.url) throw new Error('Media download answered without a URL');++ return { url: body.url, expires_at: body.expires_at || null, internal_url: internal(body.url) };++ }++++ async function remove(mediaId) {++ const res = await call(`${api}/${encodeURIComponent(mediaId)}`, { method: 'DELETE' });++ if (res.status === 404) return false;++ await json(res, 'delete');++ return true;++ }++++ return { upload, downloadUrl, remove, namespace: o.namespace || 'tools' };++}++++module.exports = { createMediaResults };nnew file mode 100644@@@ -0,0 +1,121 @@++'use strict';++// ═══════════════════════════════════════════════════════════════++// Job store — the satellite's own SQLite (better-sqlite3 handle passed in).++//++// Two tables, created on first use (idempotent):++// tool_jobs one row per job; the durable truth a restart recovers from++// tool_job_events the ordered event log per job; its AUTOINCREMENT seq is the SSE event id,++// so a reconnect with Last-Event-ID replays exactly what was missed++// ═══════════════════════════════════════════════════════════════++++const TERMINAL = ['succeeded', 'failed', 'cancelled'];++const STATES = ['queued', 'running', ...TERMINAL];++++const SCHEMA = `++CREATE TABLE IF NOT EXISTS tool_jobs (++ id TEXT PRIMARY KEY,++ type TEXT NOT NULL,++ type_version INTEGER NOT NULL DEFAULT 1,++ owner TEXT NOT NULL,++ state TEXT NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled')),++ progress REAL,++ progress_message TEXT,++ input_json TEXT NOT NULL DEFAULT '{}',++ files_json TEXT NOT NULL DEFAULT '[]',++ result_json TEXT,++ error_json TEXT,++ retryable INTEGER NOT NULL DEFAULT 0,++ idempotency_key TEXT,++ request_hash TEXT,++ attempts INTEGER NOT NULL DEFAULT 0,++ max_attempts INTEGER NOT NULL DEFAULT 1,++ cancel_requested INTEGER NOT NULL DEFAULT 0,++ ttl_ms INTEGER NOT NULL DEFAULT 3600000,++ created_at INTEGER NOT NULL,++ updated_at INTEGER NOT NULL,++ started_at INTEGER,++ finished_at INTEGER,++ expires_at INTEGER++);++CREATE UNIQUE INDEX IF NOT EXISTS tool_jobs_idem ON tool_jobs(owner, idempotency_key) WHERE idempotency_key IS NOT NULL;++CREATE INDEX IF NOT EXISTS tool_jobs_state ON tool_jobs(state, id);++CREATE INDEX IF NOT EXISTS tool_jobs_expiry ON tool_jobs(expires_at) WHERE expires_at IS NOT NULL;++CREATE INDEX IF NOT EXISTS tool_jobs_owner ON tool_jobs(owner, state);++CREATE TABLE IF NOT EXISTS tool_job_events (++ seq INTEGER PRIMARY KEY AUTOINCREMENT,++ job_id TEXT NOT NULL,++ event TEXT NOT NULL,++ data TEXT NOT NULL,++ at INTEGER NOT NULL++);++CREATE INDEX IF NOT EXISTS tool_job_events_job ON tool_job_events(job_id, seq);++`;++++const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };++++function createStore(db) {++ db.pragma('journal_mode = WAL');++ db.pragma('busy_timeout = 5000');++ db.exec(SCHEMA);++++ const q = {++ insert: db.prepare(`INSERT INTO tool_jobs (id, type, type_version, owner, state, input_json, files_json, idempotency_key, request_hash, max_attempts, ttl_ms, created_at, updated_at)++ VALUES (@id, @type, @type_version, @owner, 'queued', @input_json, @files_json, @idempotency_key, @request_hash, @max_attempts, @ttl_ms, @now, @now)`),++ get: db.prepare('SELECT * FROM tool_jobs WHERE id = ?'),++ byIdem: db.prepare('SELECT * FROM tool_jobs WHERE owner = ? AND idempotency_key = ?'),++ nextQueued: db.prepare("SELECT * FROM tool_jobs WHERE state = 'queued' ORDER BY id LIMIT ?"),++ claim: db.prepare("UPDATE tool_jobs SET state = 'running', attempts = attempts + 1, started_at = @now, updated_at = @now, progress = NULL, progress_message = NULL WHERE id = @id AND state = 'queued'"),++ running: db.prepare("SELECT * FROM tool_jobs WHERE state = 'running'"),++ activeForOwner: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE owner = ? AND state IN ('queued','running')"),++ progress: db.prepare('UPDATE tool_jobs SET progress = @progress, progress_message = @message, updated_at = @now WHERE id = @id AND state = \'running\''),++ requestCancel: db.prepare("UPDATE tool_jobs SET cancel_requested = 1, updated_at = @now WHERE id = @id AND state IN ('queued','running')"),++ requeue: db.prepare("UPDATE tool_jobs SET state = 'queued', started_at = NULL, progress = NULL, progress_message = NULL, updated_at = @now WHERE id = @id AND state = 'running'"),++ finish: db.prepare(`UPDATE tool_jobs SET state = @state, result_json = @result_json, error_json = @error_json, retryable = @retryable,++ progress = CASE WHEN @state = 'succeeded' THEN 100 ELSE progress END, progress_message = NULL,++ finished_at = @now, updated_at = @now, expires_at = @now + ttl_ms WHERE id = @id AND state IN ('queued','running')`),++ setFiles: db.prepare('UPDATE tool_jobs SET files_json = @files_json, updated_at = @now WHERE id = @id'),++ event: db.prepare('INSERT INTO tool_job_events (job_id, event, data, at) VALUES (?, ?, ?, ?)'),++ eventsAfter: db.prepare('SELECT seq, event, data, at FROM tool_job_events WHERE job_id = ? AND seq > ? ORDER BY seq'),++ lastSeq: db.prepare('SELECT MAX(seq) AS seq FROM tool_job_events WHERE job_id = ?'),++ expired: db.prepare('SELECT * FROM tool_jobs WHERE expires_at IS NOT NULL AND expires_at <= ? LIMIT 500'),++ del: db.prepare('DELETE FROM tool_jobs WHERE id = ?'),++ delEvents: db.prepare('DELETE FROM tool_job_events WHERE job_id = ?'),++ counts: db.prepare('SELECT state, COUNT(*) AS n FROM tool_jobs GROUP BY state'),++ };++++ /** Append one event; returns its seq (the SSE id). */++ function appendEvent(jobId, event, data, now = Date.now()) {++ return Number(q.event.run(jobId, event, JSON.stringify(data), now).lastInsertRowid);++ }++++ return {++ db,++ TERMINAL, STATES,++ insert(row) { q.insert.run(row); },++ get: (id) => q.get.get(id) || null,++ byIdempotencyKey: (owner, key) => q.byIdem.get(owner, key) || null,++ nextQueued: (limit) => q.nextQueued.all(limit),++ claim: (id, now = Date.now()) => q.claim.run({ id, now }).changes === 1,++ running: () => q.running.all(),++ activeForOwner: (owner) => q.activeForOwner.get(owner).n,++ setProgress: (id, progress, message, now = Date.now()) => q.progress.run({ id, progress, message, now }).changes === 1,++ requestCancel: (id, now = Date.now()) => q.requestCancel.run({ id, now }).changes === 1,++ requeue: (id, now = Date.now()) => q.requeue.run({ id, now }).changes === 1,++ finish: (id, { state, result = null, error = null, retryable = false }, now = Date.now()) => q.finish.run({++ id, state, now, retryable: retryable ? 1 : 0,++ result_json: result == null ? null : JSON.stringify(result),++ error_json: error == null ? null : JSON.stringify(error),++ }).changes === 1,++ setFiles: (id, files, now = Date.now()) => q.setFiles.run({ id, files_json: JSON.stringify(files), now }),++ appendEvent,++ eventsAfter: (jobId, seq) => q.eventsAfter.all(jobId, seq).map(e => ({ seq: e.seq, event: e.event, data: parse(e.data, {}), at: e.at })),++ lastSeq: (jobId) => q.lastSeq.get(jobId).seq || 0,++ expired: (now = Date.now()) => q.expired.all(now),++ remove: db.transaction((id) => { q.delEvents.run(id); q.del.run(id); }),++ counts() { const out = Object.fromEntries(STATES.map(s => [s, 0])); for (const r of q.counts.all()) out[r.state] = r.n; return out; },++ transaction: (fn) => db.transaction(fn),++ parse,++ };++}++++module.exports = { createStore, TERMINAL, STATES };nnew file mode 100644@@@ -0,0 +1,468 @@++'use strict';++// ═══════════════════════════════════════════════════════════════++// Asynchronous jobs for a Tools satellite (roadmap Wave 11, §15.15).++//++// accepted (202) → queued → running → succeeded | failed | cancelled++//++// Durable: a job is written to the satellite's SQLite, and its input files moved under++// <dataDir>/jobs/<id>/in/, BEFORE the 202 goes out. On boot, rows left 'running' by a restart++// are re-queued (onRestart: 'requeue', while attempts remain) or failed with retryable = true++// (onRestart: 'fail'), per job type. Queued rows simply start again.++//++// Bounded: at most `concurrency` jobs run at once in this process, and each owner may have at++// most `maxActivePerOwner` queued + running jobs. Finished jobs expire (ttl chosen at submit),++// and the pruner deletes the row, its events, its files and any Media objects it made.++//++// No dependencies of its own: the app passes its better-sqlite3 handle and openvibe-contracts.++// ═══════════════════════════════════════════════════════════════++++const fs = require('fs');++const fsp = require('fs/promises');++const path = require('path');++const crypto = require('crypto');++const { createStore, TERMINAL } = require('./store');++++const HOUR = 60 * 60 * 1000;++const EVENT_FOR = { queued: 'job.queued', running: 'job.running', succeeded: 'job.succeeded', failed: 'job.failed', cancelled: 'job.cancelled' };++const SAFE_NAME = (s) => String(s || 'file').replace(/[/\\\0]/g, '_').replace(/[^\w.\- ]/g, '_').slice(-120) || 'file';++++/** A failure the HTTP layer turns into problem+json. */++class JobError extends Error {++ constructor(status, code, detail, extra) { super(detail || code); this.status = status; this.code = code; this.detail = detail; this.extra = extra; }++}++++/** Stable JSON: keys sorted, so the same input always hashes the same. */++function canonical(v) {++ if (Array.isArray(v)) return `[${v.map(canonical).join(',')}]`;++ if (v && typeof v === 'object') return `{${Object.keys(v).sort().filter(k => v[k] !== undefined).map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(',')}}`;++ return JSON.stringify(v === undefined ? null : v);++}++++function sha256File(file) {++ return new Promise((resolve, reject) => {++ const h = crypto.createHash('sha256');++ fs.createReadStream(file).on('data', c => h.update(c)).on('error', reject).on('end', () => resolve(h.digest('hex')));++ });++}++++async function moveFile(from, to) {++ try { await fsp.rename(from, to); } catch { await fsp.copyFile(from, to); await fsp.unlink(from).catch(() => {}); }++}++++/** Tool error text is meant for people; absolute server paths are not. */++const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]').slice(0, 500);++++/**++ * @param {object} o++ * @param {object} o.db better-sqlite3 Database (the satellite's own jobs database)++ * @param {object} o.contracts require('openvibe-contracts')++ * @param {string} o.service 'img' | 'audio' | 'docs' | …++ * @param {string} o.dataDir where job files live (<dataDir>/jobs/<id>/…)++ * @param {number} [o.concurrency=2]++ * @param {number} [o.maxActivePerOwner=10]++ * @param {object} [o.media] result store from ./media (null = keep results on local disk)++ * @param {number} [o.pruneIntervalMs=300000]++ * @param {number} [o.progressThrottleMs=250]++ */++function createJobSystem(o) {++ if (!o || !o.db || !o.contracts || !o.dataDir) throw new TypeError('createJobSystem needs db, contracts and dataDir');++ const { contracts } = o;++ const store = createStore(o.db);++ const log = o.log || console;++ const concurrency = Math.max(0, Number.isFinite(o.concurrency) ? o.concurrency : 2);++ const maxActivePerOwner = Math.max(1, o.maxActivePerOwner || 10);++ const media = o.media || null;++ const throttleMs = o.progressThrottleMs == null ? 250 : o.progressThrottleMs;++ const root = path.resolve(o.dataDir, 'jobs');++ fs.mkdirSync(root, { recursive: true });++++ const types = new Map();++ const active = new Map(); // id → { ctrl, reason, last: { pct, msg, at } }++ const listeners = new Map(); // id → Set<fn>++ const stopHooks = new Set(); // e.g. open SSE streams, ended on stop()++ let started = false, stopped = false, pruneTimer = null, kicking = false;++++ const jobDir = (id) => path.join(root, id);++ const problem = (status, code, detail, extra) => contracts.http.problem(status, code, { detail, extra });++++ // ── Types ──────────────────────────────────────────────────++ /**++ * define({ type, version, onRestart: 'requeue'|'fail', maxAttempts, timeoutMs, minFiles, maxFiles,++ * validate(input, files) → string|null, run(ctx) → { files: [{ path, name, mime }], data } })++ */++ function define(def) {++ if (!def || !/^[a-z][a-z0-9]*\.[a-z0-9_.]+$/.test(def.type || '')) throw new TypeError(`bad job type ${def && def.type}`);++ if (typeof def.run !== 'function') throw new TypeError(`${def.type}: run() is required`);++ types.set(def.type, {++ version: 1, onRestart: 'requeue', maxAttempts: 3, timeoutMs: 10 * 60 * 1000, minFiles: 0, maxFiles: 1,++ validate: () => null, ...def,++ onRestart: def.onRestart === 'fail' ? 'fail' : 'requeue',++ });++ return api;++ }++++ // ── Public representation ─────────────────────────────────++ const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());++ function publicResult(result, id) {++ if (!result) return null;++ return {++ files: (result.files || []).map((f, i) => {++ const out = {};++ for (const [k, v] of Object.entries(f)) if (!k.startsWith('_')) out[k] = v;++ out.url = `/api/v1/jobs/${id}/files/${i}`;++ return out;++ }),++ data: result.data || {},++ };++ }++ function view(row) {++ if (!row) return null;++ const live = row.state === 'queued' || row.state === 'running';++ return {++ id: row.id,++ object: 'tools.job',++ service: o.service,++ type: row.type,++ type_version: row.type_version,++ state: row.state,++ progress: { percent: row.progress == null ? null : Math.round(row.progress * 10) / 10, message: row.progress_message || null },++ attempts: row.attempts,++ max_attempts: row.max_attempts,++ cancel_requested: !!row.cancel_requested,++ created_at: iso(row.created_at),++ started_at: iso(row.started_at),++ finished_at: iso(row.finished_at),++ expires_at: iso(row.expires_at),++ result: publicResult(store.parse(row.result_json, null), row.id),++ error: store.parse(row.error_json, null),++ retryable: !!row.retryable,++ links: {++ self: `/api/v1/jobs/${row.id}`,++ events: `/api/v1/jobs/${row.id}/events`,++ cancel: live ? `/api/v1/jobs/${row.id}` : null,++ },++ };++ }++++ // ── Events ─────────────────────────────────────────────────++ function emit(id, event) {++ if (stopped) return;++ const data = view(store.get(id));++ if (!data) return;++ const seq = store.appendEvent(id, event, data);++ const set = listeners.get(id);++ if (set) for (const fn of [...set]) { try { fn({ seq, event, data }); } catch (err) { log.error('[Jobs] listener error:', err.message); } }++ }++ function subscribe(id, fn) {++ if (!listeners.has(id)) listeners.set(id, new Set());++ listeners.get(id).add(fn);++ return () => { const s = listeners.get(id); if (s) { s.delete(fn); if (!s.size) listeners.delete(id); } };++ }++++ // ── Submit ─────────────────────────────────────────────────++ /**++ * submit({ owner, type, input, files: [{ path | buffer, name, mime, size }], idempotencyKey, ttlMs })++ * → { job (row), replayed }. Temp input files are consumed (moved) on accept and deleted on replay.++ */++ async function submit({ owner, type, input = {}, files = [], idempotencyKey = null, ttlMs = HOUR }) {++ if (stopped) throw new JobError(503, 'tools.job.unavailable', 'The job system is shutting down');++ const discard = () => Promise.all(files.map(f => (f.path ? fsp.unlink(f.path).catch(() => {}) : null)));++ try {++ if (!owner) throw new JobError(401, 'tools.job.no_owner', 'No owner for this job');++ const def = types.get(type);++ if (!def) throw new JobError(400, 'tools.job.unknown_type', `Unknown job type "${type}". Known: ${[...types.keys()].join(', ')}`);++ if (input == null || typeof input !== 'object' || Array.isArray(input)) throw new JobError(400, 'tools.job.invalid', 'input must be a JSON object');++ if (JSON.stringify(input).length > 16384) throw new JobError(400, 'tools.job.invalid', 'input is larger than 16 KB');++ if (files.length < def.minFiles) throw new JobError(400, 'tools.job.invalid', def.minFiles === 1 ? 'This job needs a file' : `This job needs at least ${def.minFiles} files`);++ if (files.length > def.maxFiles) throw new JobError(400, 'tools.job.invalid', `At most ${def.maxFiles} file(s) for this job`);++ const bad = def.validate(input, files);++ if (bad) throw new JobError(400, 'tools.job.invalid', bad);++ if (idempotencyKey != null && !/^[\x21-\x7e]{8,200}$/.test(String(idempotencyKey))) throw new JobError(400, 'tools.job.invalid', 'Idempotency-Key must be 8-200 printable characters');++++ // Hash the request so a reused key with a different request is caught.++ const fileHashes = [];++ for (const f of files) fileHashes.push(f.buffer ? crypto.createHash('sha256').update(f.buffer).digest('hex') : await sha256File(f.path));++ const requestHash = crypto.createHash('sha256').update(canonical({ type, input, files: fileHashes })).digest('hex');++++ if (idempotencyKey != null) {++ const prior = store.byIdempotencyKey(owner, String(idempotencyKey));++ if (prior) {++ await discard();++ if (prior.request_hash !== requestHash) throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request', { job_id: prior.id });++ return { job: prior, replayed: true };++ }++ }++ if (store.activeForOwner(owner) >= maxActivePerOwner) throw new JobError(429, 'tools.job.too_many_active', `At most ${maxActivePerOwner} unfinished jobs at a time; wait for one to finish or cancel one`);++++ const id = `job_${contracts.ids.ulid()}`;++ const inDir = path.join(jobDir(id), 'in');++ await fsp.mkdir(inDir, { recursive: true });++ const stored = [];++ for (let i = 0; i < files.length; i++) {++ const f = files[i];++ const dest = path.join(inDir, `${i}-${SAFE_NAME(f.name)}`);++ if (f.buffer) await fsp.writeFile(dest, f.buffer); else await moveFile(f.path, dest);++ stored.push({ name: String(f.name || `file-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size: f.size != null ? f.size : fs.statSync(dest).size, sha256: fileHashes[i], _path: dest });++ }++ try {++ store.transaction(() => {++ store.insert({++ id, type, type_version: def.version, owner, input_json: JSON.stringify(input), files_json: JSON.stringify(stored),++ idempotency_key: idempotencyKey == null ? null : String(idempotencyKey), request_hash: requestHash,++ max_attempts: def.maxAttempts, ttl_ms: ttlMs, now: Date.now(),++ });++ })();++ } catch (err) {++ await fsp.rm(jobDir(id), { recursive: true, force: true });++ // Two submits with one key raced: the other one won, so this is a replay of it.++ if (/UNIQUE/.test(err.message) && idempotencyKey != null) {++ const prior = store.byIdempotencyKey(owner, String(idempotencyKey));++ if (prior && prior.request_hash === requestHash) return { job: prior, replayed: true };++ throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request');++ }++ throw err;++ }++ emit(id, 'job.queued');++ kick();++ return { job: store.get(id), replayed: false };++ } catch (err) {++ await discard();++ throw err;++ }++ }++++ // ── Scheduling ─────────────────────────────────────────────++ function kick() {++ if (!started || stopped || kicking) return;++ kicking = true;++ try {++ while (active.size < concurrency) {++ const [row] = store.nextQueued(1);++ if (!row) break;++ if (!store.claim(row.id)) continue;++ const entry = { ctrl: new AbortController(), reason: null, last: { pct: -1, msg: null, at: 0 } };++ active.set(row.id, entry);++ emit(row.id, 'job.running');++ setImmediate(() => execute(row.id, entry));++ }++ } finally { kicking = false; }++ }++++ function progressFn(id, entry) {++ return (percent, message) => {++ if (stopped || entry.reason) return;++ const pct = Math.max(0, Math.min(100, Number(percent) || 0));++ const msg = message == null ? entry.last.msg : String(message).slice(0, 200);++ const now = Date.now();++ if (msg === entry.last.msg && (Math.abs(pct - entry.last.pct) < 1 || now - entry.last.at < throttleMs)) return;++ entry.last = { pct, msg, at: now };++ if (store.setProgress(id, pct, msg, now)) emit(id, 'job.progress');++ };++ }++++ async function execute(id, entry) {++ const row = stopped ? null : store.get(id);++ if (!row) { active.delete(id); return; }++ const def = types.get(row.type);++ const outDir = path.join(jobDir(row.id), 'out');++ let outcome;++ const timer = setTimeout(() => { if (!entry.reason) { entry.reason = 'timeout'; entry.ctrl.abort(new Error('timeout')); } }, def.timeoutMs);++ if (timer.unref) timer.unref();++ try {++ await fsp.mkdir(outDir, { recursive: true });++ const files = store.parse(row.files_json, []).map(f => ({ name: f.name, mime: f.mime, size: f.size, sha256: f.sha256, path: f._path }));++ const out = await def.run({++ job: { id: row.id, type: row.type, attempt: row.attempts, owner: row.owner },++ input: store.parse(row.input_json, {}), files, outDir,++ signal: entry.ctrl.signal,++ progress: progressFn(row.id, entry),++ });++ if (stopped) return;++ if (entry.reason) throw new Error(entry.reason);++ const result = await storeResult(row, out || {}, entry);++ if (stopped) return;++ if (entry.reason) { await dropResult(result); throw new Error(entry.reason); }++ outcome = { state: 'succeeded', result };++ } catch (err) {++ if (stopped) return;++ const fresh = store.get(row.id) || row;++ if (entry.reason === 'cancel' || fresh.cancel_requested) outcome = { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') };++ else if (entry.reason === 'timeout') outcome = { state: 'failed', error: problem(504, 'tools.job.timeout', `The job ran longer than ${Math.round(def.timeoutMs / 1000)} s`), retryable: true };++ else {++ if (!err.expose) log.error(`[Jobs] ${row.type} ${row.id} failed:`, err.message);++ // Only codes a tool chose on purpose (tools.…) are passed on; errno codes and the like are not contracts.++ const code = /^tools\.[a-z0-9_.]+$/.test(String(err.code || '')) ? err.code : 'tools.job.failed';++ outcome = { state: 'failed', error: problem(Number.isInteger(err.status) ? err.status : 422, code, scrub(err.message) || 'The tool could not process this input'), retryable: !!err.retryable };++ }++ } finally {++ clearTimeout(timer);++ }++ try {++ store.transaction(() => store.finish(row.id, outcome))();++ emit(row.id, EVENT_FOR[outcome.state]);++ } finally {++ active.delete(row.id);++ fsp.rm(path.join(jobDir(row.id), 'in'), { recursive: true, force: true }).catch(() => {});++ if (outcome.state !== 'succeeded') fsp.rm(outDir, { recursive: true, force: true }).catch(() => {});++ kick();++ }++ }++++ /** Output files → Media objects (when configured) or kept under <job>/out for /files/:n. */++ async function storeResult(row, out, entry) {++ const outDir = path.join(jobDir(row.id), 'out');++ const files = [];++ const list = Array.isArray(out.files) ? out.files : [];++ for (let i = 0; i < list.length; i++) {++ const f = list[i];++ let local = path.resolve(f.path);++ if (path.dirname(local) !== outDir) {++ const dest = path.join(outDir, `${i}-${SAFE_NAME(f.name)}`);++ await moveFile(local, dest);++ local = dest;++ }++ const size = fs.statSync(local).size;++ const sha256 = await sha256File(local);++ const rec = { name: String(f.name || `result-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size, sha256, storage: 'local', media: null, _path: local };++ if (media && !entry.reason) {++ try {++ const progress = progressFn(row.id, entry);++ progress(99, 'Storing the result');++ rec.media = await media.upload({ path: local, name: rec.name, mime: rec.mime, size, sha256, owner: row.owner, jobId: row.id, service: o.service, type: row.type });++ rec.storage = 'media';++ delete rec._path;++ await fsp.unlink(local).catch(() => {});++ } catch (err) {++ // The result is still here; say where it is rather than pretending it reached Media.++ log.error(`[Jobs] Media upload failed for ${row.id}:`, err.message);++ rec.media_error = 'Could not store the result in OpenVibe.Media; it is kept on this server until the job expires';++ }++ }++ files.push(rec);++ }++ return { files, data: out.data && typeof out.data === 'object' ? out.data : {} };++ }++++ async function dropResult(result) {++ for (const f of (result && result.files) || []) {++ if (f._path) await fsp.unlink(f._path).catch(() => {});++ if (f.media && media) await media.remove(f.media.media_id).catch(() => {});++ }++ }++++ // ── Cancel ─────────────────────────────────────────────────++ /** → { job, changed } — queued jobs end at once; running ones are signalled and end as cancelled. */++ function cancel(id) {++ const row = store.get(id);++ if (!row) return null;++ if (TERMINAL.includes(row.state)) return { job: row, changed: false };++ if (row.state === 'queued') {++ const done = store.transaction(() => store.requestCancel(id) && store.finish(id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled before it started') }))();++ if (done) {++ emit(id, 'job.cancelled');++ fsp.rm(jobDir(id), { recursive: true, force: true }).catch(() => {});++ }++ return { job: store.get(id), changed: done };++ }++ store.requestCancel(id);++ const entry = active.get(id);++ if (entry && !entry.reason) { entry.reason = 'cancel'; entry.ctrl.abort(new Error('cancelled')); }++ emit(id, 'job.cancel_requested');++ return { job: store.get(id), changed: true };++ }++++ // ── Boot recovery, pruning ─────────────────────────────────++ function recover() {++ let requeued = 0, failed = 0;++ for (const row of store.running()) {++ const def = types.get(row.type);++ if (row.cancel_requested) {++ store.finish(row.id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') });++ emit(row.id, 'job.cancelled');++ } else if (!def) {++ store.finish(row.id, { state: 'failed', error: problem(500, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`) });++ emit(row.id, 'job.failed'); failed++;++ } else if (def.onRestart === 'requeue' && row.attempts < row.max_attempts) {++ store.requeue(row.id);++ emit(row.id, 'job.queued'); requeued++;++ } else {++ const retryable = def.onRestart === 'fail';++ store.finish(row.id, {++ state: 'failed', retryable,++ error: problem(503, 'tools.job.interrupted', retryable ? 'The service restarted while this job was running; submit it again' : `The service restarted during each of ${row.attempts} attempts`),++ });++ emit(row.id, 'job.failed'); failed++;++ }++ fs.rmSync(path.join(jobDir(row.id), 'out'), { recursive: true, force: true });++ }++ // Directories without a row: a crash between moving the upload and recording the job.++ // That job was never accepted (no 202 went out), so its files have no owner.++ for (const name of fs.readdirSync(root)) {++ if (/^job_[0-9A-Z]{26}$/.test(name) && !store.get(name)) fs.rmSync(path.join(root, name), { recursive: true, force: true });++ }++ return { requeued, failed };++ }++++ async function prune(now = Date.now()) {++ let n = 0;++ for (const row of store.expired(now)) {++ const result = store.parse(row.result_json, null);++ if (media && result) for (const f of result.files || []) if (f.media && f.media.media_id) await media.remove(f.media.media_id).catch(err => log.warn(`[Jobs] could not delete ${f.media.media_id}:`, err.message));++ await fsp.rm(jobDir(row.id), { recursive: true, force: true }).catch(() => {});++ store.remove(row.id);++ listeners.delete(row.id);++ n++;++ }++ return n;++ }++++ function start() {++ if (started) return api;++ started = true;++ const r = recover();++ if (r.requeued || r.failed) log.log(`[Jobs] ${o.service}: ${r.requeued} job(s) re-queued, ${r.failed} failed after restart`);++ prune().catch(() => {});++ pruneTimer = setInterval(() => prune().catch(err => log.error('[Jobs] prune:', err.message)), o.pruneIntervalMs || 5 * 60 * 1000);++ if (pruneTimer.unref) pruneTimer.unref();++ kick();++ return api;++ }++++ /**++ * Stop. Running jobs are left as they are in the database ('running'), which is exactly what a++ * crash or a restart leaves behind; the next start() recovers them per type.++ */++ function stop() {++ stopped = true;++ if (pruneTimer) clearInterval(pruneTimer);++ for (const entry of active.values()) { entry.reason = entry.reason || 'shutdown'; entry.ctrl.abort(new Error('shutdown')); }++ listeners.clear();++ for (const fn of [...stopHooks]) { try { fn(); } catch { /* best effort */ } }++ stopHooks.clear();++ }++ /** Run fn when the system stops; → a function that removes it. */++ function onStop(fn) { stopHooks.add(fn); return () => stopHooks.delete(fn); }++++ /** Where a result file's bytes are: { local: path } or { media: MediaRef }. */++ function resultFile(row, n) {++ const result = store.parse(row.result_json, null);++ const f = result && result.files && result.files[n];++ if (!f) return null;++ return { name: f.name, mime: f.mime, size: f.size, storage: f.storage, media: f.media || null, path: f._path || null };++ }++++ const api = {++ define, submit, cancel, subscribe, start, stop, onStop, prune, recover, view, resultFile,++ get: (id) => store.get(id),++ eventsAfter: (id, seq) => store.eventsAfter(id, seq),++ lastSeq: (id) => store.lastSeq(id),++ types: () => [...types.keys()],++ media,++ stats: () => ({ running: active.size, concurrency, ...store.counts(), results: media ? 'media' : 'local' }),++ store,++ JobError,++ };++ return api;++}++++module.exports = { createJobSystem, JobError, canonical, TERMINAL };nnew file mode 100644@@@ -0,0 +1,14 @@++'use strict';++// apps/_shared has no node_modules of its own (the apps require it by relative path), so its tests++// borrow the dependencies of the first app that has them installed.++const path = require('path');++const APPS = ['img', 'docs', 'audio', 'yt', 'gateway'].map(a => path.join(__dirname, '..', '..', a));++++function dep(name) {++ for (const dir of APPS) {++ try { return require(require.resolve(name, { paths: [dir] })); } catch { /* next app */ }++ }++ throw new Error(`${name} is not installed in any app (run npm run install:all)`);++}++++module.exports = { dep };nnew file mode 100644@@@ -0,0 +1,390 @@++'use strict';++// Wave 11 exit criteria for the shared job runtime, end to end over HTTP:++// an accepted job survives a restart (DB and app closed and reopened), running jobs are re-queued++// or failed-retryable per type, reattach by id, cancel (queued and running), idempotency keys,++// owner scoping, SSE resume with Last-Event-ID, bounded concurrency, pruning, and results stored++// as Media objects through Media's v2 object API (with the local fallback).++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const crypto = require('crypto');++const { dep } = require('./deps');++++const express = dep('express');++const cookieParser = dep('cookie-parser');++const Database = dep('better-sqlite3');++const contracts = dep('openvibe-contracts');++const jobs = require('../jobs');++++const ISSUER = 'https://openvibe.network';++const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });++const USER_A = 'usr_01JAAAAAAAAAAAAAAAAAAAAAAA';++const USER_B = 'usr_01JBBBBBBBBBBBBBBBBBBBBBBB';++const sleep = (ms) => new Promise(r => setTimeout(r, ms));++++async function until(fn, what, ms = 5000) {++ const t0 = Date.now();++ for (;;) {++ const v = await fn();++ if (v) return v;++ if (Date.now() - t0 > ms) throw new Error(`timed out waiting for ${what}`);++ await sleep(20);++ }++}++++// ── Job types used by the tests ──────────────────────────────++const gates = new Map(); // name → { promise, open() }++function gate(name) {++ if (!gates.has(name)) { let open; const promise = new Promise(r => { open = r; }); gates.set(name, { promise, open }); }++ return gates.get(name);++}++function define(system) {++ // Upper-cases its input file (or input.text); waits on input.gate when given. Re-queued after a restart.++ system.define({++ type: 'test.upper', maxFiles: 1, maxAttempts: 3,++ validate: (input) => (input.text != null && typeof input.text !== 'string' ? 'text must be a string' : null),++ async run({ input, files, outDir, progress, signal }) {++ progress(10, 'Reading');++ if (input.gate) await Promise.race([gate(input.gate).promise, new Promise((_, rej) => signal.addEventListener('abort', () => rej(new Error('aborted'))))]);++ const text = files.length ? fs.readFileSync(files[0].path, 'utf8') : input.text || '';++ progress(60, 'Writing');++ const out = path.join(outDir, 'upper.txt');++ fs.writeFileSync(out, text.toUpperCase());++ return { files: [{ path: out, name: 'upper.txt', mime: 'text/plain' }], data: { length: text.length } };++ },++ });++ // Not safe to run twice: after a restart it fails with retryable = true.++ system.define({++ type: 'test.fragile', onRestart: 'fail',++ async run({ input, signal }) {++ await Promise.race([gate(input.gate).promise, new Promise((_, rej) => signal.addEventListener('abort', () => rej(new Error('aborted'))))]);++ return { data: { ok: true } };++ },++ });++ system.define({ type: 'test.boom', async run() { throw new Error('This input at /srv/secret/path/x.png is not an image'); } });++}++++// ── A fake OpenVibe.Media (object API v2) ────────────────────++function fakeMedia() {++ const objects = new Map();++ const app = express();++ const state = { failInit: false, deleted: [] };++ app.use(express.json());++ const authed = (req, res, next) => (req.headers.authorization === 'Bearer media-token' ? next() : res.status(401).json({ error: 'no' }));++ app.post('/api/v2/tools/objects', authed, (req, res) => {++ if (state.failInit) return res.status(503).json({ code: 'down' });++ const id = `med_${contracts.ids.ulid()}`;++ objects.set(id, { id, meta: req.body, subject: req.headers['x-ov-subject'] || null, bytes: null, status: 'uploading' });++ res.status(201).json({ id, upload: { method: 'PUT', url: `https://openvibe.media/api/v2/tools/objects/${id}/content?token=tok-${id}` } });++ });++ app.put('/api/v2/tools/objects/:id/content', (req, res) => {++ const o = objects.get(req.params.id);++ if (!o || req.query.token !== `tok-${o.id}`) return res.status(401).json({ error: 'bad token' });++ const chunks = [];++ req.on('data', c => chunks.push(c)).on('end', () => { o.bytes = Buffer.concat(chunks); res.json({ id: o.id, size_bytes: o.bytes.length }); });++ });++ app.post('/api/v2/tools/objects/:id/complete', authed, (req, res) => {++ const o = objects.get(req.params.id);++ const sha = crypto.createHash('sha256').update(o.bytes).digest('hex');++ if (req.body.content_hash !== sha) return res.status(422).json({ code: 'media.object.hash_mismatch' });++ o.status = 'ready';++ res.json({ id: o.id, lifecycle_status: 'ready' });++ });++ app.get('/api/v2/tools/objects/:id/download', authed, (req, res) => res.json({ url: `https://openvibe.media/o/${req.params.id}?exp=1&sig=s`, expires_at: 'x', public: false }));++ app.delete('/api/v2/tools/objects/:id', authed, (req, res) => { state.deleted.push(req.params.id); objects.delete(req.params.id); res.json({ ok: true }); });++ app.get('/o/:id', (req, res) => { const o = objects.get(req.params.id); if (!o || req.query.sig !== 's') return res.status(404).end(); res.type('text/plain').send(o.bytes); });++ return { app, objects, state };++}++const fakeTokens = { authHeaders: async () => ({ Authorization: 'Bearer media-token' }), invalidate() {} };++++// ── A satellite: express + the job routes over one SQLite file ──++function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10 } = {}) {++ fs.mkdirSync(dir, { recursive: true });++ const db = new Database(path.join(dir, 'jobs.db'));++ const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir: dir, concurrency, media, maxActivePerOwner, progressThrottleMs: 0, log: { log() {}, warn() {}, error() {} } });++ define(system);++ const app = express();++ app.use(cookieParser());++ app.use(express.json());++ app.use((req, _res, next) => { if (req.headers['x-test-user']) req.user = { sub: 7, subject_id: req.headers['x-test-user'] }; next(); });++ const multer = dep('multer');++ const upload = multer({ dest: path.join(dir, 'tmp') });++ const resolveOwner = jobs.createOwnerResolver({ contracts, getPublicKey: () => publicKey, issuer: ISSUER, secureCookie: false });++ jobs.mountJobRoutes(app, { system, contracts, resolveOwner, receive: upload.any() });++ system.start();++ return new Promise(resolve => {++ const server = app.listen(0, '127.0.0.1', () => resolve({++ system, db, base: `http://127.0.0.1:${server.address().port}`,++ close: () => new Promise(r => { system.stop(); server.closeAllConnections(); server.close(() => { db.close(); r(); }); }),++ }));++ });++}++++/** A browser-ish client with its own cookie jar (the jobs session cookie). */++function client(base, { user, bearer } = {}) {++ let cookie = '';++ return async function call(p, init = {}) {++ const headers = { ...(init.headers || {}) };++ if (cookie) headers.cookie = cookie;++ if (user) headers['x-test-user'] = user;++ if (bearer) headers.authorization = `Bearer ${bearer}`;++ const res = await fetch((typeof base === 'function' ? base() : base) + p, { ...init, headers, redirect: 'manual' });++ const set = res.headers.get('set-cookie');++ if (set) cookie = set.split(';')[0];++ const type = res.headers.get('content-type') || '';++ const body = /json/.test(type) ? await res.json() : await res.text();++ return { status: res.status, headers: res.headers, body };++ };++}++const json = (body, headers = {}) => ({ method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body: JSON.stringify(body) });++++/** Read an SSE stream until `stop(events)` or the server ends it. */++async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}) {++ const ac = new AbortController();++ const timer = setTimeout(() => ac.abort(), ms);++ const res = await fetch(url, { headers, signal: ac.signal });++ const events = [];++ if (res.status !== 200) { clearTimeout(timer); return { status: res.status, events }; }++ const dec = new TextDecoder();++ let buf = '';++ try {++ for await (const chunk of res.body) {++ buf += dec.decode(chunk, { stream: true });++ let i;++ while ((i = buf.indexOf('\n\n')) >= 0) {++ const frame = buf.slice(0, i); buf = buf.slice(i + 2);++ const e = {};++ for (const line of frame.split('\n')) { const m = /^(id|event|data): (.*)$/.exec(line); if (m) e[m[1]] = m[2]; }++ if (e.data) { e.data = JSON.parse(e.data); e.id = Number(e.id); events.push(e); }++ }++ if (stop(events)) break;++ }++ } catch (err) { if (err.name !== 'AbortError') throw err; }++ clearTimeout(timer); ac.abort();++ return { status: res.status, events };++}++++(async () => {++ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-jobs-'));++ try {++ // ── Submit, reattach by id, result download ─────────────++ let s = await satellite(path.join(root, 'a'));++ const alice = client(s.base, { user: USER_A });++ let r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'hello' } }));++ assert.strictEqual(r.status, 202, 'accepted');++ assert.match(r.body.id, /^job_[0-9A-Z]{26}$/);++ assert.strictEqual(r.headers.get('location'), `/api/v1/jobs/${r.body.id}`);++ const first = r.body.id;++ const done = await until(async () => { const g = await alice(`/api/v1/jobs/${first}`); return g.body.state === 'succeeded' && g.body; }, 'first job');++ assert.strictEqual(done.result.data.length, 5);++ assert.strictEqual(done.result.files[0].storage, 'local');++ assert.ok(done.expires_at && done.finished_at, 'a finished job says when it expires');++ r = await alice(done.result.files[0].url);++ assert.strictEqual(r.body, 'HELLO');++ assert.match(r.headers.get('content-disposition'), /^attachment; filename="upper.txt"/);++++ // Multipart with a file (the way the UI submits).++ const fd = new FormData();++ fd.append('type', 'test.upper'); fd.append('input', '{}');++ fd.append('file', new Blob(['from a file']), 'in.txt');++ r = await alice('/api/v1/jobs', { method: 'POST', body: fd });++ assert.strictEqual(r.status, 202);++ const multi = await until(async () => { const g = await alice(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'succeeded' && g.body; }, 'multipart job');++ assert.strictEqual((await alice(multi.result.files[0].url)).body, 'FROM A FILE');++++ // Validation and tool failures are problem+json.++ r = await alice('/api/v1/jobs', json({ type: 'nope', input: {} }));++ assert.strictEqual(r.status, 400); assert.strictEqual(r.body.code, 'tools.job.unknown_type'); assert.match(r.headers.get('content-type'), /problem\+json/);++ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 5 } }));++ assert.strictEqual(r.body.code, 'tools.job.invalid');++ r = await alice('/api/v1/jobs', json({ type: 'test.boom' }));++ const boom = await until(async () => { const g = await alice(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'failed' && g.body; }, 'failed job');++ assert.strictEqual(boom.error.code, 'tools.job.failed');++ assert.ok(!/srv\/secret/.test(boom.error.detail), 'server paths are not shown to people');++++ // ── Idempotency ─────────────────────────────────────────++ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'once' } }, { 'idempotency-key': 'key-000000001' }));++ const once = r.body.id;++ assert.strictEqual(r.status, 202);++ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'once' } }, { 'idempotency-key': 'key-000000001' }));++ assert.strictEqual(r.status, 200); assert.strictEqual(r.body.id, once, 'same key, same request → same job');++ assert.strictEqual(r.headers.get('idempotent-replayed'), 'true');++ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'twice' } }, { 'idempotency-key': 'key-000000001' }));++ assert.strictEqual(r.status, 409); assert.strictEqual(r.body.code, 'tools.job.idempotency_conflict');++ r = await client(s.base, { user: USER_B })('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'once' } }, { 'idempotency-key': 'key-000000001' }));++ assert.strictEqual(r.status, 202); assert.notStrictEqual(r.body.id, once, 'keys are per owner');++++ // ── Owner scoping ───────────────────────────────────────++ const bob = client(s.base, { user: USER_B });++ const guest1 = client(s.base), guest2 = client(s.base), nobody = client(s.base);++ for (const who of [bob, guest1, nobody]) {++ assert.strictEqual((await who(`/api/v1/jobs/${first}`)).status, 404);++ assert.strictEqual((await who(`/api/v1/jobs/${first}`, { method: 'DELETE' })).status, 404);++ assert.strictEqual((await who(`/api/v1/jobs/${first}/events`)).status, 404);++ assert.strictEqual((await who(`/api/v1/jobs/${first}/files/0`)).status, 404);++ }++ r = await guest1('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'guest' } }));++ assert.strictEqual(r.status, 202);++ const guestJob = r.body.id;++ assert.strictEqual((await guest1(`/api/v1/jobs/${guestJob}`)).status, 200, 'the session that made it can reattach');++ assert.strictEqual((await guest2(`/api/v1/jobs/${guestJob}`)).status, 404, 'another browser cannot');++ assert.strictEqual((await alice(`/api/v1/jobs/${guestJob}`)).status, 404, 'nor can a signed-in stranger');++ assert.ok(s.db.prepare('SELECT owner FROM tool_jobs').all().every(x => /^(user:usr_[0-9A-Z]{26}|session:[0-9a-f]{40})$/.test(x.owner)), 'owners are canonical subjects or hashed sessions, never raw cookies');++++ // ── SSE: events carry ids; Last-Event-ID resumes after them ──++ const all = await readSse(`${s.base}/api/v1/jobs/${first}/events`, { headers: { 'x-test-user': USER_A } });++ assert.strictEqual(all.status, 200);++ assert.deepStrictEqual(all.events.map(e => e.event), ['job.queued', 'job.running', 'job.progress', 'job.progress', 'job.succeeded']);++ assert.ok(all.events.every((e, i) => i === 0 || e.id > all.events[i - 1].id), 'ids increase');++ const resumed = await readSse(`${s.base}/api/v1/jobs/${first}/events`, { headers: { 'x-test-user': USER_A, 'last-event-id': String(all.events[2].id) } });++ assert.deepStrictEqual(resumed.events.map(e => e.id), all.events.slice(3).map(e => e.id), 'only what came after Last-Event-ID');++ const caughtUp = await readSse(`${s.base}/api/v1/jobs/${first}/events?last_event_id=${all.events[4].id}`, { headers: { 'x-test-user': USER_A } });++ assert.strictEqual(caughtUp.status, 204, 'finished and nothing new: 204 stops EventSource reconnecting');++ // Live: a stream opened on a running job sees the rest as it happens.++ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'live', gate: 'live' } }));++ const liveId = r.body.id;++ await until(async () => (await alice(`/api/v1/jobs/${liveId}`)).body.state === 'running', 'live job running');++ const streaming = readSse(`${s.base}/api/v1/jobs/${liveId}/events`, { headers: { 'x-test-user': USER_A } });++ await sleep(100);++ gate('live').open();++ const live = await streaming;++ assert.strictEqual(live.events[live.events.length - 1].event, 'job.succeeded');++ assert.strictEqual(live.events[live.events.length - 1].data.result.files.length, 1, 'the terminal event carries the result');++++ // ── Cancel ──────────────────────────────────────────────++ await s.close();++ s = await satellite(path.join(root, 'b'), { concurrency: 1 });++ const carol = client(s.base, { user: USER_A });++ const blocker = (await carol('/api/v1/jobs', json({ type: 'test.upper', input: { gate: 'block' } }))).body.id;++ const waiting = (await carol('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'later' } }))).body.id;++ await until(async () => (await carol(`/api/v1/jobs/${blocker}`)).body.state === 'running', 'blocker running');++ assert.strictEqual((await carol(`/api/v1/jobs/${waiting}`)).body.state, 'queued', 'concurrency 1: the second job waits');++ r = await carol(`/api/v1/jobs/${waiting}`, { method: 'DELETE' });++ assert.strictEqual(r.status, 200); assert.strictEqual(r.body.state, 'cancelled', 'a queued job is cancelled at once');++ r = await carol(`/api/v1/jobs/${blocker}`, { method: 'DELETE' });++ assert.strictEqual(r.status, 202); assert.strictEqual(r.body.cancel_requested, true, 'a running job is signalled');++ const cancelled = await until(async () => { const g = await carol(`/api/v1/jobs/${blocker}`); return g.body.state === 'cancelled' && g.body; }, 'running job cancelled');++ assert.strictEqual(cancelled.error.code, 'tools.job.cancelled');++ assert.strictEqual((await carol(`/api/v1/jobs/${blocker}`, { method: 'DELETE' })).status, 200, 'cancelling twice is fine');++ const fin = (await carol('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'x' } }))).body.id;++ await until(async () => (await carol(`/api/v1/jobs/${fin}`)).body.state === 'succeeded', 'job to finish');++ r = await carol(`/api/v1/jobs/${fin}`, { method: 'DELETE' });++ assert.strictEqual(r.status, 409); assert.strictEqual(r.body.code, 'tools.job.already_finished');++++ // ── Bounded concurrency and the per-owner cap ───────────++ await s.close();++ s = await satellite(path.join(root, 'c'), { concurrency: 2, maxActivePerOwner: 4 });++ const dave = client(s.base, { user: USER_B });++ const ids = [];++ for (let i = 0; i < 4; i++) ids.push((await dave('/api/v1/jobs', json({ type: 'test.upper', input: { gate: 'pool' } }))).body.id);++ await until(async () => s.system.stats().running === 2, 'two running');++ await sleep(50);++ assert.strictEqual(s.system.stats().running, 2, 'never more than the concurrency limit');++ assert.strictEqual(s.system.stats().queued, 2);++ r = await dave('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'one too many' } }));++ assert.strictEqual(r.status, 429); assert.strictEqual(r.body.code, 'tools.job.too_many_active');++ gate('pool').open();++ await until(async () => s.system.stats().succeeded === 4, 'pool drained');++++ // ── Restart: an accepted job is not lost ────────────────++ await s.close();++ const dir = path.join(root, 'd');++ s = await satellite(dir, { concurrency: 0 }); // accepts, runs nothing: every job stays queued++ const erin = client(() => s.base, { user: USER_A }); // follows the satellite across restarts++ const fd2 = new FormData();++ fd2.append('type', 'test.upper'); fd2.append('input', '{}'); fd2.append('file', new Blob(['survives']), 'keep.txt');++ r = await erin('/api/v1/jobs', { method: 'POST', body: fd2 });++ assert.strictEqual(r.status, 202);++ const accepted = r.body.id;++ await s.close(); // DB and app closed: the process is gone++ s = await satellite(dir, { concurrency: 2 }); // and back++ const back = await until(async () => { const g = await erin(`/api/v1/jobs/${accepted}`); return g.body.state === 'succeeded' && g.body; }, 'accepted job after restart');++ assert.strictEqual((await erin(back.result.files[0].url)).body, 'SURVIVES', 'its uploaded input survived too');++++ // Running when the process died: re-queued (test.upper) or failed-retryable (test.fragile).++ const again = (await erin('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'rerun', gate: 'never-1' } }))).body.id;++ const fragile = (await erin('/api/v1/jobs', json({ type: 'test.fragile', input: { gate: 'never-2' } }))).body.id;++ await until(async () => (await erin(`/api/v1/jobs/${again}`)).body.state === 'running' && (await erin(`/api/v1/jobs/${fragile}`)).body.state === 'running', 'both running');++ await s.close();++ gates.get('never-1').open(); // the gate is open after the restart++ s = await satellite(dir, { concurrency: 2 });++ const rerun = await until(async () => { const g = await erin(`/api/v1/jobs/${again}`); return g.body.state === 'succeeded' && g.body; }, 're-queued job');++ assert.strictEqual(rerun.attempts, 2, 'the interrupted attempt counts');++ const broken = (await erin(`/api/v1/jobs/${fragile}`)).body;++ assert.strictEqual(broken.state, 'failed');++ assert.strictEqual(broken.retryable, true);++ assert.strictEqual(broken.error.code, 'tools.job.interrupted');++ const history = await readSse(`${s.base}/api/v1/jobs/${again}/events`, { headers: { 'x-test-user': USER_A } });++ assert.deepStrictEqual(history.events.map(e => e.event).filter(e => e !== 'job.progress'), ['job.queued', 'job.running', 'job.queued', 'job.running', 'job.succeeded'], 'the event log spans the restart');++++ // ── Pruning ─────────────────────────────────────────────++ const pruned = await s.system.prune(Date.now() + 25 * 60 * 60 * 1000);++ assert.ok(pruned >= 3);++ assert.strictEqual((await erin(`/api/v1/jobs/${accepted}`)).status, 404, 'expired jobs are gone');++ assert.ok(!fs.existsSync(path.join(dir, 'jobs', accepted)), 'with their files');++ assert.strictEqual(s.db.prepare('SELECT COUNT(*) AS n FROM tool_job_events WHERE job_id = ?').get(accepted).n, 0, 'and their events');++ await s.close();++++ // ── Results as Media objects ────────────────────────────++ const fm = fakeMedia();++ const mediaServer = await new Promise(res => { const srv = fm.app.listen(0, '127.0.0.1', () => res(srv)); });++ const media = jobs.createMediaResults({ internalUrl: `http://127.0.0.1:${mediaServer.address().port}`, tokens: fakeTokens });++ s = await satellite(path.join(root, 'e'), { media });++ const frank = client(s.base, { user: USER_B });++ r = await frank('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'stored in media' } }));++ const m = await until(async () => { const g = await frank(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'succeeded' && g.body; }, 'media job');++ const f0 = m.result.files[0];++ assert.strictEqual(f0.storage, 'media');++ assert.match(f0.media.media_id, /^med_[0-9A-Z]{26}$/, 'the result is a Media object reference');++ const obj = fm.objects.get(f0.media.media_id);++ assert.strictEqual(obj.status, 'ready');++ assert.strictEqual(obj.subject, USER_B, 'owned by the person who ran the job');++ assert.strictEqual(obj.meta.visibility, 'private');++ assert.strictEqual(obj.meta.metadata.job_id, m.id);++ assert.ok(!fs.existsSync(path.join(root, 'e', 'jobs', m.id, 'out', '0-upper.txt')), 'no local copy is kept');++ r = await frank(`${f0.url}?inline=1`);++ assert.strictEqual(r.status, 302); assert.match(r.headers.get('location'), /^https:\/\/openvibe\.media\/o\/med_/, 'previews load from Media');++ r = await frank(f0.url);++ assert.strictEqual(r.body, 'STORED IN MEDIA', 'downloads stream through with the right name');++ assert.match(r.headers.get('content-disposition'), /attachment/);++ // Media down: the result stays local and says so.++ fm.state.failInit = true;++ r = await frank('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'fallback' } }));++ const fb = await until(async () => { const g = await frank(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'succeeded' && g.body; }, 'fallback job');++ assert.strictEqual(fb.result.files[0].storage, 'local');++ assert.ok(fb.result.files[0].media_error);++ assert.strictEqual((await frank(fb.result.files[0].url)).body, 'FALLBACK');++ await s.system.prune(Date.now() + 25 * 60 * 60 * 1000);++ assert.ok(fm.state.deleted.includes(f0.media.media_id), 'pruning deletes the Media object');++ await s.close();++ mediaServer.close();++++ // ── Service/app principals (tools.job.* capabilities) ───++ s = await satellite(path.join(root, 'f'));++ const token = (claims) => contracts.serviceAuth.signServiceToken({++ iss: ISSUER, sub: 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC', actor_type: 'app', aud: ['openvibe.tools'],++ cap: ['tools.job.create', 'tools.job.read'], iat: Math.floor(Date.now() / 1000), exp: Math.floor(Date.now() / 1000) + 300, jti: crypto.randomBytes(8).toString('hex'), ...claims,++ }, privateKey);++ const dev = client(s.base, { bearer: token({}) });++ r = await dev('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'sdk' } }));++ assert.strictEqual(r.status, 202, 'an app with tools.job.create can submit');++ assert.strictEqual(r.headers.get('set-cookie'), null, 'principals get no session cookie');++ const devJob = r.body.id;++ await until(async () => (await dev(`/api/v1/jobs/${devJob}`)).body.state === 'succeeded', 'app job');++ assert.strictEqual(s.system.get(devJob).owner, 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC');++ r = await dev(`/api/v1/jobs/${devJob}`, { method: 'DELETE' });++ assert.strictEqual(r.status, 403, 'cancel needs tools.job.cancel');++ r = await client(s.base, { bearer: token({ cap: ['tools.job.read'] }) })('/api/v1/jobs', json({ type: 'test.upper', input: {} }));++ assert.strictEqual(r.status, 403); assert.strictEqual(r.body.code, 'capability.denied');++ r = await client(s.base, { bearer: token({ aud: ['openvibe.media'] }) })('/api/v1/jobs', json({ type: 'test.upper', input: {} }));++ assert.strictEqual(r.status, 401); assert.strictEqual(r.body.code, 'token.wrong_audience');++ r = await client(s.base, { bearer: token({ sub: 'app:app_01JDDDDDDDDDDDDDDDDDDDDDDD' }) })(`/api/v1/jobs/${devJob}`);++ assert.strictEqual(r.status, 404, 'another app cannot read it');++ await s.close();++++ console.log('jobs (shared runtime): all checks passed');++ } finally {++ fs.rmSync(root, { recursive: true, force: true });++ }++})().catch((err) => { console.error(err); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,59 @@++'use strict';++// Start a satellite as its own process (the way systemd does), so a test can kill it outright and++// start it again over the same data directory.++const { spawn } = require('child_process');++const net = require('net');++const path = require('path');++++function freePort() {++ return new Promise((resolve, reject) => {++ const srv = net.createServer();++ srv.unref();++ srv.on('error', reject);++ srv.listen(0, '127.0.0.1', () => { const { port } = srv.address(); srv.close(() => resolve(port)); });++ });++}++++/**++ * @param {string} app 'img' | 'audio' | 'docs' | …++ * @param {object} env extra environment (DATA_DIR, TOOLS_JOBS_CONCURRENCY_…)++ * @returns {Promise<{ base, port, kill(signal), exited: Promise, output: () => string }>}++ */++async function startApp(app, env = {}, port) {++ port = port || await freePort();++ const dir = path.join(__dirname, '..', '..', app);++ let out = '';++ const child = spawn(process.execPath, ['server/index.js'], {++ cwd: dir,++ env: {++ ...process.env,++ NODE_ENV: 'test',++ PORT: String(port),++ HOST: '127.0.0.1',++ // Never reach the real Network from a test: the key fetch fails fast and users are anonymous.++ OV_NETWORK_URL: 'http://127.0.0.1:9',++ OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',++ TOOLS_JOB_RESULTS: 'local',++ ...env,++ },++ stdio: ['ignore', 'pipe', 'pipe'],++ });++ child.stdout.on('data', d => { out += d; });++ child.stderr.on('data', d => { out += d; });++ const exited = new Promise(r => child.on('exit', r));++ const base = `http://127.0.0.1:${port}`;++ const t0 = Date.now();++ for (;;) {++ if (child.exitCode != null) throw new Error(`${app} exited during boot:\n${out}`);++ try { const r = await fetch(`${base}/api/health`); if (r.ok) break; } catch { /* not up yet */ }++ if (Date.now() - t0 > 15000) { child.kill('SIGKILL'); throw new Error(`${app} did not start:\n${out}`); }++ await new Promise(r => setTimeout(r, 100));++ }++ return {++ base, port, exited,++ output: () => out,++ kill(signal = 'SIGKILL') { child.kill(signal); return exited; },++ };++}++++module.exports = { startApp, freePort };@@@ -10,6 +10,7 @@},"license": "MIT","dependencies": {++ "better-sqlite3": "^11.0.0","cookie-parser": "^1.4.7","cors": "^2.8.5","dotenv": "^16.4.7",@@@ -17,10 +18,10 @@"express-rate-limit": "^7.5.0","fluent-ffmpeg": "^2.1.3","helmet": "^8.1.0",-- "better-sqlite3": "^11.0.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0","jsonwebtoken": "^9.0.2","multer": "^1.4.5-lts.1",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.7.0",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0","uuid": "^11.1.0"}}@@@ -338,6 +338,7 @@<script src="https://openvibe.network/shared/footer.js"></script><script src="https://openvibe.network/shared/notification-ui.js"></script><script src="https://openvibe.network/shared/account-switcher.js"></script>++ <script src="/js/ov-jobs.js"></script><script src="/js/app.js"></script></body></html>@@@ -75,6 +75,7 @@} else {selectTool('convert');}++ reattach();}/* ---------- Branding ---------- */@@@ -373,6 +374,22 @@resultPanel.style.display = 'none';processing.style.display = '';++ // Jobs: the work is accepted, followed live (ffmpeg's progress) and survives a reload.++ if (window.OVJobs) {++ const input = {};++ for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;++ try {++ const job = await OVJobs.submit({ type: 'audio.process', input, files: [uploadedFile] });++ OVJobs.remember(job.id);++ followJob(job);++ } catch (err) {++ showToast(err.message, 'error');++ resetUI();++ }++ return;++ }++++ // Without the jobs helper: the synchronous endpoint, as before.try {const res = await fetch('/api/process', { method: 'POST', body: formData });const data = await res.json();@@@ -460,9 +477,76 @@return o;}++ /* ---------- Jobs ---------- */++ let watcher = null;++ let currentJob = null;++++ function processingText(job) {++ const pct = job.progress && job.progress.percent;++ if (job.cancel_requested) return 'Cancelling...';++ if (job.state === 'queued') return 'Waiting for a free slot...';++ const label = (job.progress && job.progress.message) || 'Processing your audio';++ return pct != null ? `${label}... ${Math.round(pct)}%` : `${label}...`;++ }++++ function cancelButton() {++ let btn = processing.querySelector('.job-cancel');++ if (!btn) {++ btn = document.createElement('button');++ btn.type = 'button';++ btn.className = 'btn-secondary job-cancel';++ btn.textContent = 'Cancel';++ btn.addEventListener('click', () => { if (currentJob) OVJobs.cancel(currentJob).catch(() => {}); });++ processing.appendChild(btn);++ }++ return btn;++ }++++ function followJob(job) {++ currentJob = job.id;++ optionsPanel.style.display = 'none';++ uploadZone.style.display = 'none';++ resultPanel.style.display = 'none';++ processing.style.display = '';++ const label = processing.querySelector('p');++ const update = (j) => { if (label) label.textContent = processingText(j); };++ cancelButton().style.display = '';++ update(job);++ if (watcher) watcher.close();++ watcher = OVJobs.watch(job.id, {++ onUpdate: update,++ onDone(done) {++ watcher = null;++ cancelButton().style.display = 'none';++ if (done.state === 'succeeded') { resultId = done.id; return showResult(jobResult(done)); }++ OVJobs.forget();++ if (done.state !== 'cancelled') showToast((done.error && done.error.detail) || 'Processing failed', 'error');++ resetUI();++ },++ onGone() { OVJobs.forget(); resetUI(); },++ });++ }++++ /** A finished job in the shape the synchronous endpoint answers with. */++ function jobResult(job) {++ const f = (job.result && job.result.files && job.result.files[0]) || {};++ return {++ ...(job.result && job.result.data),++ download: { id: job.id, downloadUrl: f.url, previewUrl: f.url ? `${f.url}?inline=1` : null, filename: f.name, expiresIn: OVJobs.expiresIn(job) },++ };++ }++++ /** After a reload: pick the job up again from ?job=<id> or this tab's session. */++ function reattach() {++ const id = window.OVJobs && OVJobs.recall();++ if (!id) return;++ OVJobs.get(id).then(followJob).catch(() => OVJobs.forget());++ }++/* ---------- Result ---------- */function initResult() {anotherBtn.addEventListener('click', () => {++ if (window.OVJobs) OVJobs.forget();clearFile();resetUI();});@@@ -473,8 +557,8 @@resultPanel.style.display = '';const fileId = data.download.id;-- const previewUrl = `/api/preview/${fileId}`;-- const downloadUrl = `/api/download/${fileId}`;++ const previewUrl = data.download.previewUrl || `/api/preview/${fileId}`;++ const downloadUrl = data.download.downloadUrl || `/api/download/${fileId}`;const outputMime = data.output?.mime || '';if (outputMime.startsWith('image/')) {@@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {next();}--module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey };++/** The Network public key (PEM) once loaded; the job routes verify service tokens with it. */++function getPublicKey() { return publicKey; }++++module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey, getPublicKey, ISSUER };@@@ -14,20 +14,25 @@ const path = require('path');const fs = require('fs');const config = require('./config');--const { optionalAuth } = require('./auth');--const { resolveContext } = require('./domain-map');++const auth = require('./auth');++const { optionalAuth } = auth;++const { resolveContext, DOMAIN_MAP } = require('./domain-map');const { getTool, listTools } = require('./tools');const { readMetadata } = require('./tools/metadata');const { uploadSingle, uploadMultiple } = require('./middleware/upload');const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');const retention = require('./retention/manager');const { probe, getDuration, cleanTmp } = require('./tools/ffmpeg-helper');++const { buildOptions, describe, defineJobs } = require('./process');++const { hostGuard, ownHost } = require('../../_shared/host-role');++const jobsRuntime = require('../../_shared/jobs');++const contracts = require('openvibe-contracts');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');const { AnalyticsTracker } = require('openvibe-shared/analytics');const { internalOk } = require('../../_shared/internal-auth');--const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');++const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');@@@ -47,8 +52,8 @@ app.use(helmet({scriptSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://openvibe.network"],styleSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://fonts.gstatic.com"],fontSrc: ["'self'", "https://fonts.gstatic.com", "https://cdnjs.cloudflare.com"],-- imgSrc: ["'self'", "data:", "blob:"],-- mediaSrc: ["'self'", "blob:"],++ imgSrc: ["'self'", "data:", "blob:", jobsRuntime.mediaOrigin()], // job result previews are 302s to Media++ mediaSrc: ["'self'", "blob:", jobsRuntime.mediaOrigin()],connectSrc: ["'self'", "https://openvibe.network", "https://*.openvibe.tools"],workerSrc: ["'self'", "blob:"],scriptSrcAttr: ["'unsafe-inline'"],@@@ -58,6 +63,7 @@ app.use(helmet({}));app.use(cookieParser());app.use(express.json({ limit: '1mb' }));++app.use(contracts.http.middleware()); // traceparent + X-OpenVibe-Request-Id on every response// ── CORS ─────────────────────────────────────────────────────app.use(cors({@@@ -79,9 +85,16 @@ app.use(analytics.middleware());// ── Auth (optional on all routes) ────────────────────────────app.use(optionalAuth);++// ── Hosts ────────────────────────────────────────────────────++// Through the gateway the X-OV-* headers name the tool and its canonical host (a custom domain++// included); aliases the gateway missed are redirected. Hosts this app does not serve go to the++// tools index. API paths are never redirected.++const knowsHost = (h) => !!DOMAIN_MAP[h];++app.use(hostGuard({ knows: knowsHost }));++// ── Attach domain context to every request ───────────────────app.use((req, _res, next) => {-- req.ctx = resolveContext(req.headers.host);++ req.ctx = resolveContext(ownHost(req, knowsHost));next();});@@@ -90,7 +103,7 @@ app.use((req, _res, next) => {// Health checkapp.get('/api/health', (_req, res) => {const stats = retention.getStats();-- res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats });++ res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats, jobs: jobs.stats() });});// Domain context (frontend calls this on load to get branding)@@@ -138,18 +151,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,return res.status(400).json({ error: `Unknown tool: ${toolId}` });}-- // Build options from body params-- const options = { ...req.body };-- delete options.tool;-- delete options.file;---- // If this is a format-specific domain, enforce that format-- if (req.ctx.defaultFormat && toolId === 'convert') {-- options.format = req.ctx.defaultFormat;-- }---- // Execute the tool-- const result = await tool.handler(req.file.path, options);++ // Execute the tool (same options and code path as the audio.process job)++ const result = await tool.handler(req.file.path, buildOptions(req.body, toolId, req.ctx));// Clean up the uploaded input filecleanTmp(req.file.path);@@@ -166,24 +169,7 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,// Probe input for size comparisonconst inputSize = req.file.size;-- res.json({-- success: true,-- tool: toolId,-- download: saved,-- output: {-- mime: result.mime,-- ext: result.ext,-- size: saved.size,-- sizeKB: Math.round(saved.size / 1024 * 10) / 10,-- duration: result.duration || null,-- },-- input: {-- size: inputSize,-- sizeKB: Math.round(inputSize / 1024 * 10) / 10,-- },-- ...(result.metadata && { metadata: result.metadata }),-- ...(result.preset && { preset: result.preset }),-- });++ res.json({ success: true, download: saved, ...describe(toolId, result, saved.size, inputSize) });} catch (err) {cleanTmp(req.file?.path);console.error('[Process] Error:', err.message);@@@ -191,6 +177,24 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,}});++// ── Jobs (/api/v1/jobs) ──────────────────────────────────────++// The same operation as /api/process, asynchronous and durable: accepted into data/jobs.db,++// followed over SSE (ffmpeg's own progress), cancellable (ffmpeg is killed), reattachable by id++// after a reload or a restart (apps/_shared/jobs).++const jobs = jobsRuntime.setupJobs({++ app, service: 'audio', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts,++ getPublicKey: auth.getPublicKey, issuer: auth.ISSUER,++ define: defineJobs,++ receive: uploadSingle,++ limiters: [burstLimiter, processLimiter],++ defaults(req, input) {++ const out = { ...input };++ if (!out.tool) out.tool = req.ctx.defaultOp || 'convert';++ if (req.ctx.defaultFormat && out.tool === 'convert') out.format = req.ctx.defaultFormat;++ return out;++ },++});++// ── File Download ────────────────────────────────────────────app.get('/api/download/:id', (req, res) => {const entry = retention.getFile(req.params.id);@@@ -291,6 +295,7 @@ function shutdown() {console.log('[Audio.OpenVibe] Shutting down...');analytics.destroy();analyticsDb.close();++ jobs.close(); // running jobs stay 'running' in data/jobs.db; the next boot re-queues themserver.close(() => process.exit(0));setTimeout(() => process.exit(1), 5000);}nnew file mode 100644@@@ -0,0 +1,92 @@++'use strict';++++// ═══════════════════════════════════════════════════════════════++// Audio.OpenVibe — one audio operation, shared by the synchronous endpoint (/api/process) and the++// audio.process job (defineJobs below).++//++// Jobs can be cancelled and report progress although the tools build their own ffmpeg commands:++// FfmpegCommand#run is wrapped once, and a command started inside a job's AsyncLocalStorage++// context reports ffmpeg's percent to the job and is killed when the job's signal aborts.++// Commands started outside a job (the synchronous endpoint) are untouched.++// ═══════════════════════════════════════════════════════════════++++const fs = require('fs');++const path = require('path');++const { AsyncLocalStorage } = require('async_hooks');++const ffmpeg = require('fluent-ffmpeg');++const { getTool, listTools } = require('./tools');++++const jobContext = new AsyncLocalStorage();++const RESERVED = new Set(['tool', 'file', 'files']);++++const proto = ffmpeg.prototype;++if (!proto.__ovJobs) {++ const run = proto.run;++ proto.run = function runInJob(...args) {++ const job = jobContext.getStore();++ if (job) {++ const kill = () => {++ if (this.ffmpegProc) { try { this.ffmpegProc.kill('SIGKILL'); } catch { /* already gone */ } } else this.once('start', () => { try { this.ffmpegProc.kill('SIGKILL'); } catch { /* gone */ } });++ };++ if (job.signal.aborted) kill(); else job.signal.addEventListener('abort', kill, { once: true });++ this.on('progress', (p) => { if (p && Number.isFinite(p.percent)) job.progress(10 + Math.min(100, Math.max(0, p.percent)) * 0.8, 'Encoding'); });++ }++ return run.apply(this, args);++ };++ proto.__ovJobs = true;++}++++/** Tool options from a request body or a job input, with the host's forced format. */++function buildOptions(src, toolId, ctx) {++ const options = {};++ for (const [k, v] of Object.entries(src || {})) if (!RESERVED.has(k)) options[k] = v;++ if (ctx && ctx.defaultFormat && toolId === 'convert') options.format = ctx.defaultFormat;++ return options;++}++++/** The JSON the synchronous endpoint has always answered with, minus the download block. */++function describe(toolId, result, size, inputSize) {++ return {++ tool: toolId,++ output: { mime: result.mime, ext: result.ext, size, sizeKB: Math.round(size / 1024 * 10) / 10, duration: result.duration || null },++ input: { size: inputSize, sizeKB: Math.round(inputSize / 1024 * 10) / 10 },++ ...(result.metadata && { metadata: result.metadata }),++ ...(result.preset && { preset: result.preset }),++ };++}++++/** Job type audio.process: input { tool, …options } + one audio or video file. */++function defineJobs(system) {++ const known = new Set(listTools().map(t => t.id));++ system.define({++ type: 'audio.process',++ version: 1,++ minFiles: 1,++ maxFiles: 1,++ maxAttempts: 2,++ onRestart: 'requeue', // ffmpeg over the stored input: safe to run again++ timeoutMs: 15 * 60 * 1000,++ validate(input) {++ const id = String(input.tool || 'convert');++ if (!known.has(id)) return `Unknown tool: ${id}`;++ for (const [k, v] of Object.entries(input)) if (v != null && typeof v === 'object') return `Option ${k} must be a string or a number`;++ return null;++ },++ async run({ input, files, progress, signal }) {++ const toolId = String(input.tool || 'convert');++ const tool = getTool(toolId);++ const src = files[0];++ progress(5, 'Reading the audio');++ const result = await jobContext.run({ signal, progress }, () => tool.handler(src.path, buildOptions(input, toolId, null)));++ if (signal.aborted) { fs.rm(result.outputPath, { force: true }, () => {}); throw new Error('cancelled'); }++ const size = fs.statSync(result.outputPath).size;++ const base = path.basename(src.name || 'audio', path.extname(src.name || '')) || 'openvibeaudio-output';++ return {++ files: [{ path: result.outputPath, name: `${base}.${result.ext}`, mime: result.mime }],++ data: describe(toolId, result, size, src.size),++ };++ },++ });++}++++module.exports = { buildOptions, describe, defineJobs, jobContext };@@@ -8,10 +8,15 @@// hosts as duplicates of each other and social shares have no card. This stamps each host's// own title, description, canonical, Open Graph / Twitter tags and structured data into the// HTML on the way out. Rendered variants are cached per host; the file itself is read once.++//++// Canonical host: through the gateway (X-OV-* headers) the canonical, og:url and JSON-LD use the++// host it names — a descriptive host or a custom domain; reached directly, the host itself.// ═══════════════════════════════════════════════════════════════const fs = require('fs');const path = require('path');++const { canonicalHostFor, ownHost } = require('../../_shared/host-role');++const { DOMAIN_MAP } = require('./domain-map');const PUBLIC = path.join(__dirname, '..', 'public');const HUB_HOST = 'audio.openvibe.tools';@@@ -30,16 +35,18 @@ function baseHtml() {/*** Render index.html for one host with that host's own metadata.* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)-- * @param {string} host request hostname, e.g. 'png.openvibe.tools'++ * @param {string} host the satellite's own hostname for this tool, e.g. 'png.openvibe.tools'++ * @param {string} [canonicalHost] the host canonical URLs use (the gateway's; defaults to `host`)*/--function renderIndex(ctx, host) {-- const key = host;++function renderIndex(ctx, host, canonicalHost) {++ const canon = canonicalHost || host;++ const key = `${host}|${canon}`;if (_cache.has(key)) return _cache.get(key);-- const url = `https://${host}/`;++ const url = `https://${canon}/`;const title = ctx.seoTitle || ctx.brandName || HUB_NAME;const desc = ctx.seoDescription || '';-- const image = `https://${host}/og.png`;++ const image = `https://${canon}/og.png`;let html = baseHtml();// Replace what the file already sets, so there is never a second competing value.@@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {/** Express handler: the SPA with this host's SEO tags applied. */function sendIndex(req, res) {-- const host = String(req.headers.host || HUB_HOST).split(':')[0].toLowerCase();++ const own = ownHost(req, h => !!DOMAIN_MAP[h]);++ const host = DOMAIN_MAP[own] ? own : HUB_HOST;try {res.setHeader('Content-Type', 'text/html; charset=utf-8');res.setHeader('Cache-Control', 'no-cache');-- return res.send(renderIndex(req.ctx || {}, host));++ res.setHeader('Vary', 'X-OV-Canonical-Host');++ return res.send(renderIndex(req.ctx || {}, host, canonicalHostFor(req, host)));} catch (err) {console.error('[SEO] render failed:', err.message);return res.sendFile(path.join(PUBLIC, 'index.html'));nnew file mode 100644@@@ -0,0 +1,84 @@++'use strict';++// Audio.OpenVibe as a real process, with real ffmpeg: a job RUNNING when the process is killed is++// re-queued by the next process and finishes; a running job is cancelled (ffmpeg killed); progress++// events come from ffmpeg itself.++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const { execFileSync } = require('child_process');++const { startApp } = require('../../_shared/test/spawn');++++const sleep = (ms) => new Promise(r => setTimeout(r, ms));++++try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); } catch {++ console.log('audio jobs: SKIPPED (no ffmpeg on this machine)');++ process.exit(0);++}++++async function waitFor(app, id, cookie, pred, what, ms = 30000) {++ const t0 = Date.now();++ for (;;) {++ const job = await (await fetch(`${app.base}/api/v1/jobs/${id}`, { headers: { cookie } })).json();++ if (pred(job)) return job;++ if (Date.now() - t0 > ms) throw new Error(`timed out waiting for ${what}: ${JSON.stringify(job)}`);++ await sleep(50);++ }++}++++(async () => {++ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-audio-'));++ const env = { DATA_DIR: data, UPLOADS_DIR: path.join(data, 'uploads'), OUTPUT_DIR: path.join(data, 'output') };++ // Three minutes of tone: small to upload, a couple of seconds to re-encode.++ const input = path.join(data, 'tone.mp3');++ execFileSync('ffmpeg', ['-loglevel', 'error', '-y', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=180', '-ac', '1', '-b:a', '16k', input]);++ const bytes = fs.readFileSync(input);++ const submit = async (app, cookie, tool) => {++ const fd = new FormData();++ fd.append('type', 'audio.process');++ fd.append('input', JSON.stringify(tool));++ fd.append('file', new Blob([bytes], { type: 'audio/mpeg' }), 'tone.mp3');++ const r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd, headers: cookie ? { cookie } : {} });++ assert.strictEqual(r.status, 202, await r.clone().text());++ return { job: await r.json(), cookie: cookie || r.headers.get('set-cookie').split(';')[0] };++ };++ let app;++ try {++ app = await startApp('audio', env);++++ // Killed while ffmpeg runs → the next process runs it again.++ const { job, cookie } = await submit(app, null, { tool: 'convert', format: 'flac' });++ await waitFor(app, job.id, cookie, j => j.state === 'running', 'running');++ await app.kill('SIGKILL');++ app = await startApp('audio', env, app.port);++ const done = await waitFor(app, job.id, cookie, j => j.state === 'succeeded' || j.state === 'failed', 'finished after restart');++ assert.strictEqual(done.state, 'succeeded', JSON.stringify(done.error));++ assert.strictEqual(done.attempts, 2, 'the interrupted attempt is counted');++ assert.strictEqual(done.result.files[0].name, 'tone.flac');++ const r = await fetch(`${app.base}${done.result.files[0].url}`, { headers: { cookie } });++ assert.strictEqual(Buffer.from(await r.arrayBuffer()).slice(0, 4).toString(), 'fLaC', 'the result is FLAC');++ const ranged = await fetch(`${app.base}${done.result.files[0].url}?inline=1`, { headers: { cookie, range: 'bytes=0-3' } });++ assert.strictEqual(ranged.status, 206, 'previews support range requests (audio seeking)');++++ // Progress from ffmpeg, then cancel: ffmpeg is killed and the job ends cancelled.++ const second = await submit(app, cookie, { tool: 'convert', format: 'mp3', bitrate: '320' });++ const moving = await waitFor(app, second.job.id, cookie, j => j.state === 'running' && j.progress.message === 'Encoding' || j.state === 'succeeded', 'ffmpeg progress');++ if (moving.state === 'running') {++ const c = await fetch(`${app.base}/api/v1/jobs/${second.job.id}`, { method: 'DELETE', headers: { cookie } });++ assert.ok([200, 202].includes(c.status));++ const cancelled = await waitFor(app, second.job.id, cookie, j => j.state !== 'running', 'cancelled');++ assert.strictEqual(cancelled.state, 'cancelled');++ assert.strictEqual(cancelled.error.code, 'tools.job.cancelled');++ } else {++ console.log('audio jobs: (this machine encoded too fast to cancel mid-run; cancel is covered by the shared tests)');++ }++++ await app.kill('SIGTERM');++ console.log('audio jobs: all checks passed');++ } catch (err) {++ if (app) { console.error(app.output()); await app.kill('SIGKILL'); }++ throw err;++ } finally {++ fs.rmSync(data, { recursive: true, force: true });++ }++})().catch((err) => { console.error(err); process.exit(1); });@@@ -10,16 +10,17 @@},"license": "MIT","dependencies": {++ "better-sqlite3": "^11.0.0","cookie-parser": "^1.4.7","cors": "^2.8.5","dotenv": "^16.4.7","express": "^4.21.2","express-rate-limit": "^7.5.0","helmet": "^8.1.0",-- "better-sqlite3": "^11.0.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0","jsonwebtoken": "^9.0.2","multer": "^1.4.5-lts.1",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.7.0",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0","pdf-lib": "^1.17.1","sharp": "^0.33.5","uuid": "^11.1.0"@@@ -223,6 +223,7 @@<script src="https://openvibe.network/shared/footer.js"></script><script src="https://openvibe.network/shared/notification-ui.js"></script><script src="https://openvibe.network/shared/account-switcher.js"></script>++ <script src="/js/ov-jobs.js"></script><script src="/js/app.js"></script></body></html>@@@ -65,6 +65,7 @@initOptions();initSubdomains();initResult();++ reattach();}// ── Branding ─────────────────────────────────────────────@@@ -434,6 +435,21 @@// Add tool-specific optionsaddToolOptions(formData);++ // Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).++ if (window.OVJobs) {++ const input = {};++ for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;++ try {++ const job = await OVJobs.submit({ type: 'docs.process', input, files: isMulti ? selectedFiles : [selectedFiles[0]], fileField: isMulti ? 'files' : 'file' });++ OVJobs.remember(job.id);++ followJob(job);++ } catch (err) {++ failProcessing(err.message);++ }++ return;++ }++++ // Without the jobs helper: the synchronous endpoints, as before.try {const token = getCookie('ov_token') || localStorage.getItem('ov_token');const headers = {};@@@ -509,9 +525,85 @@}}++ // ── Jobs ─────────────────────────────────────────────────++ let watcher = null;++ let currentJob = null;++++ function processingText(job) {++ const pct = job.progress && job.progress.percent;++ if (job.cancel_requested) return 'Cancelling...';++ if (job.state === 'queued') return 'Waiting for a free slot...';++ const label = (job.progress && job.progress.message) || 'Processing your document';++ return pct != null ? `${label}... ${Math.round(pct)}%` : `${label}...`;++ }++++ function cancelButton() {++ let btn = processingEl.querySelector('.job-cancel');++ if (!btn) {++ btn = document.createElement('button');++ btn.type = 'button';++ btn.className = 'btn-secondary job-cancel';++ btn.textContent = 'Cancel';++ btn.addEventListener('click', () => { if (currentJob) OVJobs.cancel(currentJob).catch(() => {}); });++ processingEl.appendChild(btn);++ }++ return btn;++ }++++ function followJob(job) {++ currentJob = job.id;++ optionsPanel.style.display = 'none';++ resultPanel.style.display = 'none';++ processingEl.style.display = '';++ processBtn.disabled = true;++ const label = processingEl.querySelector('p');++ const update = (j) => { if (label) label.textContent = processingText(j); };++ cancelButton().style.display = '';++ update(job);++ if (watcher) watcher.close();++ watcher = OVJobs.watch(job.id, {++ onUpdate: update,++ onDone(done) {++ watcher = null;++ cancelButton().style.display = 'none';++ if (done.state === 'succeeded') {++ const data = jobResult(done);++ return data.viewOnly ? showMetadataResult(data) : showResult(data);++ }++ OVJobs.forget();++ failProcessing(done.state === 'cancelled' ? null : (done.error && done.error.detail) || 'Processing failed');++ },++ onGone() { OVJobs.forget(); failProcessing(null); },++ });++ }++++ /** A finished job in the shape the synchronous endpoints answer with. */++ function jobResult(job) {++ const f = job.result && job.result.files && job.result.files[0];++ return {++ ...(job.result && job.result.data),++ download: f ? { id: job.id, downloadUrl: f.url, filename: f.name, expiresIn: OVJobs.expiresIn(job) } : undefined,++ };++ }++++ function failProcessing(message) {++ processingEl.style.display = 'none';++ optionsPanel.style.display = selectedFiles.length ? '' : 'none';++ processBtn.disabled = !selectedFiles.length;++ if (message) alert(`Error: ${message}`);++ }++++ /** After a reload: pick the job up again from ?job=<id> or this tab's session. */++ function reattach() {++ const id = window.OVJobs && OVJobs.recall();++ if (!id) return;++ OVJobs.get(id).then(followJob).catch(() => OVJobs.forget());++ }++// ── Result ───────────────────────────────────────────────function initResult() {anotherBtn.addEventListener('click', () => {++ if (window.OVJobs) OVJobs.forget();resetUpload();(toolTabs.style.display !== 'none' ? toolTabs : uploadArea).scrollIntoView({ behavior: 'smooth' });});@@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {next();}--module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey };++/** The Network public key (PEM) once loaded; the job routes verify service tokens with it. */++function getPublicKey() { return publicKey; }++++module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey, getPublicKey, ISSUER };@@@ -14,18 +14,23 @@ const path = require('path');const fs = require('fs');const config = require('./config');--const { optionalAuth } = require('./auth');--const { resolveContext } = require('./domain-map');++const auth = require('./auth');++const { optionalAuth } = auth;++const { resolveContext, DOMAIN_MAP } = require('./domain-map');const { getTool, listTools } = require('./tools');--const { uploadSingle, uploadMultiple } = require('./middleware/upload');++const { uploadSingle, uploadMultiple, uploadAny } = require('./middleware/upload');const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');const retention = require('./retention/manager');++const { buildOptions, defineJobs } = require('./process');++const { hostGuard, ownHost } = require('../../_shared/host-role');++const jobsRuntime = require('../../_shared/jobs');++const contracts = require('openvibe-contracts');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');const { AnalyticsTracker } = require('openvibe-shared/analytics');const { internalOk } = require('../../_shared/internal-auth');--const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');++const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');@@@ -55,6 +60,7 @@ app.use(helmet({}));app.use(cookieParser());app.use(express.json({ limit: '1mb' }));++app.use(contracts.http.middleware()); // traceparent + X-OpenVibe-Request-Id on every response// ── CORS ─────────────────────────────────────────────────────app.use(cors({@@@ -76,9 +82,16 @@ app.use(analytics.middleware());// ── Auth (optional on all routes) ────────────────────────────app.use(optionalAuth);++// ── Hosts ────────────────────────────────────────────────────++// Through the gateway the X-OV-* headers name the tool and its canonical host (a custom domain++// included); aliases the gateway missed are redirected. Hosts this app does not serve go to the++// tools index. API paths are never redirected.++const knowsHost = (h) => !!DOMAIN_MAP[h];++app.use(hostGuard({ knows: knowsHost }));++// ── Attach domain context to every request ───────────────────app.use((req, _res, next) => {-- req.ctx = resolveContext(req.headers.host);++ req.ctx = resolveContext(ownHost(req, knowsHost));next();});@@@ -87,7 +100,7 @@ app.use((req, _res, next) => {// Health checkapp.get('/api/health', (_req, res) => {const stats = retention.getStats();-- res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats });++ res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats, jobs: jobs.stats() });});// Domain context (frontend calls this on load to get branding)@@@ -133,22 +146,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,const tool = getTool(toolId);if (!tool) return res.status(400).json({ error: `Unknown tool: ${toolId}` });-- // Build options from body params-- const options = {};-- for (const key of ['format', 'quality', 'angle', 'pages', 'order', 'ranges', 'mode',-- 'text', 'fontSize', 'opacity', 'rotation', 'color',-- 'pageSize', 'dpi', 'password', 'userPassword', 'ownerPassword',-- 'title', 'author', 'subject', 'keywords', 'creator',-- 'level', 'defaultFormat']) {-- if (req.body[key] !== undefined) options[key] = req.body[key];-- }---- // Use context default format if available-- if (req.ctx.defaultFormat && !options.format) {-- options.defaultFormat = req.ctx.defaultFormat;-- }---- const result = await tool.handler(req.file.buffer, options);++ // Same options and code path as the docs.process job++ const result = await tool.handler(req.file.buffer, buildOptions(req.body, req.ctx, false));// Some tools return view-only results (like metadata view)if (result.viewOnly) {@@@ -196,12 +195,7 @@ app.post('/api/process/multi', burstLimiter, processLimiter, uploadMultiple, asyif (!tool.multiFile) return res.status(400).json({ error: `Tool "${toolId}" does not support multiple files. Use /api/process instead.` });const buffers = req.files.map(f => f.buffer);-- const options = {};-- for (const key of ['order', 'pageSize', 'quality', 'format']) {-- if (req.body[key] !== undefined) options[key] = req.body[key];-- }---- const result = await tool.handler(buffers, options);++ const result = await tool.handler(buffers, buildOptions(req.body, req.ctx, true));// Save outputconst firstName = req.files[0]?.originalname || 'output';@@@ -232,6 +226,23 @@ app.post('/api/process/multi', burstLimiter, processLimiter, uploadMultiple, asy}});++// ── Jobs (/api/v1/jobs) ──────────────────────────────────────++// The same operations as /api/process and /api/process/multi, asynchronous and durable: accepted++// into data/jobs.db, followed over SSE, reattachable by id after a reload or a restart.++const jobs = jobsRuntime.setupJobs({++ app, service: 'docs', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts,++ getPublicKey: auth.getPublicKey, issuer: auth.ISSUER,++ define: defineJobs,++ receive: uploadAny,++ limiters: [burstLimiter, processLimiter],++ defaults(req, input) {++ const out = { ...input };++ if (!out.tool && req.ctx.defaultOp) out.tool = req.ctx.defaultOp;++ if (!out.format && !out.defaultFormat && req.ctx.defaultFormat) out.defaultFormat = req.ctx.defaultFormat;++ return out;++ },++});++// ── File Download ────────────────────────────────────────────app.get('/api/download/:id', (req, res) => {const entry = retention.getFile(req.params.id);@@@ -301,6 +312,7 @@ function shutdown() {console.log('[Docs.OpenVibe] Shutting down...');analytics.destroy();analyticsDb.close();++ jobs.close(); // running jobs stay 'running' in data/jobs.db; the next boot re-queues themserver.close(() => process.exit(0));setTimeout(() => process.exit(1), 5000);}@@@ -69,4 +69,25 @@ function uploadMultiple(req, res, next) {});}--module.exports = { uploadSingle, uploadMultiple };++/**++ * One file in 'file' or several in 'files' (the job endpoint takes either shape).++ * Attaches req.files as { file?: [...], files?: [...] }.++ */++function uploadAny(req, res, next) {++ upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: 50 }])(req, res, (err) => {++ if (err) {++ if (err.code === 'LIMIT_FILE_SIZE') {++ return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });++ }++ if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') {++ return res.status(400).json({ error: 'Send one file in "file" or up to 50 in "files".' });++ }++ return res.status(400).json({ error: err.message });++ }++ const n = req.files ? Object.values(req.files).flat().length : 0;++ if (!n) return res.status(400).json({ error: 'No files uploaded. Send a file in the "file" field.' });++ next();++ });++}++++module.exports = { uploadSingle, uploadMultiple, uploadAny };nnew file mode 100644@@@ -0,0 +1,82 @@++'use strict';++++// ═══════════════════════════════════════════════════════════════++// Docs.OpenVibe — one document operation, shared by the synchronous endpoints++// (/api/process, /api/process/multi) and the docs.process job (see defineJobs).++// ═══════════════════════════════════════════════════════════════++++const fs = require('fs');++const fsp = require('fs/promises');++const path = require('path');++const { getTool, listTools } = require('./tools');++++const SINGLE_KEYS = ['format', 'quality', 'angle', 'pages', 'order', 'ranges', 'mode',++ 'text', 'fontSize', 'opacity', 'rotation', 'color',++ 'pageSize', 'dpi', 'password', 'userPassword', 'ownerPassword',++ 'title', 'author', 'subject', 'keywords', 'creator',++ 'level', 'defaultFormat'];++const MULTI_KEYS = ['order', 'pageSize', 'quality', 'format'];++++/** Tool options from a request body or a job input. */++function buildOptions(src, ctx, multi) {++ const options = {};++ for (const key of multi ? MULTI_KEYS : SINGLE_KEYS) if (src[key] !== undefined) options[key] = src[key];++ if (!multi && ctx && ctx.defaultFormat && !options.format) options.defaultFormat = ctx.defaultFormat;++ return options;++}++++/** The JSON the synchronous endpoints have always answered with, minus the download block. */++function describe(toolId, result, fileCount) {++ if (result.viewOnly) { const { buffer, ...rest } = result; return { tool: toolId, ...rest }; }++ return {++ tool: toolId,++ output: { mime: result.mime, ext: result.ext, size: result.buffer.length, sizeKB: Math.round(result.buffer.length / 1024 * 10) / 10 },++ ...(fileCount != null && { fileCount }),++ ...(result.pageCount !== undefined && { pageCount: result.pageCount }),++ ...(result.savings && { savings: result.savings }),++ ...(result.metadata && { metadata: result.metadata }),++ ...(result.note && { note: result.note }),++ };++}++++/** Job type docs.process: input { tool, …options } + one PDF, or several files for merge / img2pdf. */++function defineJobs(system) {++ const known = new Set(listTools().map(t => t.id));++ system.define({++ type: 'docs.process',++ version: 1,++ minFiles: 1,++ maxFiles: 50,++ maxAttempts: 3,++ onRestart: 'requeue', // a pure function of the stored input: safe to run again++ timeoutMs: 10 * 60 * 1000,++ validate(input, files) {++ const id = String(input.tool || '');++ if (!known.has(id)) return id ? `Unknown tool: ${id}` : 'No tool specified.';++ const tool = getTool(id);++ if (!tool.multiFile && files.length !== 1) return `Tool "${id}" takes exactly one file`;++ if (id === 'merge' && files.length < 2) return 'Upload at least 2 files to merge.';++ for (const k of Object.keys(input)) if (k !== 'tool' && !SINGLE_KEYS.includes(k)) return `Unknown option: ${k}`;++ return null;++ },++ async run({ input, files, outDir, progress, signal }) {++ const toolId = String(input.tool);++ const tool = getTool(toolId);++ progress(5, files.length > 1 ? `Reading ${files.length} files` : 'Reading the document');++ const buffers = [];++ for (const f of files) buffers.push(await fsp.readFile(f.path));++ if (signal.aborted) throw new Error('cancelled');++ progress(20, 'Processing');++ const result = await tool.handler(tool.multiFile ? buffers : buffers[0], buildOptions(input, null, tool.multiFile));++ if (result.viewOnly) return { files: [], data: describe(toolId, result) };++ progress(90, 'Saving');++ const first = files[0].name || 'document';++ const name = `${path.basename(first, path.extname(first)) || 'openvibedocs-output'}.${result.ext}`;++ const out = path.join(outDir, `result.${result.ext}`);++ fs.writeFileSync(out, result.buffer);++ return { files: [{ path: out, name, mime: result.mime }], data: describe(toolId, result, tool.multiFile ? files.length : undefined) };++ },++ });++}++++module.exports = { buildOptions, describe, defineJobs, SINGLE_KEYS, MULTI_KEYS };@@@ -8,10 +8,15 @@// hosts as duplicates of each other and social shares have no card. This stamps each host's// own title, description, canonical, Open Graph / Twitter tags and structured data into the// HTML on the way out. Rendered variants are cached per host; the file itself is read once.++//++// Canonical host: through the gateway (X-OV-* headers) the canonical, og:url and JSON-LD use the++// host it names — a descriptive host or a custom domain; reached directly, the host itself.// ═══════════════════════════════════════════════════════════════const fs = require('fs');const path = require('path');++const { canonicalHostFor, ownHost } = require('../../_shared/host-role');++const { DOMAIN_MAP } = require('./domain-map');const PUBLIC = path.join(__dirname, '..', 'public');const HUB_HOST = 'docs.openvibe.tools';@@@ -30,16 +35,18 @@ function baseHtml() {/*** Render index.html for one host with that host's own metadata.* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)-- * @param {string} host request hostname, e.g. 'png.openvibe.tools'++ * @param {string} host the satellite's own hostname for this tool, e.g. 'png.openvibe.tools'++ * @param {string} [canonicalHost] the host canonical URLs use (the gateway's; defaults to `host`)*/--function renderIndex(ctx, host) {-- const key = host;++function renderIndex(ctx, host, canonicalHost) {++ const canon = canonicalHost || host;++ const key = `${host}|${canon}`;if (_cache.has(key)) return _cache.get(key);-- const url = `https://${host}/`;++ const url = `https://${canon}/`;const title = ctx.seoTitle || ctx.brandName || HUB_NAME;const desc = ctx.seoDescription || '';-- const image = `https://${host}/og.png`;++ const image = `https://${canon}/og.png`;let html = baseHtml();// Replace what the file already sets, so there is never a second competing value.@@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {/** Express handler: the SPA with this host's SEO tags applied. */function sendIndex(req, res) {-- const host = String(req.headers.host || HUB_HOST).split(':')[0].toLowerCase();++ const own = ownHost(req, h => !!DOMAIN_MAP[h]);++ const host = DOMAIN_MAP[own] ? own : HUB_HOST;try {res.setHeader('Content-Type', 'text/html; charset=utf-8');res.setHeader('Cache-Control', 'no-cache');-- return res.send(renderIndex(req.ctx || {}, host));++ res.setHeader('Vary', 'X-OV-Canonical-Host');++ return res.send(renderIndex(req.ctx || {}, host, canonicalHostFor(req, host)));} catch (err) {console.error('[SEO] render failed:', err.message);return res.sendFile(path.join(PUBLIC, 'index.html'));nnew file mode 100644@@@ -0,0 +1,102 @@++'use strict';++// Docs.OpenVibe as a real process: a multi-file merge job accepted before a SIGKILL finishes after++// the restart; single-file and view-only jobs; the synchronous endpoints still answer; canonical host.++const assert = require('assert');++const fs = require('fs');++const http = require('http');++const os = require('os');++const path = require('path');++const { PDFDocument } = require('pdf-lib');++const { startApp } = require('../../_shared/test/spawn');++++const sleep = (ms) => new Promise(r => setTimeout(r, ms));++++async function pdf(pages, title) {++ const doc = await PDFDocument.create();++ for (let i = 0; i < pages; i++) doc.addPage([200, 300]);++ if (title) doc.setTitle(title);++ return Buffer.from(await doc.save());++}++++async function finished(app, id, cookie) {++ for (let i = 0; i < 150; i++) {++ const job = await (await fetch(`${app.base}/api/v1/jobs/${id}`, { headers: { cookie } })).json();++ if (['succeeded', 'failed', 'cancelled'].includes(job.state)) return job;++ await sleep(100);++ }++ throw new Error(`job ${id} did not finish`);++}++++(async () => {++ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-docs-'));++ const env = { DATA_DIR: data, UPLOADS_DIR: path.join(data, 'uploads'), OUTPUT_DIR: path.join(data, 'output') };++ const a = await pdf(2, 'First'), b = await pdf(3);++ let app;++ try {++ app = await startApp('docs', { ...env, TOOLS_JOBS_CONCURRENCY_DOCS: '0' });++ const fd = new FormData();++ fd.append('type', 'docs.process');++ fd.append('input', JSON.stringify({ tool: 'merge' }));++ fd.append('files', new Blob([a], { type: 'application/pdf' }), 'first.pdf');++ fd.append('files', new Blob([b], { type: 'application/pdf' }), 'second.pdf');++ let r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd });++ assert.strictEqual(r.status, 202, await r.clone().text());++ const cookie = r.headers.get('set-cookie').split(';')[0];++ const merge = await r.json();++ await app.kill('SIGKILL');++++ app = await startApp('docs', env, app.port);++ const merged = await finished(app, merge.id, cookie);++ assert.strictEqual(merged.state, 'succeeded', JSON.stringify(merged.error));++ assert.strictEqual(merged.result.data.pageCount, 5);++ assert.strictEqual(merged.result.data.fileCount, 2);++ r = await fetch(`${app.base}${merged.result.files[0].url}`, { headers: { cookie } });++ assert.strictEqual((await PDFDocument.load(Buffer.from(await r.arrayBuffer()))).getPageCount(), 5, 'the merged PDF has every page');++++ // Single file, and a view-only tool (no result file, only data).++ const one = async (input, file = a) => {++ const f = new FormData();++ f.append('type', 'docs.process'); f.append('input', JSON.stringify(input));++ f.append('file', new Blob([file], { type: 'application/pdf' }), 'doc.pdf');++ const res = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: f, headers: { cookie } });++ return finished(app, (await res.json()).id, cookie);++ };++ const rotated = await one({ tool: 'rotate', angle: '90', pages: 'all' });++ assert.strictEqual(rotated.state, 'succeeded');++ assert.strictEqual(rotated.result.files[0].name, 'doc.pdf');++ const info = await one({ tool: 'metadata', mode: 'view' });++ assert.strictEqual(info.state, 'succeeded');++ assert.strictEqual(info.result.files.length, 0);++ assert.strictEqual(info.result.data.viewOnly, true);++ assert.strictEqual(info.result.data.metadata.title, 'First');++ const bad = await one({ tool: 'rotate', angle: '90' }, Buffer.from('%PDF-1.4 not really'));++ assert.strictEqual(bad.state, 'failed');++ assert.strictEqual(bad.error.code, 'tools.job.failed');++ r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', headers: { 'content-type': 'application/json', cookie }, body: JSON.stringify({ type: 'docs.process', input: { tool: 'merge' } }) });++ assert.strictEqual(r.status, 400, 'a job without files is refused');++++ // The synchronous endpoint is unchanged (after the burst limiter's 5 s window: jobs count too).++ await sleep(5100);++ const sync = new FormData();++ sync.append('files', new Blob([a], { type: 'application/pdf' }), 'a.pdf');++ sync.append('files', new Blob([b], { type: 'application/pdf' }), 'b.pdf');++ sync.append('tool', 'merge');++ r = await fetch(`${app.base}/api/process/multi`, { method: 'POST', body: sync });++ const body = await r.json();++ assert.strictEqual(body.success, true, JSON.stringify(body)); assert.strictEqual(body.pageCount, 5);++++ // Canonical host through the gateway.++ const html = await new Promise((resolve, reject) => http.get({ host: '127.0.0.1', port: app.port, path: '/', headers: { Host: 'mergepdf.openvibe.tools', 'X-OV-Tool': 'mergepdf', 'X-OV-Host-Role': 'short', 'X-OV-Canonical-Host': 'merge-pdf.openvibe.tools' } }, (res) => {++ let t = ''; res.setEncoding('utf8'); res.on('data', c => { t += c; }); res.on('end', () => resolve(t));++ }).on('error', reject));++ assert.ok(html.includes('<link rel="canonical" href="https://merge-pdf.openvibe.tools/">'), 'canonical follows X-OV-Canonical-Host');++++ await app.kill('SIGTERM');++ console.log('docs jobs + canonical host: all checks passed');++ } catch (err) {++ if (app) { console.error(app.output()); await app.kill('SIGKILL'); }++ throw err;++ } finally {++ fs.rmSync(data, { recursive: true, force: true });++ }++})().catch((err) => { console.error(err); process.exit(1); });@@@ -10,16 +10,17 @@},"license": "MIT","dependencies": {++ "better-sqlite3": "^11.0.0","cookie-parser": "^1.4.7","cors": "^2.8.5","dotenv": "^16.4.7","express": "^4.21.2","express-rate-limit": "^7.5.0","helmet": "^8.1.0",-- "better-sqlite3": "^11.0.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0","jsonwebtoken": "^9.0.2","multer": "^1.4.5-lts.1",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.7.0",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0","sharp": "^0.33.5","to-ico": "^1.1.5","uuid": "^11.1.0"@@@ -172,6 +172,7 @@<script src="https://openvibe.network/shared/footer.js"></script><script src="https://openvibe.network/shared/notification-ui.js"></script><script src="https://openvibe.network/shared/account-switcher.js"></script>++ <script src="/js/ov-jobs.js"></script><script src="/js/app.js"></script></body></html>@@@ -64,6 +64,7 @@initOptions();initSubdomains();initResult();++ reattach();}// ── Branding ─────────────────────────────────────────────@@@ -356,6 +357,21 @@}}++ // Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).++ if (window.OVJobs) {++ const input = {};++ for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;++ try {++ const job = await OVJobs.submit({ type: 'img.process', input, files: [selectedFile] });++ OVJobs.remember(job.id);++ followJob(job);++ } catch (err) {++ failProcessing(err.message);++ }++ return;++ }++++ // Without the jobs helper: the synchronous endpoint, as before.try {const token = getCookie('ov_token') || localStorage.getItem('ov_token');const headers = {};@@@ -378,9 +394,82 @@}}++ // ── Jobs ─────────────────────────────────────────────────++ let watcher = null;++ let currentJob = null;++++ function processingText(job) {++ const pct = job.progress && job.progress.percent;++ if (job.cancel_requested) return 'Cancelling...';++ if (job.state === 'queued') return 'Waiting for a free slot...';++ const label = (job.progress && job.progress.message) || 'Processing your image';++ return pct != null ? `${label}... ${Math.round(pct)}%` : `${label}...`;++ }++++ function cancelButton() {++ let btn = processingEl.querySelector('.job-cancel');++ if (!btn) {++ btn = document.createElement('button');++ btn.type = 'button';++ btn.className = 'btn-secondary job-cancel';++ btn.textContent = 'Cancel';++ btn.addEventListener('click', () => { if (currentJob) OVJobs.cancel(currentJob).catch(() => {}); });++ processingEl.appendChild(btn);++ }++ return btn;++ }++++ function followJob(job) {++ currentJob = job.id;++ optionsPanel.style.display = 'none';++ resultPanel.style.display = 'none';++ processingEl.style.display = '';++ processBtn.disabled = true;++ const label = processingEl.querySelector('p');++ const update = (j) => { if (label) label.textContent = processingText(j); };++ cancelButton().style.display = '';++ update(job);++ if (watcher) watcher.close();++ watcher = OVJobs.watch(job.id, {++ onUpdate: update,++ onDone(done) {++ watcher = null;++ cancelButton().style.display = 'none';++ if (done.state === 'succeeded') return showResult(jobResult(done));++ OVJobs.forget();++ failProcessing(done.state === 'cancelled' ? null : (done.error && done.error.detail) || 'Processing failed');++ },++ onGone() { OVJobs.forget(); failProcessing(null); },++ });++ }++++ /** A finished job in the shape the synchronous endpoint answers with. */++ function jobResult(job) {++ const f = (job.result && job.result.files && job.result.files[0]) || {};++ return {++ ...(job.result && job.result.data),++ download: { id: job.id, downloadUrl: f.url, previewUrl: f.url ? `${f.url}?inline=1` : null, filename: f.name, expiresIn: OVJobs.expiresIn(job) },++ };++ }++++ function failProcessing(message) {++ processingEl.style.display = 'none';++ optionsPanel.style.display = selectedFile ? '' : 'none';++ processBtn.disabled = !selectedFile;++ if (message) alert(`Error: ${message}`);++ }++++ /** After a reload: pick the job up again from ?job=<id> or this tab's session. */++ function reattach() {++ const id = window.OVJobs && OVJobs.recall();++ if (!id) return;++ OVJobs.get(id).then(followJob).catch(() => OVJobs.forget());++ }++// ── Result ───────────────────────────────────────────────function initResult() {anotherBtn.addEventListener('click', () => {++ if (window.OVJobs) OVJobs.forget();resetUpload();toolTabs.scrollIntoView({ behavior: 'smooth' });});@@@ -393,7 +482,10 @@// Preview image (if it's a previewable type)const previewable = ['image/png', 'image/jpeg', 'image/webp', 'image/gif', 'image/bmp', 'image/avif'];if (previewable.includes(data.output.mime)) {-- resultPreview.innerHTML = `<img src="${data.download.downloadUrl}" alt="Result">`;++ const img = document.createElement('img');++ img.src = data.download.previewUrl || data.download.downloadUrl;++ img.alt = 'Result';++ resultPreview.replaceChildren(img);} else {resultPreview.innerHTML = `<div style="font-size:48px;color:var(--accent);padding:24px"><i class="fa-solid fa-file-image"></i></div>`;}@@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {next();}--module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey };++/** The Network public key (PEM) once loaded; the job routes verify service tokens with it. */++function getPublicKey() { return publicKey; }++++module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey, getPublicKey, ISSUER };@@@ -14,18 +14,23 @@ const path = require('path');const fs = require('fs');const config = require('./config');--const { optionalAuth } = require('./auth');--const { resolveContext } = require('./domain-map');++const auth = require('./auth');++const { optionalAuth } = auth;++const { resolveContext, DOMAIN_MAP } = require('./domain-map');const { getTool, listTools } = require('./tools');const { uploadSingle } = require('./middleware/upload');const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');const retention = require('./retention/manager');++const { buildOptions, describe, processBuffer, defineJobs } = require('./process');++const { hostGuard, ownHost } = require('../../_shared/host-role');++const jobsRuntime = require('../../_shared/jobs');++const contracts = require('openvibe-contracts');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');const { AnalyticsTracker } = require('openvibe-shared/analytics');const { internalOk } = require('../../_shared/internal-auth');--const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');++const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');@@@ -45,7 +50,7 @@ app.use(helmet({scriptSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://openvibe.network"],styleSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://fonts.gstatic.com"],fontSrc: ["'self'", "https://fonts.gstatic.com", "https://cdnjs.cloudflare.com"],-- imgSrc: ["'self'", "data:", "blob:"],++ imgSrc: ["'self'", "data:", "blob:", jobsRuntime.mediaOrigin()], // job result previews are 302s to MediaconnectSrc: ["'self'", "https://openvibe.network", "https://*.openvibe.tools"],workerSrc: ["'self'", "blob:"],scriptSrcAttr: ["'unsafe-inline'"],@@@ -55,6 +60,7 @@ app.use(helmet({}));app.use(cookieParser());app.use(express.json({ limit: '1mb' }));++app.use(contracts.http.middleware()); // traceparent + X-OpenVibe-Request-Id on every response// ── CORS ─────────────────────────────────────────────────────app.use(cors({@@@ -77,9 +83,16 @@ app.use(analytics.middleware());// ── Auth (optional on all routes) ────────────────────────────app.use(optionalAuth);++// ── Hosts ────────────────────────────────────────────────────++// Through the gateway the X-OV-* headers name the tool and its canonical host (a custom domain++// included); aliases the gateway missed are redirected. Hosts this app does not serve go to the++// tools index. API paths are never redirected.++const knowsHost = (h) => !!DOMAIN_MAP[h];++app.use(hostGuard({ knows: knowsHost }));++// ── Attach domain context to every request ───────────────────app.use((req, _res, next) => {-- req.ctx = resolveContext(req.headers.host);++ req.ctx = resolveContext(ownHost(req, knowsHost));next();});@@@ -88,7 +101,7 @@ app.use((req, _res, next) => {// Health checkapp.get('/api/health', (_req, res) => {const stats = retention.getStats();-- res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats });++ res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats, jobs: jobs.stats() });});// Domain context (frontend calls this on load to get branding)@@@ -122,23 +135,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,return res.status(400).json({ error: `Unknown tool: ${toolId}` });}-- // Build options from body params-- const options = {-- format: req.body.format || req.ctx.defaultFormat || undefined,-- quality: req.body.quality || undefined,-- width: req.body.width || undefined,-- height: req.body.height || undefined,-- percentage: req.body.percentage || undefined,-- fit: req.body.fit || undefined,-- background: req.body.background || undefined,-- left: req.body.left || undefined,-- top: req.body.top || undefined,-- aspect: req.body.aspect || undefined,-- inputFormat: req.file.mimetype.split('/')[1],-- };---- // Execute the tool-- const result = await tool.handler(req.file.buffer, options);++ // Execute the tool (the same code the img.process job runs)++ const result = await processBuffer(req.file.buffer, toolId, buildOptions(req.body, req.ctx, req.file.mimetype));// Save to retentionconst saved = retention.saveOutput(@@@ -149,21 +147,7 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,req.file.originalname,);-- res.json({-- success: true,-- tool: toolId,-- download: saved,-- output: {-- mime: result.mime,-- ext: result.ext,-- size: result.buffer.length,-- sizeKB: Math.round(result.buffer.length / 1024 * 10) / 10,-- },-- // Include tool-specific metadata-- ...(result.savings && { savings: result.savings }),-- ...(result.dimensions && { dimensions: result.dimensions }),-- ...(result.crop && { crop: result.crop }),-- });++ res.json({ success: true, download: saved, ...describe(toolId, result) });} catch (err) {console.error('[Process] Error:', err.message);res.status(422).json({ error: err.message || 'Image processing failed' });@@@ -177,20 +161,7 @@ app.post('/api/process/direct', burstLimiter, processLimiter, uploadSingle, asynconst tool = getTool(toolId);if (!tool) return res.status(400).json({ error: `Unknown tool: ${toolId}` });-- const options = {-- format: req.body.format || req.ctx.defaultFormat || undefined,-- quality: req.body.quality || undefined,-- width: req.body.width || undefined,-- height: req.body.height || undefined,-- percentage: req.body.percentage || undefined,-- fit: req.body.fit || undefined,-- left: req.body.left || undefined,-- top: req.body.top || undefined,-- aspect: req.body.aspect || undefined,-- inputFormat: req.file.mimetype.split('/')[1],-- };---- const result = await tool.handler(req.file.buffer, options);++ const result = await processBuffer(req.file.buffer, toolId, buildOptions(req.body, req.ctx, req.file.mimetype));const baseName = path.basename(req.file.originalname, path.extname(req.file.originalname));res.set({@@@ -205,6 +176,23 @@ app.post('/api/process/direct', burstLimiter, processLimiter, uploadSingle, asyn}});++// ── Jobs (/api/v1/jobs) ──────────────────────────────────────++// The same operation as /api/process, asynchronous and durable: accepted into data/jobs.db,++// followed over SSE, reattachable by id after a reload or a restart (apps/_shared/jobs).++const jobs = jobsRuntime.setupJobs({++ app, service: 'img', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts,++ getPublicKey: auth.getPublicKey, issuer: auth.ISSUER,++ define: defineJobs,++ receive: uploadSingle,++ limiters: [burstLimiter, processLimiter],++ defaults(req, input) {++ const out = { ...input };++ if (!out.tool) out.tool = req.ctx.defaultOp || 'convert';++ if (!out.format && req.ctx.defaultFormat) out.format = req.ctx.defaultFormat;++ return out;++ },++});++// ── File Download ────────────────────────────────────────────app.get('/api/download/:id', (req, res) => {const entry = retention.getFile(req.params.id);@@@ -274,6 +262,7 @@ function shutdown() {console.log('[Img.OpenVibe] Shutting down...');analytics.destroy();analyticsDb.close();++ jobs.close(); // running jobs stay 'running' in data/jobs.db; the next boot re-queues themserver.close(() => process.exit(0));setTimeout(() => process.exit(1), 5000);}nnew file mode 100644@@@ -0,0 +1,75 @@++'use strict';++++// ═══════════════════════════════════════════════════════════════++// Img.OpenVibe — one image operation, shared by the synchronous endpoints++// (/api/process, /api/process/direct) and the img.process job (server/jobs.js).++// ═══════════════════════════════════════════════════════════════++++const fs = require('fs');++const fsp = require('fs/promises');++const path = require('path');++const { getTool, listTools } = require('./tools');++++const OPTION_KEYS = ['format', 'quality', 'width', 'height', 'percentage', 'fit', 'background', 'left', 'top', 'aspect'];++++/** Tool options from a request body or a job input, with the host's defaults. */++function buildOptions(src, ctx, inputMime) {++ const options = {};++ for (const k of OPTION_KEYS) options[k] = src[k] != null && src[k] !== '' ? src[k] : undefined;++ if (!options.format && ctx && ctx.defaultFormat) options.format = ctx.defaultFormat;++ options.inputFormat = String(inputMime || '').split('/')[1];++ return options;++}++++/** The JSON the synchronous endpoint has always answered with, minus the download block. */++function describe(toolId, result) {++ return {++ tool: toolId,++ output: { mime: result.mime, ext: result.ext, size: result.buffer.length, sizeKB: Math.round(result.buffer.length / 1024 * 10) / 10 },++ ...(result.savings && { savings: result.savings }),++ ...(result.dimensions && { dimensions: result.dimensions }),++ ...(result.crop && { crop: result.crop }),++ };++}++++async function processBuffer(buffer, toolId, options) {++ const tool = getTool(toolId);++ if (!tool) throw Object.assign(new Error(`Unknown tool: ${toolId}`), { code: 'tools.job.invalid', status: 400 });++ return tool.handler(buffer, options);++}++++/** Job type img.process: input { tool, format?, quality?, width?, … } + one image file. */++function defineJobs(system) {++ const known = new Set(listTools().map(t => t.id));++ system.define({++ type: 'img.process',++ version: 1,++ minFiles: 1,++ maxFiles: 1,++ maxAttempts: 3,++ onRestart: 'requeue', // a pure function of the stored input: safe to run again++ timeoutMs: 5 * 60 * 1000,++ validate(input) {++ if (!known.has(String(input.tool || 'convert'))) return `Unknown tool: ${input.tool}. Known: ${[...known].join(', ')}`;++ for (const k of Object.keys(input)) if (k !== 'tool' && !OPTION_KEYS.includes(k)) return `Unknown option: ${k}`;++ return null;++ },++ async run({ input, files, outDir, progress, signal }) {++ const toolId = String(input.tool || 'convert');++ const src = files[0];++ progress(5, 'Reading the image');++ const buffer = await fsp.readFile(src.path);++ if (signal.aborted) throw new Error('cancelled');++ progress(20, 'Processing');++ const result = await processBuffer(buffer, toolId, buildOptions(input, null, src.mime));++ progress(90, 'Saving');++ const base = path.basename(src.name || 'image', path.extname(src.name || '')) || 'openvibeimg-output';++ const name = `${base}.${result.ext}`;++ const out = path.join(outDir, `result.${result.ext}`);++ fs.writeFileSync(out, result.buffer);++ return { files: [{ path: out, name, mime: result.mime }], data: describe(toolId, result) };++ },++ });++}++++module.exports = { buildOptions, describe, processBuffer, defineJobs, OPTION_KEYS };@@@ -8,10 +8,15 @@// hosts as duplicates of each other and social shares have no card. This stamps each host's// own title, description, canonical, Open Graph / Twitter tags and structured data into the// HTML on the way out. Rendered variants are cached per host; the file itself is read once.++//++// Canonical host: through the gateway (X-OV-* headers) the canonical, og:url and JSON-LD use the++// host it names — a descriptive host or a custom domain; reached directly, the host itself.// ═══════════════════════════════════════════════════════════════const fs = require('fs');const path = require('path');++const { canonicalHostFor, ownHost } = require('../../_shared/host-role');++const { DOMAIN_MAP } = require('./domain-map');const PUBLIC = path.join(__dirname, '..', 'public');const HUB_HOST = 'img.openvibe.tools';@@@ -30,16 +35,18 @@ function baseHtml() {/*** Render index.html for one host with that host's own metadata.* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)-- * @param {string} host request hostname, e.g. 'png.openvibe.tools'++ * @param {string} host the satellite's own hostname for this tool, e.g. 'png.openvibe.tools'++ * @param {string} [canonicalHost] the host canonical URLs use (the gateway's; defaults to `host`)*/--function renderIndex(ctx, host) {-- const key = host;++function renderIndex(ctx, host, canonicalHost) {++ const canon = canonicalHost || host;++ const key = `${host}|${canon}`;if (_cache.has(key)) return _cache.get(key);-- const url = `https://${host}/`;++ const url = `https://${canon}/`;const title = ctx.seoTitle || ctx.brandName || HUB_NAME;const desc = ctx.seoDescription || '';-- const image = `https://${host}/og.png`;++ const image = `https://${canon}/og.png`;let html = baseHtml();// Replace what the file already sets, so there is never a second competing value.@@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {/** Express handler: the SPA with this host's SEO tags applied. */function sendIndex(req, res) {-- const host = String(req.headers.host || HUB_HOST).split(':')[0].toLowerCase();++ const own = ownHost(req, h => !!DOMAIN_MAP[h]);++ const host = DOMAIN_MAP[own] ? own : HUB_HOST;try {res.setHeader('Content-Type', 'text/html; charset=utf-8');res.setHeader('Cache-Control', 'no-cache');-- return res.send(renderIndex(req.ctx || {}, host));++ res.setHeader('Vary', 'X-OV-Canonical-Host');++ return res.send(renderIndex(req.ctx || {}, host, canonicalHostFor(req, host)));} catch (err) {console.error('[SEO] render failed:', err.message);return res.sendFile(path.join(PUBLIC, 'index.html'));nnew file mode 100644@@@ -0,0 +1,105 @@++'use strict';++// Img.OpenVibe as a real process: an image job accepted by one process is finished by the next one++// after a SIGKILL; the synchronous endpoint keeps working; the page honours X-OV-Canonical-Host.++const assert = require('assert');++const fs = require('fs');++const http = require('http');++const os = require('os');++const path = require('path');++const sharp = require('sharp');++const { startApp } = require('../../_shared/test/spawn');++++const sleep = (ms) => new Promise(r => setTimeout(r, ms));++++(async () => {++ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-img-'));++ const env = { DATA_DIR: data, UPLOADS_DIR: path.join(data, 'uploads'), OUTPUT_DIR: path.join(data, 'output') };++ const png = await sharp({ create: { width: 64, height: 48, channels: 3, background: { r: 200, g: 30, b: 90 } } }).png().toBuffer();++ let app;++ try {++ // Accepts but runs nothing, so the job is still queued when the process dies.++ app = await startApp('img', { ...env, TOOLS_JOBS_CONCURRENCY_IMG: '0' });++ const fd = new FormData();++ fd.append('type', 'img.process');++ fd.append('input', JSON.stringify({ tool: 'convert', format: 'webp', quality: '80' }));++ fd.append('file', new Blob([png], { type: 'image/png' }), 'red.png');++ let r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd, headers: { 'Idempotency-Key': 'img-test-0001' } });++ assert.strictEqual(r.status, 202, await r.clone().text());++ const cookie = r.headers.get('set-cookie').split(';')[0];++ const job = await r.json();++ assert.strictEqual(job.type, 'img.process');++ assert.strictEqual(job.state, 'queued');++ await app.kill('SIGKILL');++++ app = await startApp('img', env, app.port);++ let done;++ for (let i = 0; i < 100; i++) {++ done = await (await fetch(`${app.base}/api/v1/jobs/${job.id}`, { headers: { cookie } })).json();++ if (done.state === 'succeeded' || done.state === 'failed') break;++ await sleep(100);++ }++ assert.strictEqual(done.state, 'succeeded', JSON.stringify(done.error));++ assert.strictEqual(done.result.data.output.ext, 'webp');++ assert.strictEqual(done.result.files[0].name, 'red.webp');++ r = await fetch(`${app.base}${done.result.files[0].url}`, { headers: { cookie } });++ const bytes = Buffer.from(await r.arrayBuffer());++ assert.strictEqual(bytes.slice(8, 12).toString(), 'WEBP', 'the result is a WebP');++ assert.strictEqual((await sharp(bytes).metadata()).width, 64);++ r = await fetch(`${app.base}/api/v1/jobs/${job.id}`);++ assert.strictEqual(r.status, 404, 'another browser cannot see it');++++ // A format host fills in its format: through the gateway, the jpg tool converts to JPG.++ const fd2 = new FormData();++ fd2.append('type', 'img.process');++ fd2.append('file', new Blob([png], { type: 'image/png' }), 'x.png');++ r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd2, headers: { cookie, 'X-OV-Tool': 'jpg' } });++ const hosted = await r.json();++ for (let i = 0; i < 100 && hosted.state !== 'succeeded'; i++) { await sleep(100); Object.assign(hosted, await (await fetch(`${app.base}/api/v1/jobs/${hosted.id}`, { headers: { cookie } })).json()); }++ assert.strictEqual(hosted.result.data.output.ext, 'jpg');++++ // Errors on the job API are problem+json, including the upload middleware's.++ r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: (() => { const f = new FormData(); f.append('type', 'img.process'); return f; })() });++ assert.strictEqual(r.status, 400);++ assert.match(r.headers.get('content-type'), /problem\+json/);++ assert.strictEqual((await r.json()).code, 'tools.job.invalid');++++ // The synchronous endpoint is unchanged.++ const sync = new FormData();++ sync.append('tool', 'resize'); sync.append('width', '32');++ sync.append('file', new Blob([png], { type: 'image/png' }), 'red.png');++ r = await fetch(`${app.base}/api/process`, { method: 'POST', body: sync });++ const body = await r.json();++ assert.strictEqual(body.success, true);++ assert.match(body.download.downloadUrl, /^\/api\/download\/[a-f0-9]{32}$/);++ assert.strictEqual(body.dimensions.resized.width, 32);++++ // Canonical host: through the gateway the page names the host the gateway says.++ // (fetch cannot set Host, so pages are requested with http.request)++ const page = (headers) => new Promise((resolve, reject) => {++ http.get({ host: '127.0.0.1', port: app.port, path: '/', headers }, (res) => {++ let text = ''; res.setEncoding('utf8'); res.on('data', c => { text += c; });++ res.on('end', () => resolve({ status: res.statusCode, headers: { get: (k) => res.headers[k.toLowerCase()] }, text: async () => text }));++ }).on('error', reject);++ });++ let html = await (await page({ Host: 'png.openvibe.tools', 'X-OV-Tool': 'png', 'X-OV-Host-Role': 'short', 'X-OV-Canonical-Host': 'png-converter.example.com', 'X-OV-Short-Host': 'png.openvibe.tools' })).text();++ assert.ok(html.includes('<link rel="canonical" href="https://png-converter.example.com/">'), 'canonical follows X-OV-Canonical-Host');++ assert.ok(html.includes('"url":"https://png-converter.example.com/"'), 'so does the JSON-LD');++ assert.ok(/OpenVibePNG/.test(html), 'and the page is still the PNG tool');++ html = await (await page({ Host: 'png-converter.example.com', 'X-OV-Tool': 'png', 'X-OV-Host-Role': 'canonical', 'X-OV-Canonical-Host': 'png-converter.example.com' })).text();++ assert.ok(/OpenVibePNG/.test(html), 'a custom domain gets its tool through X-OV-Tool');++ html = await (await page({ Host: 'webp.openvibe.tools' })).text();++ assert.ok(html.includes('<link rel="canonical" href="https://webp.openvibe.tools/">'), 'reached directly, each host is its own canonical');++ r = await page({ Host: 'pngs.openvibe.tools', 'X-OV-Tool': 'png', 'X-OV-Host-Role': 'alias', 'X-OV-Canonical-Host': 'png-converter.example.com', 'X-OV-Short-Host': 'png.openvibe.tools' });++ assert.strictEqual(r.status, 301); assert.strictEqual(r.headers.get('location'), 'https://png.openvibe.tools/');++ r = await page({ Host: 'made-up.example.org' });++ assert.strictEqual(r.status, 301, 'a host this app does not serve goes to the tools index');++++ await app.kill('SIGTERM');++ console.log('img jobs + canonical host: all checks passed');++ } catch (err) {++ if (app) { console.error(app.output()); await app.kill('SIGKILL'); }++ throw err;++ } finally {++ fs.rmSync(data, { recursive: true, force: true });++ }++})().catch((err) => { console.error(err); process.exit(1); });@@@ -21,6 +21,13 @@ for app in apps/*/; dofidonedone++# Apps that run jobs (apps/_shared/jobs, required by relative path) hand it their own better-sqlite3 and++# openvibe-contracts: check both load under this Node (a native-module ABI mismatch shows up here, not in++# a crash loop) and that the shared runtime itself loads.++for app in img audio docs; do++ (cd "apps/$app" && node -e "const D=require('better-sqlite3'); new D(':memory:').close(); require('openvibe-contracts'); require('../_shared/jobs')") \++ || { echo "ABORT: apps/$app cannot load the jobs runtime; nothing restarted" >&2; exit 1; }++doneUNITS=$(systemctl list-unit-files 'openvibe-tools*' --no-legend | awk '{print $1}')sudo systemctl restart $UNITSsleep 5@@@ -2,8 +2,9 @@"name": "openvibe-tools","version": "1.0.0","private": true,-- "description": "OpenVibe.Tools — the tools suite on openvibe.tools and *.openvibe.tools",++ "description": "OpenVibe.Tools \u2014 the tools suite on openvibe.tools and *.openvibe.tools","scripts": {++ "test": "node test/run.js","install:all": "npm --prefix apps/gateway install && npm --prefix apps/maps install && npm --prefix apps/food install && npm --prefix apps/img install && npm --prefix apps/yt install && npm --prefix apps/audio install && npm --prefix apps/text install && npm --prefix apps/docs install","start:gateway": "npm --prefix apps/gateway start","start:maps": "npm --prefix apps/maps start",@@@ -14,5 +15,8 @@"start:text": "npm --prefix apps/text start","start:docs": "npm --prefix apps/docs start"},-- "license": "MIT"++ "license": "MIT",++ "engines": {++ "node": ">=22"++ }}nnew file mode 100644@@@ -0,0 +1,53 @@++'use strict';++// npm test — every app's tests (apps/*/test/*.test.js, apps/_shared/test/*.test.js), each in its own++// process from its app directory, after a syntax check of every server file. Run on Node 22.++const { spawnSync } = require('child_process');++const fs = require('fs');++const path = require('path');++++const ROOT = path.join(__dirname, '..');++const APPS = path.join(ROOT, 'apps');++const only = process.argv.slice(2);++++function walk(dir, out = []) {++ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {++ if (e.name === 'node_modules' || e.name === 'data' || e.name.startsWith('.')) continue;++ const p = path.join(dir, e.name);++ if (e.isDirectory()) walk(p, out); else if (e.name.endsWith('.js')) out.push(p);++ }++ return out;++}++++const apps = fs.readdirSync(APPS).filter(a => fs.statSync(path.join(APPS, a)).isDirectory()).sort();++++// 1. Syntax: every server-side file and the shared runtime.++let bad = 0;++const sources = apps.flatMap(a => (fs.existsSync(path.join(APPS, a, 'server')) ? walk(path.join(APPS, a, 'server')) : []))++ .concat(walk(path.join(APPS, '_shared')));++for (const f of sources) {++ const r = spawnSync(process.execPath, ['--check', f], { encoding: 'utf8' });++ if (r.status !== 0) { bad++; console.error(`syntax: ${path.relative(ROOT, f)}\n${r.stderr}`); }++}++console.log(`syntax: ${sources.length - bad}/${sources.length} files ok`);++++// 2. Tests.++const tests = apps.flatMap(a => {++ const dir = path.join(APPS, a, 'test');++ return fs.existsSync(dir) ? fs.readdirSync(dir).filter(f => f.endsWith('.test.js')).sort().map(f => ({ app: a, file: path.join(dir, f) })) : [];++}).filter(t => !only.length || only.some(o => t.file.includes(o)));++++const failed = [];++for (const t of tests) {++ const name = path.relative(ROOT, t.file);++ const t0 = Date.now();++ const r = spawnSync(process.execPath, [t.file], { cwd: path.join(APPS, t.app), encoding: 'utf8', timeout: 180_000, env: { ...process.env, NODE_ENV: 'test' } });++ const ms = Date.now() - t0;++ if (r.status === 0) {++ console.log(`ok ${name} (${ms} ms)${r.stdout.trim() ? ` — ${r.stdout.trim().split('\n').pop()}` : ''}`);++ } else {++ failed.push(name);++ console.error(`FAIL ${name} (${ms} ms)\n${r.stdout}${r.stderr}${r.error ? r.error.message : ''}`);++ }++}++console.log(`\n${tests.length - failed.length}/${tests.length} test files passed${failed.length ? `; failed: ${failed.join(', ')}` : ''}`);++process.exit(failed.length || bad ? 1 : 0);
new file mode 100644
new file mode 100644
@@ -0,0 +1,22 @@
@@ -0,0 +1,22 @@
+name: ci
+on:
+ push:
+ branches: [main]
+ pull_request:
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 22.22.1 # production runs Node 22
+ - name: ffmpeg (audio tests)
+ run: sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg
+ - run: npm run install:all
+ - run: npm test
+ - name: contracts check (satellites that use openvibe-contracts)
+ run: |
+ for app in img audio docs; do
+ (cd apps/$app && npx openvibe-contracts-check --service tools --src server --src ../_shared)
+ done
@@ -73,6 +73,50 @@ function canonicalUrlFor(req, fallbackHost, pathname) {
@@ -73,6 +73,50 @@ function canonicalUrlFor(req, fallbackHost, pathname) {
return `https://${canonicalHostFor(req, fallbackHost)}${p}`;
return `https://${canonicalHostFor(req, fallbackHost)}${p}`;
}
}
+/**
+ * Which host of the satellite's own domain map a request is for. Through the gateway the tool id
+ * decides (a descriptive host, a mirror or a custom domain is not in the satellite's map, but
+ * `<tool>.openvibe.tools` is); otherwise the Host header.
+ * @param {(host: string) => boolean} has is this hostname in the satellite's map?
+ */
+function ownHost(req, has) {
+ const info = req.ovHost || hostRole(req);
+ const byTool = info.tool ? `${info.tool}.openvibe.tools` : '';
+ if (byTool && has(byTool)) return byTool;
+ return requestHost(req);
+}
+
+/**
+ * Swap `https://<fromHost>` for `https://<toHost>` inside <head> — for static pages whose canonical,
+ * og:url and JSON-LD were written for one host (maps, food).
+ */
+function restampHead(html, fromHost, toHost) {
+ if (!fromHost || !toHost || fromHost === toHost) return html;
+ const end = html.search(/<\/head>/i);
+ if (end < 0) return html;
+ const from = new RegExp(`https://${fromHost.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=[/"'])`, 'g');
+ return html.slice(0, end).replace(from, `https://${toHost}`) + html.slice(end);
+}
+
+/**
+ * Handler for a static page written for one host: the canonical, og:url and JSON-LD in its <head>
+ * are rewritten to the canonical host of this request (the gateway's X-OV-Canonical-Host, or
+ * `ownHost` itself when reached directly). The file is read once.
+ */
+function stampedPage(file, ownHostName) {
+ let html = null;
+ const cache = new Map();
+ return function sendStampedPage(req, res) {
+ if (html === null) html = require('fs').readFileSync(file, 'utf8');
+ const canonical = canonicalHostFor(req, ownHostName);
+ if (!cache.has(canonical)) { if (cache.size > 50) cache.clear(); cache.set(canonical, restampHead(html, ownHostName, canonical)); }
+ res.setHeader('Content-Type', 'text/html; charset=utf-8');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.setHeader('Vary', 'X-OV-Canonical-Host');
+ return res.send(cache.get(canonical));
+ };
+}
+
/**
/**
* Express middleware. Sets `req.ovHost` and turns away hosts this satellite does not serve.
* Express middleware. Sets `req.ovHost` and turns away hosts this satellite does not serve.
*
*
@@ -117,4 +161,4 @@ function hostGuard(opts) {
@@ -117,4 +161,4 @@ function hostGuard(opts) {
};
};
}
}
-module.exports = { hostRole, hostGuard, canonicalHostFor, canonicalUrlFor, requestHost, isLocalHost, cleanHost, TOOLS_HOME };
+module.exports = { hostRole, hostGuard, canonicalHostFor, canonicalUrlFor, ownHost, restampHead, stampedPage, requestHost, isLocalHost, cleanHost, TOOLS_HOME };
new file mode 100644
new file mode 100644
@@ -0,0 +1,116 @@
@@ -0,0 +1,116 @@
+/* OpenVibe Tools — browser helper for /api/v1/jobs (served at /js/ov-jobs.js by every satellite that runs jobs).
+ *
+ * const job = await OVJobs.submit({ type: 'img.process', input: { tool: 'convert', format: 'webp' }, files: [file] });
+ * OVJobs.remember(job.id); // ?job=<id> in the address + sessionStorage: a reload reattaches
+ * OVJobs.watch(job.id, { onUpdate(job) {…}, onDone(job) {…} });
+ * const pending = OVJobs.recall(); // on load: the job to reattach to, if any
+ *
+ * Progress arrives over SSE (EventSource resumes with Last-Event-ID by itself); if the stream cannot be
+ * kept open the helper polls GET /api/v1/jobs/:id instead. Errors are problem+json; submit() throws an
+ * Error whose message is the problem's detail.
+ */
+(function () {
+ 'use strict';
+ var TERMINAL = { succeeded: 1, failed: 1, cancelled: 1 };
+ var EVENTS = ['job.queued', 'job.running', 'job.progress', 'job.cancel_requested', 'job.succeeded', 'job.failed', 'job.cancelled'];
+ var KEY = 'ov-job:' + location.host + location.pathname;
+
+ function newKey() {
+ if (window.crypto && crypto.randomUUID) return crypto.randomUUID();
+ return String(Date.now()) + '-' + Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2);
+ }
+ function problemError(res, body) {
+ var err = new Error((body && (body.detail || body.error || body.title)) || ('Server error (' + res.status + ')'));
+ err.status = res.status; err.problem = body || null;
+ return err;
+ }
+ function readJson(res) { return res.json().catch(function () { return null; }); }
+
+ function submit(o) {
+ var fd = new FormData();
+ fd.append('type', o.type);
+ fd.append('input', JSON.stringify(o.input || {}));
+ var files = o.files || [];
+ var field = o.fileField || (files.length > 1 ? 'files' : 'file');
+ for (var i = 0; i < files.length; i++) fd.append(field, files[i]);
+ return fetch('/api/v1/jobs', {
+ method: 'POST', body: fd, credentials: 'same-origin',
+ headers: { 'Idempotency-Key': o.idempotencyKey || newKey(), Accept: 'application/json' },
+ }).then(function (res) {
+ return readJson(res).then(function (body) { if (!res.ok) throw problemError(res, body); return body; });
+ });
+ }
+
+ function get(id) {
+ return fetch('/api/v1/jobs/' + encodeURIComponent(id), { credentials: 'same-origin', headers: { Accept: 'application/json' } })
+ .then(function (res) { return readJson(res).then(function (body) { if (!res.ok) throw problemError(res, body); return body; }); });
+ }
+
+ function cancel(id) {
+ return fetch('/api/v1/jobs/' + encodeURIComponent(id), { method: 'DELETE', credentials: 'same-origin', headers: { Accept: 'application/json' } })
+ .then(function (res) { return readJson(res).then(function (body) { if (!res.ok) throw problemError(res, body); return body; }); });
+ }
+
+ /** Follow a job until it ends. → { close() } */
+ function watch(id, h) {
+ h = h || {};
+ var done = false, es = null, timer = null, failures = 0;
+ function finish(job) {
+ if (done) return; done = true;
+ if (es) es.close();
+ if (timer) clearTimeout(timer);
+ if (h.onDone) h.onDone(job);
+ }
+ function update(job) {
+ if (done || !job) return;
+ if (h.onUpdate) h.onUpdate(job);
+ if (TERMINAL[job.state]) finish(job);
+ }
+ function poll() {
+ if (done) return;
+ get(id).then(function (job) { failures = 0; update(job); if (!done) timer = setTimeout(poll, 1500); })
+ .catch(function (err) {
+ if (err.status === 404) { done = true; if (h.onGone) h.onGone(err); return; }
+ failures++; timer = setTimeout(poll, Math.min(15000, 1500 * failures));
+ });
+ }
+ if (!window.EventSource) { poll(); return { close: function () { done = true; if (timer) clearTimeout(timer); } }; }
+ es = new EventSource('/api/v1/jobs/' + encodeURIComponent(id) + '/events', { withCredentials: true });
+ EVENTS.forEach(function (name) {
+ es.addEventListener(name, function (e) { try { update(JSON.parse(e.data)); } catch (_) { /* ignore a bad frame */ } });
+ });
+ es.onerror = function () {
+ // CLOSED: the server said 204 (nothing new — the job has ended) or refused; ask once, then poll.
+ if (!done && es.readyState === 2) { es = null; poll(); }
+ };
+ return { close: function () { done = true; if (es) es.close(); if (timer) clearTimeout(timer); } };
+ }
+
+ function remember(id) {
+ try { sessionStorage.setItem(KEY, id); } catch (_) { /* storage may be off */ }
+ try { var u = new URL(location.href); u.searchParams.set('job', id); history.replaceState(history.state, '', u.toString()); } catch (_) { /* old browser */ }
+ }
+ function recall() {
+ var fromUrl = null;
+ try { fromUrl = new URL(location.href).searchParams.get('job'); } catch (_) { /* old browser */ }
+ if (fromUrl && /^job_[0-9A-Z]{26}$/.test(fromUrl)) return fromUrl;
+ try { var s = sessionStorage.getItem(KEY); if (s && /^job_[0-9A-Z]{26}$/.test(s)) return s; } catch (_) { /* storage may be off */ }
+ return null;
+ }
+ function forget() {
+ try { sessionStorage.removeItem(KEY); } catch (_) { /* storage may be off */ }
+ try { var u = new URL(location.href); if (u.searchParams.has('job')) { u.searchParams.delete('job'); history.replaceState(history.state, '', u.toString()); } } catch (_) { /* old browser */ }
+ }
+
+ function fileUrl(job, n, opts) {
+ var f = job && job.result && job.result.files && job.result.files[n || 0];
+ if (!f) return null;
+ return f.url + (opts && opts.inline ? '?inline=1' : '');
+ }
+ /** Milliseconds until a finished job's files expire (null while it is still running). */
+ function expiresIn(job) {
+ return job && job.expires_at ? Math.max(0, Date.parse(job.expires_at) - Date.now()) : null;
+ }
+
+ window.OVJobs = { submit: submit, get: get, cancel: cancel, watch: watch, remember: remember, recall: recall, forget: forget, fileUrl: fileUrl, expiresIn: expiresIn, TERMINAL: TERMINAL };
+})();
new file mode 100644
new file mode 100644
@@ -0,0 +1,261 @@
@@ -0,0 +1,261 @@
+'use strict';
+// ═══════════════════════════════════════════════════════════════
+// Jobs over HTTP — the same routes on every satellite that runs jobs.
+//
+// POST /api/v1/jobs submit (JSON { type, input, idempotency_key } or multipart:
+// type, input (JSON text), file/files; Idempotency-Key header)
+// → 202 + Location (new) | 200 + Idempotent-Replayed: true
+// GET /api/v1/jobs/:id the job (reattach by id)
+// DELETE /api/v1/jobs/:id cancel → 200 cancelled | 202 cancel requested (running)
+// GET /api/v1/jobs/:id/events SSE: job.queued|running|progress|cancel_requested|succeeded|failed|cancelled;
+// every event has an id; reconnecting with Last-Event-ID replays
+// only what came after it; 204 once there is nothing left to say
+// GET /api/v1/jobs/:id/files/:n a result file (attachment; ?inline=1 for previews)
+// GET /js/ov-jobs.js the browser helper (submit, watch, reattach)
+//
+// Owner scoping: a job is visible only to whoever created it —
+// a Network user (user:usr_…, from the ov_token subject_id), a service/app principal presenting a
+// Network client-credentials token for audience openvibe.tools with tools.job.* capabilities, or,
+// for everyone else, this browser's jobs session cookie. Anyone else gets the same 404 as for a
+// job that does not exist. Errors are RFC 9457 problem+json.
+// ═══════════════════════════════════════════════════════════════
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { Readable } = require('stream');
+
+const ID_RE = /^job_[0-9A-HJKMNP-TV-Z]{26}$/;
+const SESSION_RE = /^[A-Za-z0-9_-]{32,64}$/;
+const TERMINAL = new Set(['succeeded', 'failed', 'cancelled']);
+const CLIENT_JS = path.join(__dirname, 'client.js');
+const CAPS = { create: 'tools.job.create', read: 'tools.job.read', cancel: 'tools.job.cancel' };
+
+const b64json = (part) => { try { return JSON.parse(Buffer.from(String(part).replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8')); } catch { return null; } };
+
+function contentDisposition(kind, name) {
+ const ascii = String(name || 'result').replace(/[^\x20-\x7e]/g, '_').replace(/["\\]/g, '_');
+ return `${kind}; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(String(name || 'result'))}`;
+}
+
+/**
+ * Who is asking. → { owner, kind: 'principal'|'user'|'session', claims? } | { error: [status, code, detail] } | null
+ *
+ * @param {object} o
+ * @param {object} o.contracts
+ * @param {() => string|null} o.getPublicKey the Network public key PEM the app already loads
+ * @param {string} o.issuer the Network issuer URL
+ * @param {string} [o.audience='openvibe.tools']
+ * @param {string} [o.cookieName='ov_tools_jobs']
+ * @param {boolean} [o.secureCookie]
+ */
+function createOwnerResolver(o) {
+ const audience = o.audience || 'openvibe.tools';
+ const cookieName = o.cookieName || 'ov_tools_jobs';
+ const { contracts } = o;
+
+ function principalCapability(claims, cap) {
+ const c = contracts.capabilities.check(claims, cap);
+ if (c.allowed) return null;
+ // The tools.job.* manifests are proposed for the next contracts release; until the installed
+ // version knows them, an explicit grant in the token (which only Network can sign) is enough.
+ if (c.code === 'capability.unknown' && contracts.capabilities.grants(claims.cap, cap)) return null;
+ return [403, c.code === 'capability.unknown' ? 'capability.denied' : c.code, `${cap} not granted`];
+ }
+
+ return function resolveOwner(req, res, { create = false, action = 'read' } = {}) {
+ const auth = String(req.headers.authorization || '');
+ if (auth.startsWith('Bearer ')) {
+ const token = auth.slice(7).trim();
+ const parts = token.split('.');
+ const claims = parts.length === 3 ? b64json(parts[1]) : null;
+ const looksLikePrincipal = claims && (Array.isArray(claims.cap) || /^(svc|app|mod):/.test(String(claims.sub || '')));
+ if (looksLikePrincipal) {
+ const r = contracts.serviceAuth.verifyServiceToken(token, { publicKey: o.getPublicKey(), issuer: o.issuer, audience });
+ if (!r.ok) return { error: [401, r.code, r.reason] };
+ const denied = principalCapability(r.claims, CAPS[action] || CAPS.read);
+ if (denied) return { error: denied };
+ return { owner: r.claims.sub, kind: 'principal', claims: r.claims };
+ }
+ }
+ const sid = req.user && req.user.subject_id;
+ if (sid && contracts.ids.isSubjectId('user', sid)) return { owner: `user:${sid}`, kind: 'user' };
+
+ let session = req.cookies && req.cookies[cookieName];
+ if (!SESSION_RE.test(String(session || ''))) {
+ if (!create) return null;
+ session = crypto.randomBytes(24).toString('base64url');
+ res.cookie(cookieName, session, {
+ httpOnly: true, sameSite: 'lax', path: '/', maxAge: 7 * 24 * 60 * 60 * 1000,
+ secure: o.secureCookie != null ? o.secureCookie : process.env.NODE_ENV === 'production',
+ });
+ }
+ // Only a hash of the cookie is stored, so the database cannot be used to take over a session.
+ return { owner: `session:${crypto.createHash('sha256').update(session).digest('hex').slice(0, 40)}`, kind: 'session' };
+ };
+}
+
+/**
+ * Mount the job routes on an Express app.
+ *
+ * @param {object} app
+ * @param {object} o
+ * @param {object} o.system createJobSystem(...)
+ * @param {object} o.contracts
+ * @param {function} o.resolveOwner createOwnerResolver(...)
+ * @param {function} [o.receive] multipart middleware that sets req.file / req.files (the app's multer)
+ * @param {function[]} [o.limiters] middleware run before accepting a submit (rate limits)
+ * @param {(req, who) => number} [o.ttlMs] how long a finished job is kept
+ * @param {(req, input) => object} [o.defaults] fills input from the host (e.g. the format a png.* host implies)
+ */
+function mountJobRoutes(app, o) {
+ const { system, contracts } = o;
+ const send = (res, status, code, detail, extra) => contracts.http.sendProblem(res, status, code, { detail, extra, ctx: res.req && res.req.ov });
+ const ttlMs = o.ttlMs || ((_req, who) => (who.kind === 'session' ? 60 * 60 * 1000 : 24 * 60 * 60 * 1000));
+ const noStore = (res) => res.set('Cache-Control', 'no-store');
+
+ function who(req, res, opts) {
+ const w = o.resolveOwner(req, res, opts);
+ if (w && w.error) { send(res, ...w.error); return null; }
+ return w || { owner: null };
+ }
+ /** The job, if it exists AND belongs to the caller; otherwise the same 404 either way. */
+ function load(req, res, action) {
+ const w = who(req, res, { action });
+ if (!w) return null;
+ const id = String(req.params.id || '');
+ const row = ID_RE.test(id) ? system.get(id) : null;
+ if (!row || !w.owner || row.owner !== w.owner) { send(res, 404, 'tools.job.not_found', 'No such job'); return null; }
+ return row;
+ }
+
+ app.get('/js/ov-jobs.js', (_req, res) => {
+ res.set({ 'Content-Type': 'application/javascript; charset=utf-8', 'Cache-Control': 'no-cache' });
+ res.sendFile(CLIENT_JS);
+ });
+
+ const receive = o.receive || ((_req, _res, next) => next());
+ // The apps' upload middleware answers { error } (their legacy shape); on this API those become problem+json.
+ const multipartOnly = (req, res, next) => {
+ if (!req.is('multipart/form-data')) return next();
+ const json = res.json.bind(res);
+ res.json = (body) => {
+ res.json = json;
+ if (res.statusCode >= 400 && body && body.error && !body.code) return send(res, res.statusCode, res.statusCode === 413 ? 'tools.job.too_large' : 'tools.job.invalid', String(body.error));
+ return json(body);
+ };
+ receive(req, res, (err) => { res.json = json; next(err); });
+ };
+
+ app.post('/api/v1/jobs', ...(o.limiters || []), multipartOnly, async (req, res) => {
+ noStore(res);
+ const w = who(req, res, { create: true, action: 'create' });
+ if (!w) return;
+ const b = req.body || {};
+ let input = b.input;
+ if (typeof input === 'string') {
+ try { input = input.trim() ? JSON.parse(input) : {}; } catch { return send(res, 400, 'tools.job.invalid', 'input must be JSON'); }
+ }
+ if (input == null) input = {};
+ if (typeof input !== 'object' || Array.isArray(input)) return send(res, 400, 'tools.job.invalid', 'input must be a JSON object');
+ if (o.defaults) input = o.defaults(req, input) || input;
+ const list = Array.isArray(req.files) ? req.files : req.files && typeof req.files === 'object' ? Object.values(req.files).flat() : req.file ? [req.file] : [];
+ const files = list.map(f => ({ path: f.path, buffer: f.path ? undefined : f.buffer, name: f.originalname, mime: f.mimetype, size: f.size }));
+ const key = req.headers['idempotency-key'] != null ? String(req.headers['idempotency-key']) : (b.idempotency_key != null ? String(b.idempotency_key) : null);
+ try {
+ const { job, replayed } = await system.submit({ owner: w.owner, type: String(b.type || ''), input, files, idempotencyKey: key, ttlMs: ttlMs(req, w) });
+ res.set('Location', `/api/v1/jobs/${job.id}`);
+ if (replayed) res.set('Idempotent-Replayed', 'true');
+ return res.status(replayed ? 200 : 202).json(system.view(job));
+ } catch (err) {
+ if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);
+ console.error('[Jobs] submit failed:', err.message);
+ return send(res, 500, 'tools.job.submit_failed', 'The job could not be accepted');
+ }
+ });
+
+ app.get('/api/v1/jobs/:id', (req, res) => {
+ noStore(res);
+ const row = load(req, res, 'read');
+ if (row) res.json(system.view(row));
+ });
+
+ app.delete('/api/v1/jobs/:id', (req, res) => {
+ noStore(res);
+ const row = load(req, res, 'cancel');
+ if (!row) return;
+ if (row.state === 'succeeded' || row.state === 'failed') return send(res, 409, 'tools.job.already_finished', `The job already ${row.state}`, { state: row.state });
+ const { job } = system.cancel(row.id);
+ return res.status(job.state === 'cancelled' ? 200 : 202).json(system.view(job));
+ });
+
+ app.get('/api/v1/jobs/:id/events', (req, res) => {
+ const row = load(req, res, 'read');
+ if (!row) return;
+ const raw = req.headers['last-event-id'] != null ? req.headers['last-event-id'] : req.query.last_event_id;
+ const after = Math.max(0, parseInt(raw, 10) || 0);
+ const pending = system.eventsAfter(row.id, after);
+ // Finished and nothing new since the client's last event: 204 tells EventSource to stop reconnecting.
+ if (TERMINAL.has(row.state) && !pending.length) return res.status(204).set('Cache-Control', 'no-store').end();
+
+ res.writeHead(200, {
+ 'Content-Type': 'text/event-stream; charset=utf-8',
+ 'Cache-Control': 'no-store, no-transform',
+ Connection: 'keep-alive',
+ 'X-Accel-Buffering': 'no',
+ });
+ res.write('retry: 3000\n\n');
+ let last = after, closed = false, replaying = true;
+ const queue = [];
+ const close = () => { if (closed) return; closed = true; unsubscribe(); offStop(); clearInterval(keepAlive); res.end(); };
+ // On shutdown the stream ends; EventSource reconnects to the next process with Last-Event-ID.
+ const offStop = system.onStop(close);
+ const write = (e) => {
+ if (closed || e.seq <= last) return;
+ last = e.seq;
+ res.write(`id: ${e.seq}\nevent: ${e.event}\ndata: ${JSON.stringify(e.data)}\n\n`);
+ if (TERMINAL.has(e.data.state) && e.event !== 'job.progress') setImmediate(close);
+ };
+ // Subscribe before replaying so nothing emitted in between is lost; the seq check drops duplicates.
+ const unsubscribe = system.subscribe(row.id, (e) => (replaying ? queue.push(e) : write(e)));
+ for (const e of system.eventsAfter(row.id, after)) write(e);
+ replaying = false;
+ queue.forEach(write);
+ const keepAlive = setInterval(() => { if (!closed) res.write(': keep-alive\n\n'); }, 15000);
+ if (keepAlive.unref) keepAlive.unref();
+ req.on('close', close);
+ });
+
+ app.get('/api/v1/jobs/:id/files/:n', async (req, res) => {
+ const row = load(req, res, 'read');
+ if (!row) return;
+ if (row.state !== 'succeeded') return send(res, 409, 'tools.job.not_ready', `The job is ${row.state}`);
+ const f = system.resultFile(row, parseInt(req.params.n, 10));
+ if (!f) return send(res, 404, 'tools.job.file_not_found', 'No such result file');
+ const inline = ['1', 'true'].includes(String(req.query.inline || ''));
+ res.set('Cache-Control', 'private, no-store');
+ res.set('X-Content-Type-Options', 'nosniff');
+ if (f.storage === 'media' && f.media && system.media) {
+ try {
+ const signed = await system.media.downloadUrl(f.media.media_id);
+ // Previews load straight from Media (range requests, no bytes through this server);
+ // downloads come through here so the browser saves them under the right name.
+ if (inline) return res.redirect(302, signed.url);
+ const up = await fetch(signed.internal_url, { signal: AbortSignal.timeout(120_000) });
+ if (!up.ok || !up.body) return send(res, 502, 'tools.job.media_unavailable', 'OpenVibe.Media could not return the file');
+ res.set({ 'Content-Type': f.mime, 'Content-Disposition': contentDisposition('attachment', f.name) });
+ const len = up.headers.get('content-length');
+ if (len) res.set('Content-Length', len);
+ return Readable.fromWeb(up.body).on('error', () => res.destroy()).pipe(res);
+ } catch (err) {
+ console.error('[Jobs] media download failed:', err.message);
+ return send(res, 502, 'tools.job.media_unavailable', 'OpenVibe.Media could not return the file');
+ }
+ }
+ if (!f.path || !fs.existsSync(f.path)) return send(res, 410, 'tools.job.file_gone', 'The result file has expired');
+ res.set({ 'Content-Type': f.mime, 'Content-Disposition': contentDisposition(inline ? 'inline' : 'attachment', f.name) });
+ return res.sendFile(f.path);
+ });
+}
+
+module.exports = { mountJobRoutes, createOwnerResolver, contentDisposition, CAPS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,95 @@
@@ -0,0 +1,95 @@
+'use strict';
+// ═══════════════════════════════════════════════════════════════
+// Tools job runtime (roadmap Wave 11). Shared by the satellites the way apps/_shared always is:
+// required by relative path, no dependencies of its own — each app passes its better-sqlite3 and
+// openvibe-contracts. See system.js (lifecycle), http.js (routes), media.js (results in Media),
+// client.js (browser helper).
+//
+// Environment (one /etc/openvibe/tools.env for every unit):
+// TOOLS_JOBS_CONCURRENCY jobs running at once per satellite (default 2)
+// TOOLS_JOBS_CONCURRENCY_<APP> per-satellite override, e.g. TOOLS_JOBS_CONCURRENCY_AUDIO=1
+// TOOLS_JOBS_MAX_ACTIVE unfinished jobs per owner (default 10)
+// TOOLS_JOB_RESULTS local (default) | media — where result files go
+// TOOLS_MEDIA_NAMESPACE Media namespace for results (default tools)
+// OV_MEDIA_INTERNAL_URL Media on this host (default http://127.0.0.1:4100; MEDIA_URL is read as a fallback)
+// OV_MEDIA_URL Media's public origin, for the CSP of previews (default https://openvibe.media)
+// OV_NETWORK_INTERNAL_URL token endpoint host (default http://127.0.0.1:4000)
+// OV_OAUTH_CLIENT_ID / OV_OAUTH_CLIENT_SECRET the tools client (client_credentials for Media)
+// ═══════════════════════════════════════════════════════════════
+
+const fs = require('fs');
+const path = require('path');
+const { createJobSystem, JobError } = require('./system');
+const { mountJobRoutes, createOwnerResolver, contentDisposition, CAPS } = require('./http');
+const { createMediaResults } = require('./media');
+
+function envInt(name, fallback) {
+ const v = parseInt(process.env[name], 10);
+ return Number.isFinite(v) && v >= 0 ? v : fallback;
+}
+
+/** Result storage from the environment → { media, mode, reason } (media is null for local). */
+function mediaFromEnv(contracts, env = process.env) {
+ const want = String(env.TOOLS_JOB_RESULTS || 'local').toLowerCase();
+ if (want !== 'media') return { media: null, mode: 'local', reason: 'TOOLS_JOB_RESULTS is not "media"' };
+ const secret = env.OV_OAUTH_CLIENT_SECRET || '';
+ if (!secret) return { media: null, mode: 'local', reason: 'TOOLS_JOB_RESULTS=media but OV_OAUTH_CLIENT_SECRET is not set' };
+ const network = (env.OV_NETWORK_INTERNAL_URL || 'http://127.0.0.1:4000').replace(/\/+$/, '');
+ const tokens = contracts.serviceAuth.createTokenClient({
+ tokenUrl: `${network}/oauth/token`,
+ clientId: env.OV_OAUTH_CLIENT_ID || 'tools',
+ clientSecret: secret,
+ audience: 'openvibe.media',
+ scope: 'media.object.upload media.object.read',
+ });
+ const media = createMediaResults({
+ internalUrl: env.OV_MEDIA_INTERNAL_URL || env.MEDIA_URL || 'http://127.0.0.1:4100',
+ namespace: env.TOOLS_MEDIA_NAMESPACE || 'tools',
+ tokens,
+ });
+ return { media, mode: 'media', reason: null };
+}
+
+/** Media's public origin (previews are 302s to it, so the CSP must allow it). */
+function mediaOrigin(env = process.env) {
+ try { return new URL(env.OV_MEDIA_URL || env.MEDIA_PUBLIC_URL || 'https://openvibe.media').origin; } catch { return 'https://openvibe.media'; }
+}
+
+/**
+ * One call per satellite: open <dataDir>/jobs.db, build the system, let the app define its job
+ * types, mount the routes and start (boot recovery happens here).
+ *
+ * @param {object} o
+ * @param {object} o.app Express app
+ * @param {string} o.service 'img' | 'audio' | 'docs'
+ * @param {string} o.dataDir
+ * @param {Function} o.Database require('better-sqlite3')
+ * @param {object} o.contracts require('openvibe-contracts')
+ * @param {() => string|null} o.getPublicKey
+ * @param {string} o.issuer
+ * @param {(system) => void} o.define registers the app's job types
+ * @param {Function} [o.receive] multipart middleware (the app's multer)
+ * @param {Function[]} [o.limiters]
+ * @param {Function} [o.defaults] (req, input) → input with host defaults
+ */
+function setupJobs(o) {
+ fs.mkdirSync(o.dataDir, { recursive: true });
+ const db = new o.Database(path.join(o.dataDir, 'jobs.db'));
+ const results = mediaFromEnv(o.contracts);
+ if (results.reason && String(process.env.TOOLS_JOB_RESULTS || '').toLowerCase() === 'media') console.warn(`[Jobs] ${o.service}: results stay local — ${results.reason}`);
+ const system = createJobSystem({
+ db, contracts: o.contracts, service: o.service, dataDir: o.dataDir,
+ concurrency: envInt(`TOOLS_JOBS_CONCURRENCY_${o.service.toUpperCase()}`, envInt('TOOLS_JOBS_CONCURRENCY', 2)),
+ maxActivePerOwner: envInt('TOOLS_JOBS_MAX_ACTIVE', 10) || 10,
+ media: results.media,
+ });
+ o.define(system);
+ const resolveOwner = createOwnerResolver({ contracts: o.contracts, getPublicKey: o.getPublicKey, issuer: o.issuer });
+ mountJobRoutes(o.app, { system, contracts: o.contracts, resolveOwner, receive: o.receive, limiters: o.limiters, defaults: o.defaults });
+ system.start();
+ system.db = db;
+ system.close = () => { system.stop(); try { db.close(); } catch { /* already closed */ } };
+ return system;
+}
+
+module.exports = { setupJobs, createJobSystem, mountJobRoutes, createOwnerResolver, createMediaResults, mediaFromEnv, mediaOrigin, contentDisposition, JobError, CAPS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,115 @@
@@ -0,0 +1,115 @@
+'use strict';
+// ═══════════════════════════════════════════════════════════════
+// Job results as OpenVibe.Media objects (Media object API v2, roadmap Wave 4).
+//
+// POST {media}/api/v2/tools/objects init → { id: med_…, upload: { url, token } }
+// PUT {upload.url} the bytes (upload token in the URL)
+// POST {media}/api/v2/tools/objects/:id/complete verify sha256 → lifecycle ready
+// GET {media}/api/v2/tools/objects/:id/download?format=json → short-lived signed URL
+// DELETE {media}/api/v2/tools/objects/:id when the job is pruned
+//
+// Auth: a Network client-credentials token (audience openvibe.media) granting
+// media.object.upload and media.object.read for namespace "tools". The owner goes in
+// X-OV-Subject when the job belongs to a signed-in person (user:usr_…); guest and session
+// results are service-owned objects in the tools namespace. Objects are private.
+//
+// Calls go to the internal Media URL; Media's upload URL (built on its public origin) is
+// rewritten onto it, so bytes never leave the host.
+// ═══════════════════════════════════════════════════════════════
+
+const fs = require('fs');
+const { Readable } = require('stream');
+
+/**
+ * @param {object} o
+ * @param {string} o.internalUrl e.g. http://127.0.0.1:4100
+ * @param {string} [o.namespace='tools']
+ * @param {object} o.tokens contracts.serviceAuth.createTokenClient(...) (audience openvibe.media)
+ * @param {function} [o.fetchImpl]
+ * @param {number} [o.timeoutMs=120000]
+ */
+function createMediaResults(o) {
+ const base = String(o.internalUrl || '').replace(/\/+$/, '');
+ if (!base) throw new TypeError('createMediaResults needs internalUrl');
+ const ns = encodeURIComponent(o.namespace || 'tools');
+ const fetchImpl = o.fetchImpl || globalThis.fetch;
+ const timeoutMs = o.timeoutMs || 120_000;
+ const api = `${base}/api/v2/${ns}/objects`;
+
+ async function call(url, init = {}, retried = false) {
+ const headers = { Accept: 'application/json', ...(init.headers || {}), ...(await o.tokens.authHeaders()) };
+ const res = await fetchImpl(url, { ...init, headers, signal: AbortSignal.timeout(timeoutMs) });
+ if (res.status === 401 && !retried && (init.body == null || typeof init.body === 'string')) { o.tokens.invalidate(); return call(url, init, true); }
+ return res;
+ }
+ async function json(res, what) {
+ const body = await res.json().catch(() => null);
+ if (!res.ok) {
+ const err = new Error(`Media ${what} ${res.status}: ${(body && (body.detail || body.error || body.code)) || 'no body'}`);
+ err.status = res.status;
+ err.code = body && body.code;
+ throw err;
+ }
+ return body;
+ }
+ /** Media's upload URL is on its public origin; send the bytes to the internal one. */
+ function internal(url) {
+ const u = new URL(url);
+ const b = new URL(base);
+ u.protocol = b.protocol; u.host = b.host;
+ return u.toString();
+ }
+ const subjectHeader = (owner) => {
+ const m = /^user:(usr_[0-9A-HJKMNP-TV-Z]{26})$/.exec(String(owner || ''));
+ return m ? { 'X-OV-Subject': m[1] } : {};
+ };
+
+ /** Store one file → MediaRef { media_id, role, namespace, size_bytes, content_hash, mime_type }. */
+ async function upload({ path: file, name, mime, size, sha256, owner, jobId, service, type }) {
+ const init = await json(await call(api, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', ...subjectHeader(owner) },
+ body: JSON.stringify({
+ kind: 'file', visibility: 'private', size_bytes: size, mime_type: mime, filename: name, content_hash: sha256,
+ metadata: { source: 'openvibe.tools', service, job_id: jobId, job_type: type },
+ }),
+ }), 'init');
+ const id = init && init.id;
+ if (!id || !init.upload || !init.upload.url) throw new Error('Media init answered without an upload URL');
+ try {
+ const put = await fetchImpl(internal(init.upload.url), {
+ method: 'PUT',
+ headers: { 'Content-Type': mime || 'application/octet-stream', 'Content-Length': String(size) },
+ body: Readable.toWeb(fs.createReadStream(file)),
+ duplex: 'half',
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+ await json(put, 'content');
+ const done = await json(await call(`${api}/${encodeURIComponent(id)}/complete`, {
+ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ content_hash: sha256 }),
+ }), 'complete');
+ return { media_id: id, role: 'output', namespace: o.namespace || 'tools', size_bytes: size, content_hash: sha256, mime_type: mime, status: done.lifecycle_status || 'ready' };
+ } catch (err) {
+ await remove(id).catch(() => {});
+ throw err;
+ }
+ }
+
+ /** A short-lived signed URL for the private object → { url, expires_at }. */
+ async function downloadUrl(mediaId, ttlSec = 300) {
+ const body = await json(await call(`${api}/${encodeURIComponent(mediaId)}/download?format=json&ttl=${ttlSec}`), 'download');
+ if (!body || !body.url) throw new Error('Media download answered without a URL');
+ return { url: body.url, expires_at: body.expires_at || null, internal_url: internal(body.url) };
+ }
+
+ async function remove(mediaId) {
+ const res = await call(`${api}/${encodeURIComponent(mediaId)}`, { method: 'DELETE' });
+ if (res.status === 404) return false;
+ await json(res, 'delete');
+ return true;
+ }
+
+ return { upload, downloadUrl, remove, namespace: o.namespace || 'tools' };
+}
+
+module.exports = { createMediaResults };
new file mode 100644
new file mode 100644
@@ -0,0 +1,121 @@
@@ -0,0 +1,121 @@
+'use strict';
+// ═══════════════════════════════════════════════════════════════
+// Job store — the satellite's own SQLite (better-sqlite3 handle passed in).
+//
+// Two tables, created on first use (idempotent):
+// tool_jobs one row per job; the durable truth a restart recovers from
+// tool_job_events the ordered event log per job; its AUTOINCREMENT seq is the SSE event id,
+// so a reconnect with Last-Event-ID replays exactly what was missed
+// ═══════════════════════════════════════════════════════════════
+
+const TERMINAL = ['succeeded', 'failed', 'cancelled'];
+const STATES = ['queued', 'running', ...TERMINAL];
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS tool_jobs (
+ id TEXT PRIMARY KEY,
+ type TEXT NOT NULL,
+ type_version INTEGER NOT NULL DEFAULT 1,
+ owner TEXT NOT NULL,
+ state TEXT NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled')),
+ progress REAL,
+ progress_message TEXT,
+ input_json TEXT NOT NULL DEFAULT '{}',
+ files_json TEXT NOT NULL DEFAULT '[]',
+ result_json TEXT,
+ error_json TEXT,
+ retryable INTEGER NOT NULL DEFAULT 0,
+ idempotency_key TEXT,
+ request_hash TEXT,
+ attempts INTEGER NOT NULL DEFAULT 0,
+ max_attempts INTEGER NOT NULL DEFAULT 1,
+ cancel_requested INTEGER NOT NULL DEFAULT 0,
+ ttl_ms INTEGER NOT NULL DEFAULT 3600000,
+ created_at INTEGER NOT NULL,
+ updated_at INTEGER NOT NULL,
+ started_at INTEGER,
+ finished_at INTEGER,
+ expires_at INTEGER
+);
+CREATE UNIQUE INDEX IF NOT EXISTS tool_jobs_idem ON tool_jobs(owner, idempotency_key) WHERE idempotency_key IS NOT NULL;
+CREATE INDEX IF NOT EXISTS tool_jobs_state ON tool_jobs(state, id);
+CREATE INDEX IF NOT EXISTS tool_jobs_expiry ON tool_jobs(expires_at) WHERE expires_at IS NOT NULL;
+CREATE INDEX IF NOT EXISTS tool_jobs_owner ON tool_jobs(owner, state);
+CREATE TABLE IF NOT EXISTS tool_job_events (
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
+ job_id TEXT NOT NULL,
+ event TEXT NOT NULL,
+ data TEXT NOT NULL,
+ at INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS tool_job_events_job ON tool_job_events(job_id, seq);
+`;
+
+const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };
+
+function createStore(db) {
+ db.pragma('journal_mode = WAL');
+ db.pragma('busy_timeout = 5000');
+ db.exec(SCHEMA);
+
+ const q = {
+ insert: db.prepare(`INSERT INTO tool_jobs (id, type, type_version, owner, state, input_json, files_json, idempotency_key, request_hash, max_attempts, ttl_ms, created_at, updated_at)
+ VALUES (@id, @type, @type_version, @owner, 'queued', @input_json, @files_json, @idempotency_key, @request_hash, @max_attempts, @ttl_ms, @now, @now)`),
+ get: db.prepare('SELECT * FROM tool_jobs WHERE id = ?'),
+ byIdem: db.prepare('SELECT * FROM tool_jobs WHERE owner = ? AND idempotency_key = ?'),
+ nextQueued: db.prepare("SELECT * FROM tool_jobs WHERE state = 'queued' ORDER BY id LIMIT ?"),
+ claim: db.prepare("UPDATE tool_jobs SET state = 'running', attempts = attempts + 1, started_at = @now, updated_at = @now, progress = NULL, progress_message = NULL WHERE id = @id AND state = 'queued'"),
+ running: db.prepare("SELECT * FROM tool_jobs WHERE state = 'running'"),
+ activeForOwner: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE owner = ? AND state IN ('queued','running')"),
+ progress: db.prepare('UPDATE tool_jobs SET progress = @progress, progress_message = @message, updated_at = @now WHERE id = @id AND state = \'running\''),
+ requestCancel: db.prepare("UPDATE tool_jobs SET cancel_requested = 1, updated_at = @now WHERE id = @id AND state IN ('queued','running')"),
+ requeue: db.prepare("UPDATE tool_jobs SET state = 'queued', started_at = NULL, progress = NULL, progress_message = NULL, updated_at = @now WHERE id = @id AND state = 'running'"),
+ finish: db.prepare(`UPDATE tool_jobs SET state = @state, result_json = @result_json, error_json = @error_json, retryable = @retryable,
+ progress = CASE WHEN @state = 'succeeded' THEN 100 ELSE progress END, progress_message = NULL,
+ finished_at = @now, updated_at = @now, expires_at = @now + ttl_ms WHERE id = @id AND state IN ('queued','running')`),
+ setFiles: db.prepare('UPDATE tool_jobs SET files_json = @files_json, updated_at = @now WHERE id = @id'),
+ event: db.prepare('INSERT INTO tool_job_events (job_id, event, data, at) VALUES (?, ?, ?, ?)'),
+ eventsAfter: db.prepare('SELECT seq, event, data, at FROM tool_job_events WHERE job_id = ? AND seq > ? ORDER BY seq'),
+ lastSeq: db.prepare('SELECT MAX(seq) AS seq FROM tool_job_events WHERE job_id = ?'),
+ expired: db.prepare('SELECT * FROM tool_jobs WHERE expires_at IS NOT NULL AND expires_at <= ? LIMIT 500'),
+ del: db.prepare('DELETE FROM tool_jobs WHERE id = ?'),
+ delEvents: db.prepare('DELETE FROM tool_job_events WHERE job_id = ?'),
+ counts: db.prepare('SELECT state, COUNT(*) AS n FROM tool_jobs GROUP BY state'),
+ };
+
+ /** Append one event; returns its seq (the SSE id). */
+ function appendEvent(jobId, event, data, now = Date.now()) {
+ return Number(q.event.run(jobId, event, JSON.stringify(data), now).lastInsertRowid);
+ }
+
+ return {
+ db,
+ TERMINAL, STATES,
+ insert(row) { q.insert.run(row); },
+ get: (id) => q.get.get(id) || null,
+ byIdempotencyKey: (owner, key) => q.byIdem.get(owner, key) || null,
+ nextQueued: (limit) => q.nextQueued.all(limit),
+ claim: (id, now = Date.now()) => q.claim.run({ id, now }).changes === 1,
+ running: () => q.running.all(),
+ activeForOwner: (owner) => q.activeForOwner.get(owner).n,
+ setProgress: (id, progress, message, now = Date.now()) => q.progress.run({ id, progress, message, now }).changes === 1,
+ requestCancel: (id, now = Date.now()) => q.requestCancel.run({ id, now }).changes === 1,
+ requeue: (id, now = Date.now()) => q.requeue.run({ id, now }).changes === 1,
+ finish: (id, { state, result = null, error = null, retryable = false }, now = Date.now()) => q.finish.run({
+ id, state, now, retryable: retryable ? 1 : 0,
+ result_json: result == null ? null : JSON.stringify(result),
+ error_json: error == null ? null : JSON.stringify(error),
+ }).changes === 1,
+ setFiles: (id, files, now = Date.now()) => q.setFiles.run({ id, files_json: JSON.stringify(files), now }),
+ appendEvent,
+ eventsAfter: (jobId, seq) => q.eventsAfter.all(jobId, seq).map(e => ({ seq: e.seq, event: e.event, data: parse(e.data, {}), at: e.at })),
+ lastSeq: (jobId) => q.lastSeq.get(jobId).seq || 0,
+ expired: (now = Date.now()) => q.expired.all(now),
+ remove: db.transaction((id) => { q.delEvents.run(id); q.del.run(id); }),
+ counts() { const out = Object.fromEntries(STATES.map(s => [s, 0])); for (const r of q.counts.all()) out[r.state] = r.n; return out; },
+ transaction: (fn) => db.transaction(fn),
+ parse,
+ };
+}
+
+module.exports = { createStore, TERMINAL, STATES };
new file mode 100644
new file mode 100644
@@ -0,0 +1,468 @@
@@ -0,0 +1,468 @@
+'use strict';
+// ═══════════════════════════════════════════════════════════════
+// Asynchronous jobs for a Tools satellite (roadmap Wave 11, §15.15).
+//
+// accepted (202) → queued → running → succeeded | failed | cancelled
+//
+// Durable: a job is written to the satellite's SQLite, and its input files moved under
+// <dataDir>/jobs/<id>/in/, BEFORE the 202 goes out. On boot, rows left 'running' by a restart
+// are re-queued (onRestart: 'requeue', while attempts remain) or failed with retryable = true
+// (onRestart: 'fail'), per job type. Queued rows simply start again.
+//
+// Bounded: at most `concurrency` jobs run at once in this process, and each owner may have at
+// most `maxActivePerOwner` queued + running jobs. Finished jobs expire (ttl chosen at submit),
+// and the pruner deletes the row, its events, its files and any Media objects it made.
+//
+// No dependencies of its own: the app passes its better-sqlite3 handle and openvibe-contracts.
+// ═══════════════════════════════════════════════════════════════
+
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const crypto = require('crypto');
+const { createStore, TERMINAL } = require('./store');
+
+const HOUR = 60 * 60 * 1000;
+const EVENT_FOR = { queued: 'job.queued', running: 'job.running', succeeded: 'job.succeeded', failed: 'job.failed', cancelled: 'job.cancelled' };
+const SAFE_NAME = (s) => String(s || 'file').replace(/[/\\\0]/g, '_').replace(/[^\w.\- ]/g, '_').slice(-120) || 'file';
+
+/** A failure the HTTP layer turns into problem+json. */
+class JobError extends Error {
+ constructor(status, code, detail, extra) { super(detail || code); this.status = status; this.code = code; this.detail = detail; this.extra = extra; }
+}
+
+/** Stable JSON: keys sorted, so the same input always hashes the same. */
+function canonical(v) {
+ if (Array.isArray(v)) return `[${v.map(canonical).join(',')}]`;
+ if (v && typeof v === 'object') return `{${Object.keys(v).sort().filter(k => v[k] !== undefined).map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(',')}}`;
+ return JSON.stringify(v === undefined ? null : v);
+}
+
+function sha256File(file) {
+ return new Promise((resolve, reject) => {
+ const h = crypto.createHash('sha256');
+ fs.createReadStream(file).on('data', c => h.update(c)).on('error', reject).on('end', () => resolve(h.digest('hex')));
+ });
+}
+
+async function moveFile(from, to) {
+ try { await fsp.rename(from, to); } catch { await fsp.copyFile(from, to); await fsp.unlink(from).catch(() => {}); }
+}
+
+/** Tool error text is meant for people; absolute server paths are not. */
+const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]').slice(0, 500);
+
+/**
+ * @param {object} o
+ * @param {object} o.db better-sqlite3 Database (the satellite's own jobs database)
+ * @param {object} o.contracts require('openvibe-contracts')
+ * @param {string} o.service 'img' | 'audio' | 'docs' | …
+ * @param {string} o.dataDir where job files live (<dataDir>/jobs/<id>/…)
+ * @param {number} [o.concurrency=2]
+ * @param {number} [o.maxActivePerOwner=10]
+ * @param {object} [o.media] result store from ./media (null = keep results on local disk)
+ * @param {number} [o.pruneIntervalMs=300000]
+ * @param {number} [o.progressThrottleMs=250]
+ */
+function createJobSystem(o) {
+ if (!o || !o.db || !o.contracts || !o.dataDir) throw new TypeError('createJobSystem needs db, contracts and dataDir');
+ const { contracts } = o;
+ const store = createStore(o.db);
+ const log = o.log || console;
+ const concurrency = Math.max(0, Number.isFinite(o.concurrency) ? o.concurrency : 2);
+ const maxActivePerOwner = Math.max(1, o.maxActivePerOwner || 10);
+ const media = o.media || null;
+ const throttleMs = o.progressThrottleMs == null ? 250 : o.progressThrottleMs;
+ const root = path.resolve(o.dataDir, 'jobs');
+ fs.mkdirSync(root, { recursive: true });
+
+ const types = new Map();
+ const active = new Map(); // id → { ctrl, reason, last: { pct, msg, at } }
+ const listeners = new Map(); // id → Set<fn>
+ const stopHooks = new Set(); // e.g. open SSE streams, ended on stop()
+ let started = false, stopped = false, pruneTimer = null, kicking = false;
+
+ const jobDir = (id) => path.join(root, id);
+ const problem = (status, code, detail, extra) => contracts.http.problem(status, code, { detail, extra });
+
+ // ── Types ──────────────────────────────────────────────────
+ /**
+ * define({ type, version, onRestart: 'requeue'|'fail', maxAttempts, timeoutMs, minFiles, maxFiles,
+ * validate(input, files) → string|null, run(ctx) → { files: [{ path, name, mime }], data } })
+ */
+ function define(def) {
+ if (!def || !/^[a-z][a-z0-9]*\.[a-z0-9_.]+$/.test(def.type || '')) throw new TypeError(`bad job type ${def && def.type}`);
+ if (typeof def.run !== 'function') throw new TypeError(`${def.type}: run() is required`);
+ types.set(def.type, {
+ version: 1, onRestart: 'requeue', maxAttempts: 3, timeoutMs: 10 * 60 * 1000, minFiles: 0, maxFiles: 1,
+ validate: () => null, ...def,
+ onRestart: def.onRestart === 'fail' ? 'fail' : 'requeue',
+ });
+ return api;
+ }
+
+ // ── Public representation ─────────────────────────────────
+ const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());
+ function publicResult(result, id) {
+ if (!result) return null;
+ return {
+ files: (result.files || []).map((f, i) => {
+ const out = {};
+ for (const [k, v] of Object.entries(f)) if (!k.startsWith('_')) out[k] = v;
+ out.url = `/api/v1/jobs/${id}/files/${i}`;
+ return out;
+ }),
+ data: result.data || {},
+ };
+ }
+ function view(row) {
+ if (!row) return null;
+ const live = row.state === 'queued' || row.state === 'running';
+ return {
+ id: row.id,
+ object: 'tools.job',
+ service: o.service,
+ type: row.type,
+ type_version: row.type_version,
+ state: row.state,
+ progress: { percent: row.progress == null ? null : Math.round(row.progress * 10) / 10, message: row.progress_message || null },
+ attempts: row.attempts,
+ max_attempts: row.max_attempts,
+ cancel_requested: !!row.cancel_requested,
+ created_at: iso(row.created_at),
+ started_at: iso(row.started_at),
+ finished_at: iso(row.finished_at),
+ expires_at: iso(row.expires_at),
+ result: publicResult(store.parse(row.result_json, null), row.id),
+ error: store.parse(row.error_json, null),
+ retryable: !!row.retryable,
+ links: {
+ self: `/api/v1/jobs/${row.id}`,
+ events: `/api/v1/jobs/${row.id}/events`,
+ cancel: live ? `/api/v1/jobs/${row.id}` : null,
+ },
+ };
+ }
+
+ // ── Events ─────────────────────────────────────────────────
+ function emit(id, event) {
+ if (stopped) return;
+ const data = view(store.get(id));
+ if (!data) return;
+ const seq = store.appendEvent(id, event, data);
+ const set = listeners.get(id);
+ if (set) for (const fn of [...set]) { try { fn({ seq, event, data }); } catch (err) { log.error('[Jobs] listener error:', err.message); } }
+ }
+ function subscribe(id, fn) {
+ if (!listeners.has(id)) listeners.set(id, new Set());
+ listeners.get(id).add(fn);
+ return () => { const s = listeners.get(id); if (s) { s.delete(fn); if (!s.size) listeners.delete(id); } };
+ }
+
+ // ── Submit ─────────────────────────────────────────────────
+ /**
+ * submit({ owner, type, input, files: [{ path | buffer, name, mime, size }], idempotencyKey, ttlMs })
+ * → { job (row), replayed }. Temp input files are consumed (moved) on accept and deleted on replay.
+ */
+ async function submit({ owner, type, input = {}, files = [], idempotencyKey = null, ttlMs = HOUR }) {
+ if (stopped) throw new JobError(503, 'tools.job.unavailable', 'The job system is shutting down');
+ const discard = () => Promise.all(files.map(f => (f.path ? fsp.unlink(f.path).catch(() => {}) : null)));
+ try {
+ if (!owner) throw new JobError(401, 'tools.job.no_owner', 'No owner for this job');
+ const def = types.get(type);
+ if (!def) throw new JobError(400, 'tools.job.unknown_type', `Unknown job type "${type}". Known: ${[...types.keys()].join(', ')}`);
+ if (input == null || typeof input !== 'object' || Array.isArray(input)) throw new JobError(400, 'tools.job.invalid', 'input must be a JSON object');
+ if (JSON.stringify(input).length > 16384) throw new JobError(400, 'tools.job.invalid', 'input is larger than 16 KB');
+ if (files.length < def.minFiles) throw new JobError(400, 'tools.job.invalid', def.minFiles === 1 ? 'This job needs a file' : `This job needs at least ${def.minFiles} files`);
+ if (files.length > def.maxFiles) throw new JobError(400, 'tools.job.invalid', `At most ${def.maxFiles} file(s) for this job`);
+ const bad = def.validate(input, files);
+ if (bad) throw new JobError(400, 'tools.job.invalid', bad);
+ if (idempotencyKey != null && !/^[\x21-\x7e]{8,200}$/.test(String(idempotencyKey))) throw new JobError(400, 'tools.job.invalid', 'Idempotency-Key must be 8-200 printable characters');
+
+ // Hash the request so a reused key with a different request is caught.
+ const fileHashes = [];
+ for (const f of files) fileHashes.push(f.buffer ? crypto.createHash('sha256').update(f.buffer).digest('hex') : await sha256File(f.path));
+ const requestHash = crypto.createHash('sha256').update(canonical({ type, input, files: fileHashes })).digest('hex');
+
+ if (idempotencyKey != null) {
+ const prior = store.byIdempotencyKey(owner, String(idempotencyKey));
+ if (prior) {
+ await discard();
+ if (prior.request_hash !== requestHash) throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request', { job_id: prior.id });
+ return { job: prior, replayed: true };
+ }
+ }
+ if (store.activeForOwner(owner) >= maxActivePerOwner) throw new JobError(429, 'tools.job.too_many_active', `At most ${maxActivePerOwner} unfinished jobs at a time; wait for one to finish or cancel one`);
+
+ const id = `job_${contracts.ids.ulid()}`;
+ const inDir = path.join(jobDir(id), 'in');
+ await fsp.mkdir(inDir, { recursive: true });
+ const stored = [];
+ for (let i = 0; i < files.length; i++) {
+ const f = files[i];
+ const dest = path.join(inDir, `${i}-${SAFE_NAME(f.name)}`);
+ if (f.buffer) await fsp.writeFile(dest, f.buffer); else await moveFile(f.path, dest);
+ stored.push({ name: String(f.name || `file-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size: f.size != null ? f.size : fs.statSync(dest).size, sha256: fileHashes[i], _path: dest });
+ }
+ try {
+ store.transaction(() => {
+ store.insert({
+ id, type, type_version: def.version, owner, input_json: JSON.stringify(input), files_json: JSON.stringify(stored),
+ idempotency_key: idempotencyKey == null ? null : String(idempotencyKey), request_hash: requestHash,
+ max_attempts: def.maxAttempts, ttl_ms: ttlMs, now: Date.now(),
+ });
+ })();
+ } catch (err) {
+ await fsp.rm(jobDir(id), { recursive: true, force: true });
+ // Two submits with one key raced: the other one won, so this is a replay of it.
+ if (/UNIQUE/.test(err.message) && idempotencyKey != null) {
+ const prior = store.byIdempotencyKey(owner, String(idempotencyKey));
+ if (prior && prior.request_hash === requestHash) return { job: prior, replayed: true };
+ throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request');
+ }
+ throw err;
+ }
+ emit(id, 'job.queued');
+ kick();
+ return { job: store.get(id), replayed: false };
+ } catch (err) {
+ await discard();
+ throw err;
+ }
+ }
+
+ // ── Scheduling ─────────────────────────────────────────────
+ function kick() {
+ if (!started || stopped || kicking) return;
+ kicking = true;
+ try {
+ while (active.size < concurrency) {
+ const [row] = store.nextQueued(1);
+ if (!row) break;
+ if (!store.claim(row.id)) continue;
+ const entry = { ctrl: new AbortController(), reason: null, last: { pct: -1, msg: null, at: 0 } };
+ active.set(row.id, entry);
+ emit(row.id, 'job.running');
+ setImmediate(() => execute(row.id, entry));
+ }
+ } finally { kicking = false; }
+ }
+
+ function progressFn(id, entry) {
+ return (percent, message) => {
+ if (stopped || entry.reason) return;
+ const pct = Math.max(0, Math.min(100, Number(percent) || 0));
+ const msg = message == null ? entry.last.msg : String(message).slice(0, 200);
+ const now = Date.now();
+ if (msg === entry.last.msg && (Math.abs(pct - entry.last.pct) < 1 || now - entry.last.at < throttleMs)) return;
+ entry.last = { pct, msg, at: now };
+ if (store.setProgress(id, pct, msg, now)) emit(id, 'job.progress');
+ };
+ }
+
+ async function execute(id, entry) {
+ const row = stopped ? null : store.get(id);
+ if (!row) { active.delete(id); return; }
+ const def = types.get(row.type);
+ const outDir = path.join(jobDir(row.id), 'out');
+ let outcome;
+ const timer = setTimeout(() => { if (!entry.reason) { entry.reason = 'timeout'; entry.ctrl.abort(new Error('timeout')); } }, def.timeoutMs);
+ if (timer.unref) timer.unref();
+ try {
+ await fsp.mkdir(outDir, { recursive: true });
+ const files = store.parse(row.files_json, []).map(f => ({ name: f.name, mime: f.mime, size: f.size, sha256: f.sha256, path: f._path }));
+ const out = await def.run({
+ job: { id: row.id, type: row.type, attempt: row.attempts, owner: row.owner },
+ input: store.parse(row.input_json, {}), files, outDir,
+ signal: entry.ctrl.signal,
+ progress: progressFn(row.id, entry),
+ });
+ if (stopped) return;
+ if (entry.reason) throw new Error(entry.reason);
+ const result = await storeResult(row, out || {}, entry);
+ if (stopped) return;
+ if (entry.reason) { await dropResult(result); throw new Error(entry.reason); }
+ outcome = { state: 'succeeded', result };
+ } catch (err) {
+ if (stopped) return;
+ const fresh = store.get(row.id) || row;
+ if (entry.reason === 'cancel' || fresh.cancel_requested) outcome = { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') };
+ else if (entry.reason === 'timeout') outcome = { state: 'failed', error: problem(504, 'tools.job.timeout', `The job ran longer than ${Math.round(def.timeoutMs / 1000)} s`), retryable: true };
+ else {
+ if (!err.expose) log.error(`[Jobs] ${row.type} ${row.id} failed:`, err.message);
+ // Only codes a tool chose on purpose (tools.…) are passed on; errno codes and the like are not contracts.
+ const code = /^tools\.[a-z0-9_.]+$/.test(String(err.code || '')) ? err.code : 'tools.job.failed';
+ outcome = { state: 'failed', error: problem(Number.isInteger(err.status) ? err.status : 422, code, scrub(err.message) || 'The tool could not process this input'), retryable: !!err.retryable };
+ }
+ } finally {
+ clearTimeout(timer);
+ }
+ try {
+ store.transaction(() => store.finish(row.id, outcome))();
+ emit(row.id, EVENT_FOR[outcome.state]);
+ } finally {
+ active.delete(row.id);
+ fsp.rm(path.join(jobDir(row.id), 'in'), { recursive: true, force: true }).catch(() => {});
+ if (outcome.state !== 'succeeded') fsp.rm(outDir, { recursive: true, force: true }).catch(() => {});
+ kick();
+ }
+ }
+
+ /** Output files → Media objects (when configured) or kept under <job>/out for /files/:n. */
+ async function storeResult(row, out, entry) {
+ const outDir = path.join(jobDir(row.id), 'out');
+ const files = [];
+ const list = Array.isArray(out.files) ? out.files : [];
+ for (let i = 0; i < list.length; i++) {
+ const f = list[i];
+ let local = path.resolve(f.path);
+ if (path.dirname(local) !== outDir) {
+ const dest = path.join(outDir, `${i}-${SAFE_NAME(f.name)}`);
+ await moveFile(local, dest);
+ local = dest;
+ }
+ const size = fs.statSync(local).size;
+ const sha256 = await sha256File(local);
+ const rec = { name: String(f.name || `result-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size, sha256, storage: 'local', media: null, _path: local };
+ if (media && !entry.reason) {
+ try {
+ const progress = progressFn(row.id, entry);
+ progress(99, 'Storing the result');
+ rec.media = await media.upload({ path: local, name: rec.name, mime: rec.mime, size, sha256, owner: row.owner, jobId: row.id, service: o.service, type: row.type });
+ rec.storage = 'media';
+ delete rec._path;
+ await fsp.unlink(local).catch(() => {});
+ } catch (err) {
+ // The result is still here; say where it is rather than pretending it reached Media.
+ log.error(`[Jobs] Media upload failed for ${row.id}:`, err.message);
+ rec.media_error = 'Could not store the result in OpenVibe.Media; it is kept on this server until the job expires';
+ }
+ }
+ files.push(rec);
+ }
+ return { files, data: out.data && typeof out.data === 'object' ? out.data : {} };
+ }
+
+ async function dropResult(result) {
+ for (const f of (result && result.files) || []) {
+ if (f._path) await fsp.unlink(f._path).catch(() => {});
+ if (f.media && media) await media.remove(f.media.media_id).catch(() => {});
+ }
+ }
+
+ // ── Cancel ─────────────────────────────────────────────────
+ /** → { job, changed } — queued jobs end at once; running ones are signalled and end as cancelled. */
+ function cancel(id) {
+ const row = store.get(id);
+ if (!row) return null;
+ if (TERMINAL.includes(row.state)) return { job: row, changed: false };
+ if (row.state === 'queued') {
+ const done = store.transaction(() => store.requestCancel(id) && store.finish(id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled before it started') }))();
+ if (done) {
+ emit(id, 'job.cancelled');
+ fsp.rm(jobDir(id), { recursive: true, force: true }).catch(() => {});
+ }
+ return { job: store.get(id), changed: done };
+ }
+ store.requestCancel(id);
+ const entry = active.get(id);
+ if (entry && !entry.reason) { entry.reason = 'cancel'; entry.ctrl.abort(new Error('cancelled')); }
+ emit(id, 'job.cancel_requested');
+ return { job: store.get(id), changed: true };
+ }
+
+ // ── Boot recovery, pruning ─────────────────────────────────
+ function recover() {
+ let requeued = 0, failed = 0;
+ for (const row of store.running()) {
+ const def = types.get(row.type);
+ if (row.cancel_requested) {
+ store.finish(row.id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') });
+ emit(row.id, 'job.cancelled');
+ } else if (!def) {
+ store.finish(row.id, { state: 'failed', error: problem(500, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`) });
+ emit(row.id, 'job.failed'); failed++;
+ } else if (def.onRestart === 'requeue' && row.attempts < row.max_attempts) {
+ store.requeue(row.id);
+ emit(row.id, 'job.queued'); requeued++;
+ } else {
+ const retryable = def.onRestart === 'fail';
+ store.finish(row.id, {
+ state: 'failed', retryable,
+ error: problem(503, 'tools.job.interrupted', retryable ? 'The service restarted while this job was running; submit it again' : `The service restarted during each of ${row.attempts} attempts`),
+ });
+ emit(row.id, 'job.failed'); failed++;
+ }
+ fs.rmSync(path.join(jobDir(row.id), 'out'), { recursive: true, force: true });
+ }
+ // Directories without a row: a crash between moving the upload and recording the job.
+ // That job was never accepted (no 202 went out), so its files have no owner.
+ for (const name of fs.readdirSync(root)) {
+ if (/^job_[0-9A-Z]{26}$/.test(name) && !store.get(name)) fs.rmSync(path.join(root, name), { recursive: true, force: true });
+ }
+ return { requeued, failed };
+ }
+
+ async function prune(now = Date.now()) {
+ let n = 0;
+ for (const row of store.expired(now)) {
+ const result = store.parse(row.result_json, null);
+ if (media && result) for (const f of result.files || []) if (f.media && f.media.media_id) await media.remove(f.media.media_id).catch(err => log.warn(`[Jobs] could not delete ${f.media.media_id}:`, err.message));
+ await fsp.rm(jobDir(row.id), { recursive: true, force: true }).catch(() => {});
+ store.remove(row.id);
+ listeners.delete(row.id);
+ n++;
+ }
+ return n;
+ }
+
+ function start() {
+ if (started) return api;
+ started = true;
+ const r = recover();
+ if (r.requeued || r.failed) log.log(`[Jobs] ${o.service}: ${r.requeued} job(s) re-queued, ${r.failed} failed after restart`);
+ prune().catch(() => {});
+ pruneTimer = setInterval(() => prune().catch(err => log.error('[Jobs] prune:', err.message)), o.pruneIntervalMs || 5 * 60 * 1000);
+ if (pruneTimer.unref) pruneTimer.unref();
+ kick();
+ return api;
+ }
+
+ /**
+ * Stop. Running jobs are left as they are in the database ('running'), which is exactly what a
+ * crash or a restart leaves behind; the next start() recovers them per type.
+ */
+ function stop() {
+ stopped = true;
+ if (pruneTimer) clearInterval(pruneTimer);
+ for (const entry of active.values()) { entry.reason = entry.reason || 'shutdown'; entry.ctrl.abort(new Error('shutdown')); }
+ listeners.clear();
+ for (const fn of [...stopHooks]) { try { fn(); } catch { /* best effort */ } }
+ stopHooks.clear();
+ }
+ /** Run fn when the system stops; → a function that removes it. */
+ function onStop(fn) { stopHooks.add(fn); return () => stopHooks.delete(fn); }
+
+ /** Where a result file's bytes are: { local: path } or { media: MediaRef }. */
+ function resultFile(row, n) {
+ const result = store.parse(row.result_json, null);
+ const f = result && result.files && result.files[n];
+ if (!f) return null;
+ return { name: f.name, mime: f.mime, size: f.size, storage: f.storage, media: f.media || null, path: f._path || null };
+ }
+
+ const api = {
+ define, submit, cancel, subscribe, start, stop, onStop, prune, recover, view, resultFile,
+ get: (id) => store.get(id),
+ eventsAfter: (id, seq) => store.eventsAfter(id, seq),
+ lastSeq: (id) => store.lastSeq(id),
+ types: () => [...types.keys()],
+ media,
+ stats: () => ({ running: active.size, concurrency, ...store.counts(), results: media ? 'media' : 'local' }),
+ store,
+ JobError,
+ };
+ return api;
+}
+
+module.exports = { createJobSystem, JobError, canonical, TERMINAL };
new file mode 100644
new file mode 100644
@@ -0,0 +1,14 @@
@@ -0,0 +1,14 @@
+'use strict';
+// apps/_shared has no node_modules of its own (the apps require it by relative path), so its tests
+// borrow the dependencies of the first app that has them installed.
+const path = require('path');
+const APPS = ['img', 'docs', 'audio', 'yt', 'gateway'].map(a => path.join(__dirname, '..', '..', a));
+
+function dep(name) {
+ for (const dir of APPS) {
+ try { return require(require.resolve(name, { paths: [dir] })); } catch { /* next app */ }
+ }
+ throw new Error(`${name} is not installed in any app (run npm run install:all)`);
+}
+
+module.exports = { dep };
new file mode 100644
new file mode 100644
@@ -0,0 +1,390 @@
@@ -0,0 +1,390 @@
+'use strict';
+// Wave 11 exit criteria for the shared job runtime, end to end over HTTP:
+// an accepted job survives a restart (DB and app closed and reopened), running jobs are re-queued
+// or failed-retryable per type, reattach by id, cancel (queued and running), idempotency keys,
+// owner scoping, SSE resume with Last-Event-ID, bounded concurrency, pruning, and results stored
+// as Media objects through Media's v2 object API (with the local fallback).
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const crypto = require('crypto');
+const { dep } = require('./deps');
+
+const express = dep('express');
+const cookieParser = dep('cookie-parser');
+const Database = dep('better-sqlite3');
+const contracts = dep('openvibe-contracts');
+const jobs = require('../jobs');
+
+const ISSUER = 'https://openvibe.network';
+const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
+const USER_A = 'usr_01JAAAAAAAAAAAAAAAAAAAAAAA';
+const USER_B = 'usr_01JBBBBBBBBBBBBBBBBBBBBBBB';
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+async function until(fn, what, ms = 5000) {
+ const t0 = Date.now();
+ for (;;) {
+ const v = await fn();
+ if (v) return v;
+ if (Date.now() - t0 > ms) throw new Error(`timed out waiting for ${what}`);
+ await sleep(20);
+ }
+}
+
+// ── Job types used by the tests ──────────────────────────────
+const gates = new Map(); // name → { promise, open() }
+function gate(name) {
+ if (!gates.has(name)) { let open; const promise = new Promise(r => { open = r; }); gates.set(name, { promise, open }); }
+ return gates.get(name);
+}
+function define(system) {
+ // Upper-cases its input file (or input.text); waits on input.gate when given. Re-queued after a restart.
+ system.define({
+ type: 'test.upper', maxFiles: 1, maxAttempts: 3,
+ validate: (input) => (input.text != null && typeof input.text !== 'string' ? 'text must be a string' : null),
+ async run({ input, files, outDir, progress, signal }) {
+ progress(10, 'Reading');
+ if (input.gate) await Promise.race([gate(input.gate).promise, new Promise((_, rej) => signal.addEventListener('abort', () => rej(new Error('aborted'))))]);
+ const text = files.length ? fs.readFileSync(files[0].path, 'utf8') : input.text || '';
+ progress(60, 'Writing');
+ const out = path.join(outDir, 'upper.txt');
+ fs.writeFileSync(out, text.toUpperCase());
+ return { files: [{ path: out, name: 'upper.txt', mime: 'text/plain' }], data: { length: text.length } };
+ },
+ });
+ // Not safe to run twice: after a restart it fails with retryable = true.
+ system.define({
+ type: 'test.fragile', onRestart: 'fail',
+ async run({ input, signal }) {
+ await Promise.race([gate(input.gate).promise, new Promise((_, rej) => signal.addEventListener('abort', () => rej(new Error('aborted'))))]);
+ return { data: { ok: true } };
+ },
+ });
+ system.define({ type: 'test.boom', async run() { throw new Error('This input at /srv/secret/path/x.png is not an image'); } });
+}
+
+// ── A fake OpenVibe.Media (object API v2) ────────────────────
+function fakeMedia() {
+ const objects = new Map();
+ const app = express();
+ const state = { failInit: false, deleted: [] };
+ app.use(express.json());
+ const authed = (req, res, next) => (req.headers.authorization === 'Bearer media-token' ? next() : res.status(401).json({ error: 'no' }));
+ app.post('/api/v2/tools/objects', authed, (req, res) => {
+ if (state.failInit) return res.status(503).json({ code: 'down' });
+ const id = `med_${contracts.ids.ulid()}`;
+ objects.set(id, { id, meta: req.body, subject: req.headers['x-ov-subject'] || null, bytes: null, status: 'uploading' });
+ res.status(201).json({ id, upload: { method: 'PUT', url: `https://openvibe.media/api/v2/tools/objects/${id}/content?token=tok-${id}` } });
+ });
+ app.put('/api/v2/tools/objects/:id/content', (req, res) => {
+ const o = objects.get(req.params.id);
+ if (!o || req.query.token !== `tok-${o.id}`) return res.status(401).json({ error: 'bad token' });
+ const chunks = [];
+ req.on('data', c => chunks.push(c)).on('end', () => { o.bytes = Buffer.concat(chunks); res.json({ id: o.id, size_bytes: o.bytes.length }); });
+ });
+ app.post('/api/v2/tools/objects/:id/complete', authed, (req, res) => {
+ const o = objects.get(req.params.id);
+ const sha = crypto.createHash('sha256').update(o.bytes).digest('hex');
+ if (req.body.content_hash !== sha) return res.status(422).json({ code: 'media.object.hash_mismatch' });
+ o.status = 'ready';
+ res.json({ id: o.id, lifecycle_status: 'ready' });
+ });
+ app.get('/api/v2/tools/objects/:id/download', authed, (req, res) => res.json({ url: `https://openvibe.media/o/${req.params.id}?exp=1&sig=s`, expires_at: 'x', public: false }));
+ app.delete('/api/v2/tools/objects/:id', authed, (req, res) => { state.deleted.push(req.params.id); objects.delete(req.params.id); res.json({ ok: true }); });
+ app.get('/o/:id', (req, res) => { const o = objects.get(req.params.id); if (!o || req.query.sig !== 's') return res.status(404).end(); res.type('text/plain').send(o.bytes); });
+ return { app, objects, state };
+}
+const fakeTokens = { authHeaders: async () => ({ Authorization: 'Bearer media-token' }), invalidate() {} };
+
+// ── A satellite: express + the job routes over one SQLite file ──
+function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10 } = {}) {
+ fs.mkdirSync(dir, { recursive: true });
+ const db = new Database(path.join(dir, 'jobs.db'));
+ const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir: dir, concurrency, media, maxActivePerOwner, progressThrottleMs: 0, log: { log() {}, warn() {}, error() {} } });
+ define(system);
+ const app = express();
+ app.use(cookieParser());
+ app.use(express.json());
+ app.use((req, _res, next) => { if (req.headers['x-test-user']) req.user = { sub: 7, subject_id: req.headers['x-test-user'] }; next(); });
+ const multer = dep('multer');
+ const upload = multer({ dest: path.join(dir, 'tmp') });
+ const resolveOwner = jobs.createOwnerResolver({ contracts, getPublicKey: () => publicKey, issuer: ISSUER, secureCookie: false });
+ jobs.mountJobRoutes(app, { system, contracts, resolveOwner, receive: upload.any() });
+ system.start();
+ return new Promise(resolve => {
+ const server = app.listen(0, '127.0.0.1', () => resolve({
+ system, db, base: `http://127.0.0.1:${server.address().port}`,
+ close: () => new Promise(r => { system.stop(); server.closeAllConnections(); server.close(() => { db.close(); r(); }); }),
+ }));
+ });
+}
+
+/** A browser-ish client with its own cookie jar (the jobs session cookie). */
+function client(base, { user, bearer } = {}) {
+ let cookie = '';
+ return async function call(p, init = {}) {
+ const headers = { ...(init.headers || {}) };
+ if (cookie) headers.cookie = cookie;
+ if (user) headers['x-test-user'] = user;
+ if (bearer) headers.authorization = `Bearer ${bearer}`;
+ const res = await fetch((typeof base === 'function' ? base() : base) + p, { ...init, headers, redirect: 'manual' });
+ const set = res.headers.get('set-cookie');
+ if (set) cookie = set.split(';')[0];
+ const type = res.headers.get('content-type') || '';
+ const body = /json/.test(type) ? await res.json() : await res.text();
+ return { status: res.status, headers: res.headers, body };
+ };
+}
+const json = (body, headers = {}) => ({ method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body: JSON.stringify(body) });
+
+/** Read an SSE stream until `stop(events)` or the server ends it. */
+async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}) {
+ const ac = new AbortController();
+ const timer = setTimeout(() => ac.abort(), ms);
+ const res = await fetch(url, { headers, signal: ac.signal });
+ const events = [];
+ if (res.status !== 200) { clearTimeout(timer); return { status: res.status, events }; }
+ const dec = new TextDecoder();
+ let buf = '';
+ try {
+ for await (const chunk of res.body) {
+ buf += dec.decode(chunk, { stream: true });
+ let i;
+ while ((i = buf.indexOf('\n\n')) >= 0) {
+ const frame = buf.slice(0, i); buf = buf.slice(i + 2);
+ const e = {};
+ for (const line of frame.split('\n')) { const m = /^(id|event|data): (.*)$/.exec(line); if (m) e[m[1]] = m[2]; }
+ if (e.data) { e.data = JSON.parse(e.data); e.id = Number(e.id); events.push(e); }
+ }
+ if (stop(events)) break;
+ }
+ } catch (err) { if (err.name !== 'AbortError') throw err; }
+ clearTimeout(timer); ac.abort();
+ return { status: res.status, events };
+}
+
+(async () => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-jobs-'));
+ try {
+ // ── Submit, reattach by id, result download ─────────────
+ let s = await satellite(path.join(root, 'a'));
+ const alice = client(s.base, { user: USER_A });
+ let r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'hello' } }));
+ assert.strictEqual(r.status, 202, 'accepted');
+ assert.match(r.body.id, /^job_[0-9A-Z]{26}$/);
+ assert.strictEqual(r.headers.get('location'), `/api/v1/jobs/${r.body.id}`);
+ const first = r.body.id;
+ const done = await until(async () => { const g = await alice(`/api/v1/jobs/${first}`); return g.body.state === 'succeeded' && g.body; }, 'first job');
+ assert.strictEqual(done.result.data.length, 5);
+ assert.strictEqual(done.result.files[0].storage, 'local');
+ assert.ok(done.expires_at && done.finished_at, 'a finished job says when it expires');
+ r = await alice(done.result.files[0].url);
+ assert.strictEqual(r.body, 'HELLO');
+ assert.match(r.headers.get('content-disposition'), /^attachment; filename="upper.txt"/);
+
+ // Multipart with a file (the way the UI submits).
+ const fd = new FormData();
+ fd.append('type', 'test.upper'); fd.append('input', '{}');
+ fd.append('file', new Blob(['from a file']), 'in.txt');
+ r = await alice('/api/v1/jobs', { method: 'POST', body: fd });
+ assert.strictEqual(r.status, 202);
+ const multi = await until(async () => { const g = await alice(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'succeeded' && g.body; }, 'multipart job');
+ assert.strictEqual((await alice(multi.result.files[0].url)).body, 'FROM A FILE');
+
+ // Validation and tool failures are problem+json.
+ r = await alice('/api/v1/jobs', json({ type: 'nope', input: {} }));
+ assert.strictEqual(r.status, 400); assert.strictEqual(r.body.code, 'tools.job.unknown_type'); assert.match(r.headers.get('content-type'), /problem\+json/);
+ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 5 } }));
+ assert.strictEqual(r.body.code, 'tools.job.invalid');
+ r = await alice('/api/v1/jobs', json({ type: 'test.boom' }));
+ const boom = await until(async () => { const g = await alice(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'failed' && g.body; }, 'failed job');
+ assert.strictEqual(boom.error.code, 'tools.job.failed');
+ assert.ok(!/srv\/secret/.test(boom.error.detail), 'server paths are not shown to people');
+
+ // ── Idempotency ─────────────────────────────────────────
+ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'once' } }, { 'idempotency-key': 'key-000000001' }));
+ const once = r.body.id;
+ assert.strictEqual(r.status, 202);
+ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'once' } }, { 'idempotency-key': 'key-000000001' }));
+ assert.strictEqual(r.status, 200); assert.strictEqual(r.body.id, once, 'same key, same request → same job');
+ assert.strictEqual(r.headers.get('idempotent-replayed'), 'true');
+ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'twice' } }, { 'idempotency-key': 'key-000000001' }));
+ assert.strictEqual(r.status, 409); assert.strictEqual(r.body.code, 'tools.job.idempotency_conflict');
+ r = await client(s.base, { user: USER_B })('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'once' } }, { 'idempotency-key': 'key-000000001' }));
+ assert.strictEqual(r.status, 202); assert.notStrictEqual(r.body.id, once, 'keys are per owner');
+
+ // ── Owner scoping ───────────────────────────────────────
+ const bob = client(s.base, { user: USER_B });
+ const guest1 = client(s.base), guest2 = client(s.base), nobody = client(s.base);
+ for (const who of [bob, guest1, nobody]) {
+ assert.strictEqual((await who(`/api/v1/jobs/${first}`)).status, 404);
+ assert.strictEqual((await who(`/api/v1/jobs/${first}`, { method: 'DELETE' })).status, 404);
+ assert.strictEqual((await who(`/api/v1/jobs/${first}/events`)).status, 404);
+ assert.strictEqual((await who(`/api/v1/jobs/${first}/files/0`)).status, 404);
+ }
+ r = await guest1('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'guest' } }));
+ assert.strictEqual(r.status, 202);
+ const guestJob = r.body.id;
+ assert.strictEqual((await guest1(`/api/v1/jobs/${guestJob}`)).status, 200, 'the session that made it can reattach');
+ assert.strictEqual((await guest2(`/api/v1/jobs/${guestJob}`)).status, 404, 'another browser cannot');
+ assert.strictEqual((await alice(`/api/v1/jobs/${guestJob}`)).status, 404, 'nor can a signed-in stranger');
+ assert.ok(s.db.prepare('SELECT owner FROM tool_jobs').all().every(x => /^(user:usr_[0-9A-Z]{26}|session:[0-9a-f]{40})$/.test(x.owner)), 'owners are canonical subjects or hashed sessions, never raw cookies');
+
+ // ── SSE: events carry ids; Last-Event-ID resumes after them ──
+ const all = await readSse(`${s.base}/api/v1/jobs/${first}/events`, { headers: { 'x-test-user': USER_A } });
+ assert.strictEqual(all.status, 200);
+ assert.deepStrictEqual(all.events.map(e => e.event), ['job.queued', 'job.running', 'job.progress', 'job.progress', 'job.succeeded']);
+ assert.ok(all.events.every((e, i) => i === 0 || e.id > all.events[i - 1].id), 'ids increase');
+ const resumed = await readSse(`${s.base}/api/v1/jobs/${first}/events`, { headers: { 'x-test-user': USER_A, 'last-event-id': String(all.events[2].id) } });
+ assert.deepStrictEqual(resumed.events.map(e => e.id), all.events.slice(3).map(e => e.id), 'only what came after Last-Event-ID');
+ const caughtUp = await readSse(`${s.base}/api/v1/jobs/${first}/events?last_event_id=${all.events[4].id}`, { headers: { 'x-test-user': USER_A } });
+ assert.strictEqual(caughtUp.status, 204, 'finished and nothing new: 204 stops EventSource reconnecting');
+ // Live: a stream opened on a running job sees the rest as it happens.
+ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'live', gate: 'live' } }));
+ const liveId = r.body.id;
+ await until(async () => (await alice(`/api/v1/jobs/${liveId}`)).body.state === 'running', 'live job running');
+ const streaming = readSse(`${s.base}/api/v1/jobs/${liveId}/events`, { headers: { 'x-test-user': USER_A } });
+ await sleep(100);
+ gate('live').open();
+ const live = await streaming;
+ assert.strictEqual(live.events[live.events.length - 1].event, 'job.succeeded');
+ assert.strictEqual(live.events[live.events.length - 1].data.result.files.length, 1, 'the terminal event carries the result');
+
+ // ── Cancel ──────────────────────────────────────────────
+ await s.close();
+ s = await satellite(path.join(root, 'b'), { concurrency: 1 });
+ const carol = client(s.base, { user: USER_A });
+ const blocker = (await carol('/api/v1/jobs', json({ type: 'test.upper', input: { gate: 'block' } }))).body.id;
+ const waiting = (await carol('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'later' } }))).body.id;
+ await until(async () => (await carol(`/api/v1/jobs/${blocker}`)).body.state === 'running', 'blocker running');
+ assert.strictEqual((await carol(`/api/v1/jobs/${waiting}`)).body.state, 'queued', 'concurrency 1: the second job waits');
+ r = await carol(`/api/v1/jobs/${waiting}`, { method: 'DELETE' });
+ assert.strictEqual(r.status, 200); assert.strictEqual(r.body.state, 'cancelled', 'a queued job is cancelled at once');
+ r = await carol(`/api/v1/jobs/${blocker}`, { method: 'DELETE' });
+ assert.strictEqual(r.status, 202); assert.strictEqual(r.body.cancel_requested, true, 'a running job is signalled');
+ const cancelled = await until(async () => { const g = await carol(`/api/v1/jobs/${blocker}`); return g.body.state === 'cancelled' && g.body; }, 'running job cancelled');
+ assert.strictEqual(cancelled.error.code, 'tools.job.cancelled');
+ assert.strictEqual((await carol(`/api/v1/jobs/${blocker}`, { method: 'DELETE' })).status, 200, 'cancelling twice is fine');
+ const fin = (await carol('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'x' } }))).body.id;
+ await until(async () => (await carol(`/api/v1/jobs/${fin}`)).body.state === 'succeeded', 'job to finish');
+ r = await carol(`/api/v1/jobs/${fin}`, { method: 'DELETE' });
+ assert.strictEqual(r.status, 409); assert.strictEqual(r.body.code, 'tools.job.already_finished');
+
+ // ── Bounded concurrency and the per-owner cap ───────────
+ await s.close();
+ s = await satellite(path.join(root, 'c'), { concurrency: 2, maxActivePerOwner: 4 });
+ const dave = client(s.base, { user: USER_B });
+ const ids = [];
+ for (let i = 0; i < 4; i++) ids.push((await dave('/api/v1/jobs', json({ type: 'test.upper', input: { gate: 'pool' } }))).body.id);
+ await until(async () => s.system.stats().running === 2, 'two running');
+ await sleep(50);
+ assert.strictEqual(s.system.stats().running, 2, 'never more than the concurrency limit');
+ assert.strictEqual(s.system.stats().queued, 2);
+ r = await dave('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'one too many' } }));
+ assert.strictEqual(r.status, 429); assert.strictEqual(r.body.code, 'tools.job.too_many_active');
+ gate('pool').open();
+ await until(async () => s.system.stats().succeeded === 4, 'pool drained');
+
+ // ── Restart: an accepted job is not lost ────────────────
+ await s.close();
+ const dir = path.join(root, 'd');
+ s = await satellite(dir, { concurrency: 0 }); // accepts, runs nothing: every job stays queued
+ const erin = client(() => s.base, { user: USER_A }); // follows the satellite across restarts
+ const fd2 = new FormData();
+ fd2.append('type', 'test.upper'); fd2.append('input', '{}'); fd2.append('file', new Blob(['survives']), 'keep.txt');
+ r = await erin('/api/v1/jobs', { method: 'POST', body: fd2 });
+ assert.strictEqual(r.status, 202);
+ const accepted = r.body.id;
+ await s.close(); // DB and app closed: the process is gone
+ s = await satellite(dir, { concurrency: 2 }); // and back
+ const back = await until(async () => { const g = await erin(`/api/v1/jobs/${accepted}`); return g.body.state === 'succeeded' && g.body; }, 'accepted job after restart');
+ assert.strictEqual((await erin(back.result.files[0].url)).body, 'SURVIVES', 'its uploaded input survived too');
+
+ // Running when the process died: re-queued (test.upper) or failed-retryable (test.fragile).
+ const again = (await erin('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'rerun', gate: 'never-1' } }))).body.id;
+ const fragile = (await erin('/api/v1/jobs', json({ type: 'test.fragile', input: { gate: 'never-2' } }))).body.id;
+ await until(async () => (await erin(`/api/v1/jobs/${again}`)).body.state === 'running' && (await erin(`/api/v1/jobs/${fragile}`)).body.state === 'running', 'both running');
+ await s.close();
+ gates.get('never-1').open(); // the gate is open after the restart
+ s = await satellite(dir, { concurrency: 2 });
+ const rerun = await until(async () => { const g = await erin(`/api/v1/jobs/${again}`); return g.body.state === 'succeeded' && g.body; }, 're-queued job');
+ assert.strictEqual(rerun.attempts, 2, 'the interrupted attempt counts');
+ const broken = (await erin(`/api/v1/jobs/${fragile}`)).body;
+ assert.strictEqual(broken.state, 'failed');
+ assert.strictEqual(broken.retryable, true);
+ assert.strictEqual(broken.error.code, 'tools.job.interrupted');
+ const history = await readSse(`${s.base}/api/v1/jobs/${again}/events`, { headers: { 'x-test-user': USER_A } });
+ assert.deepStrictEqual(history.events.map(e => e.event).filter(e => e !== 'job.progress'), ['job.queued', 'job.running', 'job.queued', 'job.running', 'job.succeeded'], 'the event log spans the restart');
+
+ // ── Pruning ─────────────────────────────────────────────
+ const pruned = await s.system.prune(Date.now() + 25 * 60 * 60 * 1000);
+ assert.ok(pruned >= 3);
+ assert.strictEqual((await erin(`/api/v1/jobs/${accepted}`)).status, 404, 'expired jobs are gone');
+ assert.ok(!fs.existsSync(path.join(dir, 'jobs', accepted)), 'with their files');
+ assert.strictEqual(s.db.prepare('SELECT COUNT(*) AS n FROM tool_job_events WHERE job_id = ?').get(accepted).n, 0, 'and their events');
+ await s.close();
+
+ // ── Results as Media objects ────────────────────────────
+ const fm = fakeMedia();
+ const mediaServer = await new Promise(res => { const srv = fm.app.listen(0, '127.0.0.1', () => res(srv)); });
+ const media = jobs.createMediaResults({ internalUrl: `http://127.0.0.1:${mediaServer.address().port}`, tokens: fakeTokens });
+ s = await satellite(path.join(root, 'e'), { media });
+ const frank = client(s.base, { user: USER_B });
+ r = await frank('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'stored in media' } }));
+ const m = await until(async () => { const g = await frank(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'succeeded' && g.body; }, 'media job');
+ const f0 = m.result.files[0];
+ assert.strictEqual(f0.storage, 'media');
+ assert.match(f0.media.media_id, /^med_[0-9A-Z]{26}$/, 'the result is a Media object reference');
+ const obj = fm.objects.get(f0.media.media_id);
+ assert.strictEqual(obj.status, 'ready');
+ assert.strictEqual(obj.subject, USER_B, 'owned by the person who ran the job');
+ assert.strictEqual(obj.meta.visibility, 'private');
+ assert.strictEqual(obj.meta.metadata.job_id, m.id);
+ assert.ok(!fs.existsSync(path.join(root, 'e', 'jobs', m.id, 'out', '0-upper.txt')), 'no local copy is kept');
+ r = await frank(`${f0.url}?inline=1`);
+ assert.strictEqual(r.status, 302); assert.match(r.headers.get('location'), /^https:\/\/openvibe\.media\/o\/med_/, 'previews load from Media');
+ r = await frank(f0.url);
+ assert.strictEqual(r.body, 'STORED IN MEDIA', 'downloads stream through with the right name');
+ assert.match(r.headers.get('content-disposition'), /attachment/);
+ // Media down: the result stays local and says so.
+ fm.state.failInit = true;
+ r = await frank('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'fallback' } }));
+ const fb = await until(async () => { const g = await frank(`/api/v1/jobs/${r.body.id}`); return g.body.state === 'succeeded' && g.body; }, 'fallback job');
+ assert.strictEqual(fb.result.files[0].storage, 'local');
+ assert.ok(fb.result.files[0].media_error);
+ assert.strictEqual((await frank(fb.result.files[0].url)).body, 'FALLBACK');
+ await s.system.prune(Date.now() + 25 * 60 * 60 * 1000);
+ assert.ok(fm.state.deleted.includes(f0.media.media_id), 'pruning deletes the Media object');
+ await s.close();
+ mediaServer.close();
+
+ // ── Service/app principals (tools.job.* capabilities) ───
+ s = await satellite(path.join(root, 'f'));
+ const token = (claims) => contracts.serviceAuth.signServiceToken({
+ iss: ISSUER, sub: 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC', actor_type: 'app', aud: ['openvibe.tools'],
+ cap: ['tools.job.create', 'tools.job.read'], iat: Math.floor(Date.now() / 1000), exp: Math.floor(Date.now() / 1000) + 300, jti: crypto.randomBytes(8).toString('hex'), ...claims,
+ }, privateKey);
+ const dev = client(s.base, { bearer: token({}) });
+ r = await dev('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'sdk' } }));
+ assert.strictEqual(r.status, 202, 'an app with tools.job.create can submit');
+ assert.strictEqual(r.headers.get('set-cookie'), null, 'principals get no session cookie');
+ const devJob = r.body.id;
+ await until(async () => (await dev(`/api/v1/jobs/${devJob}`)).body.state === 'succeeded', 'app job');
+ assert.strictEqual(s.system.get(devJob).owner, 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC');
+ r = await dev(`/api/v1/jobs/${devJob}`, { method: 'DELETE' });
+ assert.strictEqual(r.status, 403, 'cancel needs tools.job.cancel');
+ r = await client(s.base, { bearer: token({ cap: ['tools.job.read'] }) })('/api/v1/jobs', json({ type: 'test.upper', input: {} }));
+ assert.strictEqual(r.status, 403); assert.strictEqual(r.body.code, 'capability.denied');
+ r = await client(s.base, { bearer: token({ aud: ['openvibe.media'] }) })('/api/v1/jobs', json({ type: 'test.upper', input: {} }));
+ assert.strictEqual(r.status, 401); assert.strictEqual(r.body.code, 'token.wrong_audience');
+ r = await client(s.base, { bearer: token({ sub: 'app:app_01JDDDDDDDDDDDDDDDDDDDDDDD' }) })(`/api/v1/jobs/${devJob}`);
+ assert.strictEqual(r.status, 404, 'another app cannot read it');
+ await s.close();
+
+ console.log('jobs (shared runtime): all checks passed');
+ } finally {
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+})().catch((err) => { console.error(err); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,59 @@
@@ -0,0 +1,59 @@
+'use strict';
+// Start a satellite as its own process (the way systemd does), so a test can kill it outright and
+// start it again over the same data directory.
+const { spawn } = require('child_process');
+const net = require('net');
+const path = require('path');
+
+function freePort() {
+ return new Promise((resolve, reject) => {
+ const srv = net.createServer();
+ srv.unref();
+ srv.on('error', reject);
+ srv.listen(0, '127.0.0.1', () => { const { port } = srv.address(); srv.close(() => resolve(port)); });
+ });
+}
+
+/**
+ * @param {string} app 'img' | 'audio' | 'docs' | …
+ * @param {object} env extra environment (DATA_DIR, TOOLS_JOBS_CONCURRENCY_…)
+ * @returns {Promise<{ base, port, kill(signal), exited: Promise, output: () => string }>}
+ */
+async function startApp(app, env = {}, port) {
+ port = port || await freePort();
+ const dir = path.join(__dirname, '..', '..', app);
+ let out = '';
+ const child = spawn(process.execPath, ['server/index.js'], {
+ cwd: dir,
+ env: {
+ ...process.env,
+ NODE_ENV: 'test',
+ PORT: String(port),
+ HOST: '127.0.0.1',
+ // Never reach the real Network from a test: the key fetch fails fast and users are anonymous.
+ OV_NETWORK_URL: 'http://127.0.0.1:9',
+ OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',
+ TOOLS_JOB_RESULTS: 'local',
+ ...env,
+ },
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+ child.stdout.on('data', d => { out += d; });
+ child.stderr.on('data', d => { out += d; });
+ const exited = new Promise(r => child.on('exit', r));
+ const base = `http://127.0.0.1:${port}`;
+ const t0 = Date.now();
+ for (;;) {
+ if (child.exitCode != null) throw new Error(`${app} exited during boot:\n${out}`);
+ try { const r = await fetch(`${base}/api/health`); if (r.ok) break; } catch { /* not up yet */ }
+ if (Date.now() - t0 > 15000) { child.kill('SIGKILL'); throw new Error(`${app} did not start:\n${out}`); }
+ await new Promise(r => setTimeout(r, 100));
+ }
+ return {
+ base, port, exited,
+ output: () => out,
+ kill(signal = 'SIGKILL') { child.kill(signal); return exited; },
+ };
+}
+
+module.exports = { startApp, freePort };
@@ -10,6 +10,7 @@
@@ -10,6 +10,7 @@
},
},
"license": "MIT",
"license": "MIT",
"dependencies": {
"dependencies": {
+ "better-sqlite3": "^11.0.0",
"cookie-parser": "^1.4.7",
"cookie-parser": "^1.4.7",
"cors": "^2.8.5",
"cors": "^2.8.5",
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
@@ -17,10 +18,10 @@
@@ -17,10 +18,10 @@
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"fluent-ffmpeg": "^2.1.3",
"fluent-ffmpeg": "^2.1.3",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
- "better-sqlite3": "^11.0.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0",
"jsonwebtoken": "^9.0.2",
"jsonwebtoken": "^9.0.2",
"multer": "^1.4.5-lts.1",
"multer": "^1.4.5-lts.1",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.7.0",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0",
"uuid": "^11.1.0"
"uuid": "^11.1.0"
}
}
}
}
@@ -338,6 +338,7 @@
@@ -338,6 +338,7 @@
<script src="https://openvibe.network/shared/footer.js"></script>
<script src="https://openvibe.network/shared/footer.js"></script>
<script src="https://openvibe.network/shared/notification-ui.js"></script>
<script src="https://openvibe.network/shared/notification-ui.js"></script>
<script src="https://openvibe.network/shared/account-switcher.js"></script>
<script src="https://openvibe.network/shared/account-switcher.js"></script>
+ <script src="/js/ov-jobs.js"></script>
<script src="/js/app.js"></script>
<script src="/js/app.js"></script>
</body>
</body>
</html>
</html>
@@ -75,6 +75,7 @@
@@ -75,6 +75,7 @@
} else {
} else {
selectTool('convert');
selectTool('convert');
}
}
+ reattach();
}
}
/* ---------- Branding ---------- */
/* ---------- Branding ---------- */
@@ -373,6 +374,22 @@
@@ -373,6 +374,22 @@
resultPanel.style.display = 'none';
resultPanel.style.display = 'none';
processing.style.display = '';
processing.style.display = '';
+ // Jobs: the work is accepted, followed live (ffmpeg's progress) and survives a reload.
+ if (window.OVJobs) {
+ const input = {};
+ for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;
+ try {
+ const job = await OVJobs.submit({ type: 'audio.process', input, files: [uploadedFile] });
+ OVJobs.remember(job.id);
+ followJob(job);
+ } catch (err) {
+ showToast(err.message, 'error');
+ resetUI();
+ }
+ return;
+ }
+
+ // Without the jobs helper: the synchronous endpoint, as before.
try {
try {
const res = await fetch('/api/process', { method: 'POST', body: formData });
const res = await fetch('/api/process', { method: 'POST', body: formData });
const data = await res.json();
const data = await res.json();
@@ -460,9 +477,76 @@
@@ -460,9 +477,76 @@
return o;
return o;
}
}
+ /* ---------- Jobs ---------- */
+ let watcher = null;
+ let currentJob = null;
+
+ function processingText(job) {
+ const pct = job.progress && job.progress.percent;
+ if (job.cancel_requested) return 'Cancelling...';
+ if (job.state === 'queued') return 'Waiting for a free slot...';
+ const label = (job.progress && job.progress.message) || 'Processing your audio';
+ return pct != null ? `${label}... ${Math.round(pct)}%` : `${label}...`;
+ }
+
+ function cancelButton() {
+ let btn = processing.querySelector('.job-cancel');
+ if (!btn) {
+ btn = document.createElement('button');
+ btn.type = 'button';
+ btn.className = 'btn-secondary job-cancel';
+ btn.textContent = 'Cancel';
+ btn.addEventListener('click', () => { if (currentJob) OVJobs.cancel(currentJob).catch(() => {}); });
+ processing.appendChild(btn);
+ }
+ return btn;
+ }
+
+ function followJob(job) {
+ currentJob = job.id;
+ optionsPanel.style.display = 'none';
+ uploadZone.style.display = 'none';
+ resultPanel.style.display = 'none';
+ processing.style.display = '';
+ const label = processing.querySelector('p');
+ const update = (j) => { if (label) label.textContent = processingText(j); };
+ cancelButton().style.display = '';
+ update(job);
+ if (watcher) watcher.close();
+ watcher = OVJobs.watch(job.id, {
+ onUpdate: update,
+ onDone(done) {
+ watcher = null;
+ cancelButton().style.display = 'none';
+ if (done.state === 'succeeded') { resultId = done.id; return showResult(jobResult(done)); }
+ OVJobs.forget();
+ if (done.state !== 'cancelled') showToast((done.error && done.error.detail) || 'Processing failed', 'error');
+ resetUI();
+ },
+ onGone() { OVJobs.forget(); resetUI(); },
+ });
+ }
+
+ /** A finished job in the shape the synchronous endpoint answers with. */
+ function jobResult(job) {
+ const f = (job.result && job.result.files && job.result.files[0]) || {};
+ return {
+ ...(job.result && job.result.data),
+ download: { id: job.id, downloadUrl: f.url, previewUrl: f.url ? `${f.url}?inline=1` : null, filename: f.name, expiresIn: OVJobs.expiresIn(job) },
+ };
+ }
+
+ /** After a reload: pick the job up again from ?job=<id> or this tab's session. */
+ function reattach() {
+ const id = window.OVJobs && OVJobs.recall();
+ if (!id) return;
+ OVJobs.get(id).then(followJob).catch(() => OVJobs.forget());
+ }
+
/* ---------- Result ---------- */
/* ---------- Result ---------- */
function initResult() {
function initResult() {
anotherBtn.addEventListener('click', () => {
anotherBtn.addEventListener('click', () => {
+ if (window.OVJobs) OVJobs.forget();
clearFile();
clearFile();
resetUI();
resetUI();
});
});
@@ -473,8 +557,8 @@
@@ -473,8 +557,8 @@
resultPanel.style.display = '';
resultPanel.style.display = '';
const fileId = data.download.id;
const fileId = data.download.id;
- const previewUrl = `/api/preview/${fileId}`;
- const downloadUrl = `/api/download/${fileId}`;
+ const previewUrl = data.download.previewUrl || `/api/preview/${fileId}`;
+ const downloadUrl = data.download.downloadUrl || `/api/download/${fileId}`;
const outputMime = data.output?.mime || '';
const outputMime = data.output?.mime || '';
if (outputMime.startsWith('image/')) {
if (outputMime.startsWith('image/')) {
@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {
@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {
next();
next();
}
}
-module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey };
+/** The Network public key (PEM) once loaded; the job routes verify service tokens with it. */
+function getPublicKey() { return publicKey; }
+
+module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey, getPublicKey, ISSUER };
@@ -14,20 +14,25 @@ const path = require('path');
@@ -14,20 +14,25 @@ const path = require('path');
const fs = require('fs');
const fs = require('fs');
const config = require('./config');
const config = require('./config');
-const { optionalAuth } = require('./auth');
-const { resolveContext } = require('./domain-map');
+const auth = require('./auth');
+const { optionalAuth } = auth;
+const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { getTool, listTools } = require('./tools');
const { getTool, listTools } = require('./tools');
const { readMetadata } = require('./tools/metadata');
const { readMetadata } = require('./tools/metadata');
const { uploadSingle, uploadMultiple } = require('./middleware/upload');
const { uploadSingle, uploadMultiple } = require('./middleware/upload');
const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');
const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');
const retention = require('./retention/manager');
const retention = require('./retention/manager');
const { probe, getDuration, cleanTmp } = require('./tools/ffmpeg-helper');
const { probe, getDuration, cleanTmp } = require('./tools/ffmpeg-helper');
+const { buildOptions, describe, defineJobs } = require('./process');
+const { hostGuard, ownHost } = require('../../_shared/host-role');
+const jobsRuntime = require('../../_shared/jobs');
+const contracts = require('openvibe-contracts');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
const { AnalyticsTracker } = require('openvibe-shared/analytics');
const { AnalyticsTracker } = require('openvibe-shared/analytics');
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
-const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
+const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
@@ -47,8 +52,8 @@ app.use(helmet({
@@ -47,8 +52,8 @@ app.use(helmet({
scriptSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://openvibe.network"],
scriptSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://openvibe.network"],
styleSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://fonts.gstatic.com"],
styleSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://fonts.gstatic.com"],
fontSrc: ["'self'", "https://fonts.gstatic.com", "https://cdnjs.cloudflare.com"],
fontSrc: ["'self'", "https://fonts.gstatic.com", "https://cdnjs.cloudflare.com"],
- imgSrc: ["'self'", "data:", "blob:"],
- mediaSrc: ["'self'", "blob:"],
+ imgSrc: ["'self'", "data:", "blob:", jobsRuntime.mediaOrigin()], // job result previews are 302s to Media
+ mediaSrc: ["'self'", "blob:", jobsRuntime.mediaOrigin()],
connectSrc: ["'self'", "https://openvibe.network", "https://*.openvibe.tools"],
connectSrc: ["'self'", "https://openvibe.network", "https://*.openvibe.tools"],
workerSrc: ["'self'", "blob:"],
workerSrc: ["'self'", "blob:"],
scriptSrcAttr: ["'unsafe-inline'"],
scriptSrcAttr: ["'unsafe-inline'"],
@@ -58,6 +63,7 @@ app.use(helmet({
@@ -58,6 +63,7 @@ app.use(helmet({
}));
}));
app.use(cookieParser());
app.use(cookieParser());
app.use(express.json({ limit: '1mb' }));
app.use(express.json({ limit: '1mb' }));
+app.use(contracts.http.middleware()); // traceparent + X-OpenVibe-Request-Id on every response
// ── CORS ─────────────────────────────────────────────────────
// ── CORS ─────────────────────────────────────────────────────
app.use(cors({
app.use(cors({
@@ -79,9 +85,16 @@ app.use(analytics.middleware());
@@ -79,9 +85,16 @@ app.use(analytics.middleware());
// ── Auth (optional on all routes) ────────────────────────────
// ── Auth (optional on all routes) ────────────────────────────
app.use(optionalAuth);
app.use(optionalAuth);
+// ── Hosts ────────────────────────────────────────────────────
+// Through the gateway the X-OV-* headers name the tool and its canonical host (a custom domain
+// included); aliases the gateway missed are redirected. Hosts this app does not serve go to the
+// tools index. API paths are never redirected.
+const knowsHost = (h) => !!DOMAIN_MAP[h];
+app.use(hostGuard({ knows: knowsHost }));
+
// ── Attach domain context to every request ───────────────────
// ── Attach domain context to every request ───────────────────
app.use((req, _res, next) => {
app.use((req, _res, next) => {
- req.ctx = resolveContext(req.headers.host);
+ req.ctx = resolveContext(ownHost(req, knowsHost));
next();
next();
});
});
@@ -90,7 +103,7 @@ app.use((req, _res, next) => {
@@ -90,7 +103,7 @@ app.use((req, _res, next) => {
// Health check
// Health check
app.get('/api/health', (_req, res) => {
app.get('/api/health', (_req, res) => {
const stats = retention.getStats();
const stats = retention.getStats();
- res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats });
+ res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats, jobs: jobs.stats() });
});
});
// Domain context (frontend calls this on load to get branding)
// Domain context (frontend calls this on load to get branding)
@@ -138,18 +151,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
@@ -138,18 +151,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
return res.status(400).json({ error: `Unknown tool: ${toolId}` });
return res.status(400).json({ error: `Unknown tool: ${toolId}` });
}
}
- // Build options from body params
- const options = { ...req.body };
- delete options.tool;
- delete options.file;
-
- // If this is a format-specific domain, enforce that format
- if (req.ctx.defaultFormat && toolId === 'convert') {
- options.format = req.ctx.defaultFormat;
- }
-
- // Execute the tool
- const result = await tool.handler(req.file.path, options);
+ // Execute the tool (same options and code path as the audio.process job)
+ const result = await tool.handler(req.file.path, buildOptions(req.body, toolId, req.ctx));
// Clean up the uploaded input file
// Clean up the uploaded input file
cleanTmp(req.file.path);
cleanTmp(req.file.path);
@@ -166,24 +169,7 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
@@ -166,24 +169,7 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
// Probe input for size comparison
// Probe input for size comparison
const inputSize = req.file.size;
const inputSize = req.file.size;
- res.json({
- success: true,
- tool: toolId,
- download: saved,
- output: {
- mime: result.mime,
- ext: result.ext,
- size: saved.size,
- sizeKB: Math.round(saved.size / 1024 * 10) / 10,
- duration: result.duration || null,
- },
- input: {
- size: inputSize,
- sizeKB: Math.round(inputSize / 1024 * 10) / 10,
- },
- ...(result.metadata && { metadata: result.metadata }),
- ...(result.preset && { preset: result.preset }),
- });
+ res.json({ success: true, download: saved, ...describe(toolId, result, saved.size, inputSize) });
} catch (err) {
} catch (err) {
cleanTmp(req.file?.path);
cleanTmp(req.file?.path);
console.error('[Process] Error:', err.message);
console.error('[Process] Error:', err.message);
@@ -191,6 +177,24 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
@@ -191,6 +177,24 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
}
}
});
});
+// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
+// The same operation as /api/process, asynchronous and durable: accepted into data/jobs.db,
+// followed over SSE (ffmpeg's own progress), cancellable (ffmpeg is killed), reattachable by id
+// after a reload or a restart (apps/_shared/jobs).
+const jobs = jobsRuntime.setupJobs({
+ app, service: 'audio', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts,
+ getPublicKey: auth.getPublicKey, issuer: auth.ISSUER,
+ define: defineJobs,
+ receive: uploadSingle,
+ limiters: [burstLimiter, processLimiter],
+ defaults(req, input) {
+ const out = { ...input };
+ if (!out.tool) out.tool = req.ctx.defaultOp || 'convert';
+ if (req.ctx.defaultFormat && out.tool === 'convert') out.format = req.ctx.defaultFormat;
+ return out;
+ },
+});
+
// ── File Download ────────────────────────────────────────────
// ── File Download ────────────────────────────────────────────
app.get('/api/download/:id', (req, res) => {
app.get('/api/download/:id', (req, res) => {
const entry = retention.getFile(req.params.id);
const entry = retention.getFile(req.params.id);
@@ -291,6 +295,7 @@ function shutdown() {
@@ -291,6 +295,7 @@ function shutdown() {
console.log('[Audio.OpenVibe] Shutting down...');
console.log('[Audio.OpenVibe] Shutting down...');
analytics.destroy();
analytics.destroy();
analyticsDb.close();
analyticsDb.close();
+ jobs.close(); // running jobs stay 'running' in data/jobs.db; the next boot re-queues them
server.close(() => process.exit(0));
server.close(() => process.exit(0));
setTimeout(() => process.exit(1), 5000);
setTimeout(() => process.exit(1), 5000);
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,92 @@
@@ -0,0 +1,92 @@
+'use strict';
+
+// ═══════════════════════════════════════════════════════════════
+// Audio.OpenVibe — one audio operation, shared by the synchronous endpoint (/api/process) and the
+// audio.process job (defineJobs below).
+//
+// Jobs can be cancelled and report progress although the tools build their own ffmpeg commands:
+// FfmpegCommand#run is wrapped once, and a command started inside a job's AsyncLocalStorage
+// context reports ffmpeg's percent to the job and is killed when the job's signal aborts.
+// Commands started outside a job (the synchronous endpoint) are untouched.
+// ═══════════════════════════════════════════════════════════════
+
+const fs = require('fs');
+const path = require('path');
+const { AsyncLocalStorage } = require('async_hooks');
+const ffmpeg = require('fluent-ffmpeg');
+const { getTool, listTools } = require('./tools');
+
+const jobContext = new AsyncLocalStorage();
+const RESERVED = new Set(['tool', 'file', 'files']);
+
+const proto = ffmpeg.prototype;
+if (!proto.__ovJobs) {
+ const run = proto.run;
+ proto.run = function runInJob(...args) {
+ const job = jobContext.getStore();
+ if (job) {
+ const kill = () => {
+ if (this.ffmpegProc) { try { this.ffmpegProc.kill('SIGKILL'); } catch { /* already gone */ } } else this.once('start', () => { try { this.ffmpegProc.kill('SIGKILL'); } catch { /* gone */ } });
+ };
+ if (job.signal.aborted) kill(); else job.signal.addEventListener('abort', kill, { once: true });
+ this.on('progress', (p) => { if (p && Number.isFinite(p.percent)) job.progress(10 + Math.min(100, Math.max(0, p.percent)) * 0.8, 'Encoding'); });
+ }
+ return run.apply(this, args);
+ };
+ proto.__ovJobs = true;
+}
+
+/** Tool options from a request body or a job input, with the host's forced format. */
+function buildOptions(src, toolId, ctx) {
+ const options = {};
+ for (const [k, v] of Object.entries(src || {})) if (!RESERVED.has(k)) options[k] = v;
+ if (ctx && ctx.defaultFormat && toolId === 'convert') options.format = ctx.defaultFormat;
+ return options;
+}
+
+/** The JSON the synchronous endpoint has always answered with, minus the download block. */
+function describe(toolId, result, size, inputSize) {
+ return {
+ tool: toolId,
+ output: { mime: result.mime, ext: result.ext, size, sizeKB: Math.round(size / 1024 * 10) / 10, duration: result.duration || null },
+ input: { size: inputSize, sizeKB: Math.round(inputSize / 1024 * 10) / 10 },
+ ...(result.metadata && { metadata: result.metadata }),
+ ...(result.preset && { preset: result.preset }),
+ };
+}
+
+/** Job type audio.process: input { tool, …options } + one audio or video file. */
+function defineJobs(system) {
+ const known = new Set(listTools().map(t => t.id));
+ system.define({
+ type: 'audio.process',
+ version: 1,
+ minFiles: 1,
+ maxFiles: 1,
+ maxAttempts: 2,
+ onRestart: 'requeue', // ffmpeg over the stored input: safe to run again
+ timeoutMs: 15 * 60 * 1000,
+ validate(input) {
+ const id = String(input.tool || 'convert');
+ if (!known.has(id)) return `Unknown tool: ${id}`;
+ for (const [k, v] of Object.entries(input)) if (v != null && typeof v === 'object') return `Option ${k} must be a string or a number`;
+ return null;
+ },
+ async run({ input, files, progress, signal }) {
+ const toolId = String(input.tool || 'convert');
+ const tool = getTool(toolId);
+ const src = files[0];
+ progress(5, 'Reading the audio');
+ const result = await jobContext.run({ signal, progress }, () => tool.handler(src.path, buildOptions(input, toolId, null)));
+ if (signal.aborted) { fs.rm(result.outputPath, { force: true }, () => {}); throw new Error('cancelled'); }
+ const size = fs.statSync(result.outputPath).size;
+ const base = path.basename(src.name || 'audio', path.extname(src.name || '')) || 'openvibeaudio-output';
+ return {
+ files: [{ path: result.outputPath, name: `${base}.${result.ext}`, mime: result.mime }],
+ data: describe(toolId, result, size, src.size),
+ };
+ },
+ });
+}
+
+module.exports = { buildOptions, describe, defineJobs, jobContext };
@@ -8,10 +8,15 @@
@@ -8,10 +8,15 @@
// hosts as duplicates of each other and social shares have no card. This stamps each host's
// hosts as duplicates of each other and social shares have no card. This stamps each host's
// own title, description, canonical, Open Graph / Twitter tags and structured data into the
// own title, description, canonical, Open Graph / Twitter tags and structured data into the
// HTML on the way out. Rendered variants are cached per host; the file itself is read once.
// HTML on the way out. Rendered variants are cached per host; the file itself is read once.
+//
+// Canonical host: through the gateway (X-OV-* headers) the canonical, og:url and JSON-LD use the
+// host it names — a descriptive host or a custom domain; reached directly, the host itself.
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
const fs = require('fs');
const fs = require('fs');
const path = require('path');
const path = require('path');
+const { canonicalHostFor, ownHost } = require('../../_shared/host-role');
+const { DOMAIN_MAP } = require('./domain-map');
const PUBLIC = path.join(__dirname, '..', 'public');
const PUBLIC = path.join(__dirname, '..', 'public');
const HUB_HOST = 'audio.openvibe.tools';
const HUB_HOST = 'audio.openvibe.tools';
@@ -30,16 +35,18 @@ function baseHtml() {
@@ -30,16 +35,18 @@ function baseHtml() {
/**
/**
* Render index.html for one host with that host's own metadata.
* Render index.html for one host with that host's own metadata.
* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)
* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)
- * @param {string} host request hostname, e.g. 'png.openvibe.tools'
+ * @param {string} host the satellite's own hostname for this tool, e.g. 'png.openvibe.tools'
+ * @param {string} [canonicalHost] the host canonical URLs use (the gateway's; defaults to `host`)
*/
*/
-function renderIndex(ctx, host) {
- const key = host;
+function renderIndex(ctx, host, canonicalHost) {
+ const canon = canonicalHost || host;
+ const key = `${host}|${canon}`;
if (_cache.has(key)) return _cache.get(key);
if (_cache.has(key)) return _cache.get(key);
- const url = `https://${host}/`;
+ const url = `https://${canon}/`;
const title = ctx.seoTitle || ctx.brandName || HUB_NAME;
const title = ctx.seoTitle || ctx.brandName || HUB_NAME;
const desc = ctx.seoDescription || '';
const desc = ctx.seoDescription || '';
- const image = `https://${host}/og.png`;
+ const image = `https://${canon}/og.png`;
let html = baseHtml();
let html = baseHtml();
// Replace what the file already sets, so there is never a second competing value.
// Replace what the file already sets, so there is never a second competing value.
@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {
@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {
/** Express handler: the SPA with this host's SEO tags applied. */
/** Express handler: the SPA with this host's SEO tags applied. */
function sendIndex(req, res) {
function sendIndex(req, res) {
- const host = String(req.headers.host || HUB_HOST).split(':')[0].toLowerCase();
+ const own = ownHost(req, h => !!DOMAIN_MAP[h]);
+ const host = DOMAIN_MAP[own] ? own : HUB_HOST;
try {
try {
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Cache-Control', 'no-cache');
- return res.send(renderIndex(req.ctx || {}, host));
+ res.setHeader('Vary', 'X-OV-Canonical-Host');
+ return res.send(renderIndex(req.ctx || {}, host, canonicalHostFor(req, host)));
} catch (err) {
} catch (err) {
console.error('[SEO] render failed:', err.message);
console.error('[SEO] render failed:', err.message);
return res.sendFile(path.join(PUBLIC, 'index.html'));
return res.sendFile(path.join(PUBLIC, 'index.html'));
new file mode 100644
new file mode 100644
@@ -0,0 +1,84 @@
@@ -0,0 +1,84 @@
+'use strict';
+// Audio.OpenVibe as a real process, with real ffmpeg: a job RUNNING when the process is killed is
+// re-queued by the next process and finishes; a running job is cancelled (ffmpeg killed); progress
+// events come from ffmpeg itself.
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { execFileSync } = require('child_process');
+const { startApp } = require('../../_shared/test/spawn');
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); } catch {
+ console.log('audio jobs: SKIPPED (no ffmpeg on this machine)');
+ process.exit(0);
+}
+
+async function waitFor(app, id, cookie, pred, what, ms = 30000) {
+ const t0 = Date.now();
+ for (;;) {
+ const job = await (await fetch(`${app.base}/api/v1/jobs/${id}`, { headers: { cookie } })).json();
+ if (pred(job)) return job;
+ if (Date.now() - t0 > ms) throw new Error(`timed out waiting for ${what}: ${JSON.stringify(job)}`);
+ await sleep(50);
+ }
+}
+
+(async () => {
+ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-audio-'));
+ const env = { DATA_DIR: data, UPLOADS_DIR: path.join(data, 'uploads'), OUTPUT_DIR: path.join(data, 'output') };
+ // Three minutes of tone: small to upload, a couple of seconds to re-encode.
+ const input = path.join(data, 'tone.mp3');
+ execFileSync('ffmpeg', ['-loglevel', 'error', '-y', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=180', '-ac', '1', '-b:a', '16k', input]);
+ const bytes = fs.readFileSync(input);
+ const submit = async (app, cookie, tool) => {
+ const fd = new FormData();
+ fd.append('type', 'audio.process');
+ fd.append('input', JSON.stringify(tool));
+ fd.append('file', new Blob([bytes], { type: 'audio/mpeg' }), 'tone.mp3');
+ const r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd, headers: cookie ? { cookie } : {} });
+ assert.strictEqual(r.status, 202, await r.clone().text());
+ return { job: await r.json(), cookie: cookie || r.headers.get('set-cookie').split(';')[0] };
+ };
+ let app;
+ try {
+ app = await startApp('audio', env);
+
+ // Killed while ffmpeg runs → the next process runs it again.
+ const { job, cookie } = await submit(app, null, { tool: 'convert', format: 'flac' });
+ await waitFor(app, job.id, cookie, j => j.state === 'running', 'running');
+ await app.kill('SIGKILL');
+ app = await startApp('audio', env, app.port);
+ const done = await waitFor(app, job.id, cookie, j => j.state === 'succeeded' || j.state === 'failed', 'finished after restart');
+ assert.strictEqual(done.state, 'succeeded', JSON.stringify(done.error));
+ assert.strictEqual(done.attempts, 2, 'the interrupted attempt is counted');
+ assert.strictEqual(done.result.files[0].name, 'tone.flac');
+ const r = await fetch(`${app.base}${done.result.files[0].url}`, { headers: { cookie } });
+ assert.strictEqual(Buffer.from(await r.arrayBuffer()).slice(0, 4).toString(), 'fLaC', 'the result is FLAC');
+ const ranged = await fetch(`${app.base}${done.result.files[0].url}?inline=1`, { headers: { cookie, range: 'bytes=0-3' } });
+ assert.strictEqual(ranged.status, 206, 'previews support range requests (audio seeking)');
+
+ // Progress from ffmpeg, then cancel: ffmpeg is killed and the job ends cancelled.
+ const second = await submit(app, cookie, { tool: 'convert', format: 'mp3', bitrate: '320' });
+ const moving = await waitFor(app, second.job.id, cookie, j => j.state === 'running' && j.progress.message === 'Encoding' || j.state === 'succeeded', 'ffmpeg progress');
+ if (moving.state === 'running') {
+ const c = await fetch(`${app.base}/api/v1/jobs/${second.job.id}`, { method: 'DELETE', headers: { cookie } });
+ assert.ok([200, 202].includes(c.status));
+ const cancelled = await waitFor(app, second.job.id, cookie, j => j.state !== 'running', 'cancelled');
+ assert.strictEqual(cancelled.state, 'cancelled');
+ assert.strictEqual(cancelled.error.code, 'tools.job.cancelled');
+ } else {
+ console.log('audio jobs: (this machine encoded too fast to cancel mid-run; cancel is covered by the shared tests)');
+ }
+
+ await app.kill('SIGTERM');
+ console.log('audio jobs: all checks passed');
+ } catch (err) {
+ if (app) { console.error(app.output()); await app.kill('SIGKILL'); }
+ throw err;
+ } finally {
+ fs.rmSync(data, { recursive: true, force: true });
+ }
+})().catch((err) => { console.error(err); process.exit(1); });
@@ -10,16 +10,17 @@
@@ -10,16 +10,17 @@
},
},
"license": "MIT",
"license": "MIT",
"dependencies": {
"dependencies": {
+ "better-sqlite3": "^11.0.0",
"cookie-parser": "^1.4.7",
"cookie-parser": "^1.4.7",
"cors": "^2.8.5",
"cors": "^2.8.5",
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
- "better-sqlite3": "^11.0.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0",
"jsonwebtoken": "^9.0.2",
"jsonwebtoken": "^9.0.2",
"multer": "^1.4.5-lts.1",
"multer": "^1.4.5-lts.1",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.7.0",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0",
"pdf-lib": "^1.17.1",
"pdf-lib": "^1.17.1",
"sharp": "^0.33.5",
"sharp": "^0.33.5",
"uuid": "^11.1.0"
"uuid": "^11.1.0"
@@ -223,6 +223,7 @@
@@ -223,6 +223,7 @@
<script src="https://openvibe.network/shared/footer.js"></script>
<script src="https://openvibe.network/shared/footer.js"></script>
<script src="https://openvibe.network/shared/notification-ui.js"></script>
<script src="https://openvibe.network/shared/notification-ui.js"></script>
<script src="https://openvibe.network/shared/account-switcher.js"></script>
<script src="https://openvibe.network/shared/account-switcher.js"></script>
+ <script src="/js/ov-jobs.js"></script>
<script src="/js/app.js"></script>
<script src="/js/app.js"></script>
</body>
</body>
</html>
</html>
@@ -65,6 +65,7 @@
@@ -65,6 +65,7 @@
initOptions();
initOptions();
initSubdomains();
initSubdomains();
initResult();
initResult();
+ reattach();
}
}
// ── Branding ─────────────────────────────────────────────
// ── Branding ─────────────────────────────────────────────
@@ -434,6 +435,21 @@
@@ -434,6 +435,21 @@
// Add tool-specific options
// Add tool-specific options
addToolOptions(formData);
addToolOptions(formData);
+ // Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).
+ if (window.OVJobs) {
+ const input = {};
+ for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;
+ try {
+ const job = await OVJobs.submit({ type: 'docs.process', input, files: isMulti ? selectedFiles : [selectedFiles[0]], fileField: isMulti ? 'files' : 'file' });
+ OVJobs.remember(job.id);
+ followJob(job);
+ } catch (err) {
+ failProcessing(err.message);
+ }
+ return;
+ }
+
+ // Without the jobs helper: the synchronous endpoints, as before.
try {
try {
const token = getCookie('ov_token') || localStorage.getItem('ov_token');
const token = getCookie('ov_token') || localStorage.getItem('ov_token');
const headers = {};
const headers = {};
@@ -509,9 +525,85 @@
@@ -509,9 +525,85 @@
}
}
}
}
+ // ── Jobs ─────────────────────────────────────────────────
+ let watcher = null;
+ let currentJob = null;
+
+ function processingText(job) {
+ const pct = job.progress && job.progress.percent;
+ if (job.cancel_requested) return 'Cancelling...';
+ if (job.state === 'queued') return 'Waiting for a free slot...';
+ const label = (job.progress && job.progress.message) || 'Processing your document';
+ return pct != null ? `${label}... ${Math.round(pct)}%` : `${label}...`;
+ }
+
+ function cancelButton() {
+ let btn = processingEl.querySelector('.job-cancel');
+ if (!btn) {
+ btn = document.createElement('button');
+ btn.type = 'button';
+ btn.className = 'btn-secondary job-cancel';
+ btn.textContent = 'Cancel';
+ btn.addEventListener('click', () => { if (currentJob) OVJobs.cancel(currentJob).catch(() => {}); });
+ processingEl.appendChild(btn);
+ }
+ return btn;
+ }
+
+ function followJob(job) {
+ currentJob = job.id;
+ optionsPanel.style.display = 'none';
+ resultPanel.style.display = 'none';
+ processingEl.style.display = '';
+ processBtn.disabled = true;
+ const label = processingEl.querySelector('p');
+ const update = (j) => { if (label) label.textContent = processingText(j); };
+ cancelButton().style.display = '';
+ update(job);
+ if (watcher) watcher.close();
+ watcher = OVJobs.watch(job.id, {
+ onUpdate: update,
+ onDone(done) {
+ watcher = null;
+ cancelButton().style.display = 'none';
+ if (done.state === 'succeeded') {
+ const data = jobResult(done);
+ return data.viewOnly ? showMetadataResult(data) : showResult(data);
+ }
+ OVJobs.forget();
+ failProcessing(done.state === 'cancelled' ? null : (done.error && done.error.detail) || 'Processing failed');
+ },
+ onGone() { OVJobs.forget(); failProcessing(null); },
+ });
+ }
+
+ /** A finished job in the shape the synchronous endpoints answer with. */
+ function jobResult(job) {
+ const f = job.result && job.result.files && job.result.files[0];
+ return {
+ ...(job.result && job.result.data),
+ download: f ? { id: job.id, downloadUrl: f.url, filename: f.name, expiresIn: OVJobs.expiresIn(job) } : undefined,
+ };
+ }
+
+ function failProcessing(message) {
+ processingEl.style.display = 'none';
+ optionsPanel.style.display = selectedFiles.length ? '' : 'none';
+ processBtn.disabled = !selectedFiles.length;
+ if (message) alert(`Error: ${message}`);
+ }
+
+ /** After a reload: pick the job up again from ?job=<id> or this tab's session. */
+ function reattach() {
+ const id = window.OVJobs && OVJobs.recall();
+ if (!id) return;
+ OVJobs.get(id).then(followJob).catch(() => OVJobs.forget());
+ }
+
// ── Result ───────────────────────────────────────────────
// ── Result ───────────────────────────────────────────────
function initResult() {
function initResult() {
anotherBtn.addEventListener('click', () => {
anotherBtn.addEventListener('click', () => {
+ if (window.OVJobs) OVJobs.forget();
resetUpload();
resetUpload();
(toolTabs.style.display !== 'none' ? toolTabs : uploadArea).scrollIntoView({ behavior: 'smooth' });
(toolTabs.style.display !== 'none' ? toolTabs : uploadArea).scrollIntoView({ behavior: 'smooth' });
});
});
@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {
@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {
next();
next();
}
}
-module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey };
+/** The Network public key (PEM) once loaded; the job routes verify service tokens with it. */
+function getPublicKey() { return publicKey; }
+
+module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey, getPublicKey, ISSUER };
@@ -14,18 +14,23 @@ const path = require('path');
@@ -14,18 +14,23 @@ const path = require('path');
const fs = require('fs');
const fs = require('fs');
const config = require('./config');
const config = require('./config');
-const { optionalAuth } = require('./auth');
-const { resolveContext } = require('./domain-map');
+const auth = require('./auth');
+const { optionalAuth } = auth;
+const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { getTool, listTools } = require('./tools');
const { getTool, listTools } = require('./tools');
-const { uploadSingle, uploadMultiple } = require('./middleware/upload');
+const { uploadSingle, uploadMultiple, uploadAny } = require('./middleware/upload');
const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');
const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');
const retention = require('./retention/manager');
const retention = require('./retention/manager');
+const { buildOptions, defineJobs } = require('./process');
+const { hostGuard, ownHost } = require('../../_shared/host-role');
+const jobsRuntime = require('../../_shared/jobs');
+const contracts = require('openvibe-contracts');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
const { AnalyticsTracker } = require('openvibe-shared/analytics');
const { AnalyticsTracker } = require('openvibe-shared/analytics');
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
-const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
+const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
@@ -55,6 +60,7 @@ app.use(helmet({
@@ -55,6 +60,7 @@ app.use(helmet({
}));
}));
app.use(cookieParser());
app.use(cookieParser());
app.use(express.json({ limit: '1mb' }));
app.use(express.json({ limit: '1mb' }));
+app.use(contracts.http.middleware()); // traceparent + X-OpenVibe-Request-Id on every response
// ── CORS ─────────────────────────────────────────────────────
// ── CORS ─────────────────────────────────────────────────────
app.use(cors({
app.use(cors({
@@ -76,9 +82,16 @@ app.use(analytics.middleware());
@@ -76,9 +82,16 @@ app.use(analytics.middleware());
// ── Auth (optional on all routes) ────────────────────────────
// ── Auth (optional on all routes) ────────────────────────────
app.use(optionalAuth);
app.use(optionalAuth);
+// ── Hosts ────────────────────────────────────────────────────
+// Through the gateway the X-OV-* headers name the tool and its canonical host (a custom domain
+// included); aliases the gateway missed are redirected. Hosts this app does not serve go to the
+// tools index. API paths are never redirected.
+const knowsHost = (h) => !!DOMAIN_MAP[h];
+app.use(hostGuard({ knows: knowsHost }));
+
// ── Attach domain context to every request ───────────────────
// ── Attach domain context to every request ───────────────────
app.use((req, _res, next) => {
app.use((req, _res, next) => {
- req.ctx = resolveContext(req.headers.host);
+ req.ctx = resolveContext(ownHost(req, knowsHost));
next();
next();
});
});
@@ -87,7 +100,7 @@ app.use((req, _res, next) => {
@@ -87,7 +100,7 @@ app.use((req, _res, next) => {
// Health check
// Health check
app.get('/api/health', (_req, res) => {
app.get('/api/health', (_req, res) => {
const stats = retention.getStats();
const stats = retention.getStats();
- res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats });
+ res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats, jobs: jobs.stats() });
});
});
// Domain context (frontend calls this on load to get branding)
// Domain context (frontend calls this on load to get branding)
@@ -133,22 +146,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
@@ -133,22 +146,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
const tool = getTool(toolId);
const tool = getTool(toolId);
if (!tool) return res.status(400).json({ error: `Unknown tool: ${toolId}` });
if (!tool) return res.status(400).json({ error: `Unknown tool: ${toolId}` });
- // Build options from body params
- const options = {};
- for (const key of ['format', 'quality', 'angle', 'pages', 'order', 'ranges', 'mode',
- 'text', 'fontSize', 'opacity', 'rotation', 'color',
- 'pageSize', 'dpi', 'password', 'userPassword', 'ownerPassword',
- 'title', 'author', 'subject', 'keywords', 'creator',
- 'level', 'defaultFormat']) {
- if (req.body[key] !== undefined) options[key] = req.body[key];
- }
-
- // Use context default format if available
- if (req.ctx.defaultFormat && !options.format) {
- options.defaultFormat = req.ctx.defaultFormat;
- }
-
- const result = await tool.handler(req.file.buffer, options);
+ // Same options and code path as the docs.process job
+ const result = await tool.handler(req.file.buffer, buildOptions(req.body, req.ctx, false));
// Some tools return view-only results (like metadata view)
// Some tools return view-only results (like metadata view)
if (result.viewOnly) {
if (result.viewOnly) {
@@ -196,12 +195,7 @@ app.post('/api/process/multi', burstLimiter, processLimiter, uploadMultiple, asy
@@ -196,12 +195,7 @@ app.post('/api/process/multi', burstLimiter, processLimiter, uploadMultiple, asy
if (!tool.multiFile) return res.status(400).json({ error: `Tool "${toolId}" does not support multiple files. Use /api/process instead.` });
if (!tool.multiFile) return res.status(400).json({ error: `Tool "${toolId}" does not support multiple files. Use /api/process instead.` });
const buffers = req.files.map(f => f.buffer);
const buffers = req.files.map(f => f.buffer);
- const options = {};
- for (const key of ['order', 'pageSize', 'quality', 'format']) {
- if (req.body[key] !== undefined) options[key] = req.body[key];
- }
-
- const result = await tool.handler(buffers, options);
+ const result = await tool.handler(buffers, buildOptions(req.body, req.ctx, true));
// Save output
// Save output
const firstName = req.files[0]?.originalname || 'output';
const firstName = req.files[0]?.originalname || 'output';
@@ -232,6 +226,23 @@ app.post('/api/process/multi', burstLimiter, processLimiter, uploadMultiple, asy
@@ -232,6 +226,23 @@ app.post('/api/process/multi', burstLimiter, processLimiter, uploadMultiple, asy
}
}
});
});
+// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
+// The same operations as /api/process and /api/process/multi, asynchronous and durable: accepted
+// into data/jobs.db, followed over SSE, reattachable by id after a reload or a restart.
+const jobs = jobsRuntime.setupJobs({
+ app, service: 'docs', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts,
+ getPublicKey: auth.getPublicKey, issuer: auth.ISSUER,
+ define: defineJobs,
+ receive: uploadAny,
+ limiters: [burstLimiter, processLimiter],
+ defaults(req, input) {
+ const out = { ...input };
+ if (!out.tool && req.ctx.defaultOp) out.tool = req.ctx.defaultOp;
+ if (!out.format && !out.defaultFormat && req.ctx.defaultFormat) out.defaultFormat = req.ctx.defaultFormat;
+ return out;
+ },
+});
+
// ── File Download ────────────────────────────────────────────
// ── File Download ────────────────────────────────────────────
app.get('/api/download/:id', (req, res) => {
app.get('/api/download/:id', (req, res) => {
const entry = retention.getFile(req.params.id);
const entry = retention.getFile(req.params.id);
@@ -301,6 +312,7 @@ function shutdown() {
@@ -301,6 +312,7 @@ function shutdown() {
console.log('[Docs.OpenVibe] Shutting down...');
console.log('[Docs.OpenVibe] Shutting down...');
analytics.destroy();
analytics.destroy();
analyticsDb.close();
analyticsDb.close();
+ jobs.close(); // running jobs stay 'running' in data/jobs.db; the next boot re-queues them
server.close(() => process.exit(0));
server.close(() => process.exit(0));
setTimeout(() => process.exit(1), 5000);
setTimeout(() => process.exit(1), 5000);
}
}
@@ -69,4 +69,25 @@ function uploadMultiple(req, res, next) {
@@ -69,4 +69,25 @@ function uploadMultiple(req, res, next) {
});
});
}
}
-module.exports = { uploadSingle, uploadMultiple };
+/**
+ * One file in 'file' or several in 'files' (the job endpoint takes either shape).
+ * Attaches req.files as { file?: [...], files?: [...] }.
+ */
+function uploadAny(req, res, next) {
+ upload.fields([{ name: 'file', maxCount: 1 }, { name: 'files', maxCount: 50 }])(req, res, (err) => {
+ if (err) {
+ if (err.code === 'LIMIT_FILE_SIZE') {
+ return res.status(413).json({ error: `File too large. Maximum ${Math.round(config.upload.maxFileSize / 1024 / 1024)}MB per file.` });
+ }
+ if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') {
+ return res.status(400).json({ error: 'Send one file in "file" or up to 50 in "files".' });
+ }
+ return res.status(400).json({ error: err.message });
+ }
+ const n = req.files ? Object.values(req.files).flat().length : 0;
+ if (!n) return res.status(400).json({ error: 'No files uploaded. Send a file in the "file" field.' });
+ next();
+ });
+}
+
+module.exports = { uploadSingle, uploadMultiple, uploadAny };
new file mode 100644
new file mode 100644
@@ -0,0 +1,82 @@
@@ -0,0 +1,82 @@
+'use strict';
+
+// ═══════════════════════════════════════════════════════════════
+// Docs.OpenVibe — one document operation, shared by the synchronous endpoints
+// (/api/process, /api/process/multi) and the docs.process job (see defineJobs).
+// ═══════════════════════════════════════════════════════════════
+
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const { getTool, listTools } = require('./tools');
+
+const SINGLE_KEYS = ['format', 'quality', 'angle', 'pages', 'order', 'ranges', 'mode',
+ 'text', 'fontSize', 'opacity', 'rotation', 'color',
+ 'pageSize', 'dpi', 'password', 'userPassword', 'ownerPassword',
+ 'title', 'author', 'subject', 'keywords', 'creator',
+ 'level', 'defaultFormat'];
+const MULTI_KEYS = ['order', 'pageSize', 'quality', 'format'];
+
+/** Tool options from a request body or a job input. */
+function buildOptions(src, ctx, multi) {
+ const options = {};
+ for (const key of multi ? MULTI_KEYS : SINGLE_KEYS) if (src[key] !== undefined) options[key] = src[key];
+ if (!multi && ctx && ctx.defaultFormat && !options.format) options.defaultFormat = ctx.defaultFormat;
+ return options;
+}
+
+/** The JSON the synchronous endpoints have always answered with, minus the download block. */
+function describe(toolId, result, fileCount) {
+ if (result.viewOnly) { const { buffer, ...rest } = result; return { tool: toolId, ...rest }; }
+ return {
+ tool: toolId,
+ output: { mime: result.mime, ext: result.ext, size: result.buffer.length, sizeKB: Math.round(result.buffer.length / 1024 * 10) / 10 },
+ ...(fileCount != null && { fileCount }),
+ ...(result.pageCount !== undefined && { pageCount: result.pageCount }),
+ ...(result.savings && { savings: result.savings }),
+ ...(result.metadata && { metadata: result.metadata }),
+ ...(result.note && { note: result.note }),
+ };
+}
+
+/** Job type docs.process: input { tool, …options } + one PDF, or several files for merge / img2pdf. */
+function defineJobs(system) {
+ const known = new Set(listTools().map(t => t.id));
+ system.define({
+ type: 'docs.process',
+ version: 1,
+ minFiles: 1,
+ maxFiles: 50,
+ maxAttempts: 3,
+ onRestart: 'requeue', // a pure function of the stored input: safe to run again
+ timeoutMs: 10 * 60 * 1000,
+ validate(input, files) {
+ const id = String(input.tool || '');
+ if (!known.has(id)) return id ? `Unknown tool: ${id}` : 'No tool specified.';
+ const tool = getTool(id);
+ if (!tool.multiFile && files.length !== 1) return `Tool "${id}" takes exactly one file`;
+ if (id === 'merge' && files.length < 2) return 'Upload at least 2 files to merge.';
+ for (const k of Object.keys(input)) if (k !== 'tool' && !SINGLE_KEYS.includes(k)) return `Unknown option: ${k}`;
+ return null;
+ },
+ async run({ input, files, outDir, progress, signal }) {
+ const toolId = String(input.tool);
+ const tool = getTool(toolId);
+ progress(5, files.length > 1 ? `Reading ${files.length} files` : 'Reading the document');
+ const buffers = [];
+ for (const f of files) buffers.push(await fsp.readFile(f.path));
+ if (signal.aborted) throw new Error('cancelled');
+ progress(20, 'Processing');
+ const result = await tool.handler(tool.multiFile ? buffers : buffers[0], buildOptions(input, null, tool.multiFile));
+ if (result.viewOnly) return { files: [], data: describe(toolId, result) };
+ progress(90, 'Saving');
+ const first = files[0].name || 'document';
+ const name = `${path.basename(first, path.extname(first)) || 'openvibedocs-output'}.${result.ext}`;
+ const out = path.join(outDir, `result.${result.ext}`);
+ fs.writeFileSync(out, result.buffer);
+ return { files: [{ path: out, name, mime: result.mime }], data: describe(toolId, result, tool.multiFile ? files.length : undefined) };
+ },
+ });
+}
+
+module.exports = { buildOptions, describe, defineJobs, SINGLE_KEYS, MULTI_KEYS };
@@ -8,10 +8,15 @@
@@ -8,10 +8,15 @@
// hosts as duplicates of each other and social shares have no card. This stamps each host's
// hosts as duplicates of each other and social shares have no card. This stamps each host's
// own title, description, canonical, Open Graph / Twitter tags and structured data into the
// own title, description, canonical, Open Graph / Twitter tags and structured data into the
// HTML on the way out. Rendered variants are cached per host; the file itself is read once.
// HTML on the way out. Rendered variants are cached per host; the file itself is read once.
+//
+// Canonical host: through the gateway (X-OV-* headers) the canonical, og:url and JSON-LD use the
+// host it names — a descriptive host or a custom domain; reached directly, the host itself.
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
const fs = require('fs');
const fs = require('fs');
const path = require('path');
const path = require('path');
+const { canonicalHostFor, ownHost } = require('../../_shared/host-role');
+const { DOMAIN_MAP } = require('./domain-map');
const PUBLIC = path.join(__dirname, '..', 'public');
const PUBLIC = path.join(__dirname, '..', 'public');
const HUB_HOST = 'docs.openvibe.tools';
const HUB_HOST = 'docs.openvibe.tools';
@@ -30,16 +35,18 @@ function baseHtml() {
@@ -30,16 +35,18 @@ function baseHtml() {
/**
/**
* Render index.html for one host with that host's own metadata.
* Render index.html for one host with that host's own metadata.
* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)
* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)
- * @param {string} host request hostname, e.g. 'png.openvibe.tools'
+ * @param {string} host the satellite's own hostname for this tool, e.g. 'png.openvibe.tools'
+ * @param {string} [canonicalHost] the host canonical URLs use (the gateway's; defaults to `host`)
*/
*/
-function renderIndex(ctx, host) {
- const key = host;
+function renderIndex(ctx, host, canonicalHost) {
+ const canon = canonicalHost || host;
+ const key = `${host}|${canon}`;
if (_cache.has(key)) return _cache.get(key);
if (_cache.has(key)) return _cache.get(key);
- const url = `https://${host}/`;
+ const url = `https://${canon}/`;
const title = ctx.seoTitle || ctx.brandName || HUB_NAME;
const title = ctx.seoTitle || ctx.brandName || HUB_NAME;
const desc = ctx.seoDescription || '';
const desc = ctx.seoDescription || '';
- const image = `https://${host}/og.png`;
+ const image = `https://${canon}/og.png`;
let html = baseHtml();
let html = baseHtml();
// Replace what the file already sets, so there is never a second competing value.
// Replace what the file already sets, so there is never a second competing value.
@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {
@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {
/** Express handler: the SPA with this host's SEO tags applied. */
/** Express handler: the SPA with this host's SEO tags applied. */
function sendIndex(req, res) {
function sendIndex(req, res) {
- const host = String(req.headers.host || HUB_HOST).split(':')[0].toLowerCase();
+ const own = ownHost(req, h => !!DOMAIN_MAP[h]);
+ const host = DOMAIN_MAP[own] ? own : HUB_HOST;
try {
try {
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Cache-Control', 'no-cache');
- return res.send(renderIndex(req.ctx || {}, host));
+ res.setHeader('Vary', 'X-OV-Canonical-Host');
+ return res.send(renderIndex(req.ctx || {}, host, canonicalHostFor(req, host)));
} catch (err) {
} catch (err) {
console.error('[SEO] render failed:', err.message);
console.error('[SEO] render failed:', err.message);
return res.sendFile(path.join(PUBLIC, 'index.html'));
return res.sendFile(path.join(PUBLIC, 'index.html'));
new file mode 100644
new file mode 100644
@@ -0,0 +1,102 @@
@@ -0,0 +1,102 @@
+'use strict';
+// Docs.OpenVibe as a real process: a multi-file merge job accepted before a SIGKILL finishes after
+// the restart; single-file and view-only jobs; the synchronous endpoints still answer; canonical host.
+const assert = require('assert');
+const fs = require('fs');
+const http = require('http');
+const os = require('os');
+const path = require('path');
+const { PDFDocument } = require('pdf-lib');
+const { startApp } = require('../../_shared/test/spawn');
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+async function pdf(pages, title) {
+ const doc = await PDFDocument.create();
+ for (let i = 0; i < pages; i++) doc.addPage([200, 300]);
+ if (title) doc.setTitle(title);
+ return Buffer.from(await doc.save());
+}
+
+async function finished(app, id, cookie) {
+ for (let i = 0; i < 150; i++) {
+ const job = await (await fetch(`${app.base}/api/v1/jobs/${id}`, { headers: { cookie } })).json();
+ if (['succeeded', 'failed', 'cancelled'].includes(job.state)) return job;
+ await sleep(100);
+ }
+ throw new Error(`job ${id} did not finish`);
+}
+
+(async () => {
+ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-docs-'));
+ const env = { DATA_DIR: data, UPLOADS_DIR: path.join(data, 'uploads'), OUTPUT_DIR: path.join(data, 'output') };
+ const a = await pdf(2, 'First'), b = await pdf(3);
+ let app;
+ try {
+ app = await startApp('docs', { ...env, TOOLS_JOBS_CONCURRENCY_DOCS: '0' });
+ const fd = new FormData();
+ fd.append('type', 'docs.process');
+ fd.append('input', JSON.stringify({ tool: 'merge' }));
+ fd.append('files', new Blob([a], { type: 'application/pdf' }), 'first.pdf');
+ fd.append('files', new Blob([b], { type: 'application/pdf' }), 'second.pdf');
+ let r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd });
+ assert.strictEqual(r.status, 202, await r.clone().text());
+ const cookie = r.headers.get('set-cookie').split(';')[0];
+ const merge = await r.json();
+ await app.kill('SIGKILL');
+
+ app = await startApp('docs', env, app.port);
+ const merged = await finished(app, merge.id, cookie);
+ assert.strictEqual(merged.state, 'succeeded', JSON.stringify(merged.error));
+ assert.strictEqual(merged.result.data.pageCount, 5);
+ assert.strictEqual(merged.result.data.fileCount, 2);
+ r = await fetch(`${app.base}${merged.result.files[0].url}`, { headers: { cookie } });
+ assert.strictEqual((await PDFDocument.load(Buffer.from(await r.arrayBuffer()))).getPageCount(), 5, 'the merged PDF has every page');
+
+ // Single file, and a view-only tool (no result file, only data).
+ const one = async (input, file = a) => {
+ const f = new FormData();
+ f.append('type', 'docs.process'); f.append('input', JSON.stringify(input));
+ f.append('file', new Blob([file], { type: 'application/pdf' }), 'doc.pdf');
+ const res = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: f, headers: { cookie } });
+ return finished(app, (await res.json()).id, cookie);
+ };
+ const rotated = await one({ tool: 'rotate', angle: '90', pages: 'all' });
+ assert.strictEqual(rotated.state, 'succeeded');
+ assert.strictEqual(rotated.result.files[0].name, 'doc.pdf');
+ const info = await one({ tool: 'metadata', mode: 'view' });
+ assert.strictEqual(info.state, 'succeeded');
+ assert.strictEqual(info.result.files.length, 0);
+ assert.strictEqual(info.result.data.viewOnly, true);
+ assert.strictEqual(info.result.data.metadata.title, 'First');
+ const bad = await one({ tool: 'rotate', angle: '90' }, Buffer.from('%PDF-1.4 not really'));
+ assert.strictEqual(bad.state, 'failed');
+ assert.strictEqual(bad.error.code, 'tools.job.failed');
+ r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', headers: { 'content-type': 'application/json', cookie }, body: JSON.stringify({ type: 'docs.process', input: { tool: 'merge' } }) });
+ assert.strictEqual(r.status, 400, 'a job without files is refused');
+
+ // The synchronous endpoint is unchanged (after the burst limiter's 5 s window: jobs count too).
+ await sleep(5100);
+ const sync = new FormData();
+ sync.append('files', new Blob([a], { type: 'application/pdf' }), 'a.pdf');
+ sync.append('files', new Blob([b], { type: 'application/pdf' }), 'b.pdf');
+ sync.append('tool', 'merge');
+ r = await fetch(`${app.base}/api/process/multi`, { method: 'POST', body: sync });
+ const body = await r.json();
+ assert.strictEqual(body.success, true, JSON.stringify(body)); assert.strictEqual(body.pageCount, 5);
+
+ // Canonical host through the gateway.
+ const html = await new Promise((resolve, reject) => http.get({ host: '127.0.0.1', port: app.port, path: '/', headers: { Host: 'mergepdf.openvibe.tools', 'X-OV-Tool': 'mergepdf', 'X-OV-Host-Role': 'short', 'X-OV-Canonical-Host': 'merge-pdf.openvibe.tools' } }, (res) => {
+ let t = ''; res.setEncoding('utf8'); res.on('data', c => { t += c; }); res.on('end', () => resolve(t));
+ }).on('error', reject));
+ assert.ok(html.includes('<link rel="canonical" href="https://merge-pdf.openvibe.tools/">'), 'canonical follows X-OV-Canonical-Host');
+
+ await app.kill('SIGTERM');
+ console.log('docs jobs + canonical host: all checks passed');
+ } catch (err) {
+ if (app) { console.error(app.output()); await app.kill('SIGKILL'); }
+ throw err;
+ } finally {
+ fs.rmSync(data, { recursive: true, force: true });
+ }
+})().catch((err) => { console.error(err); process.exit(1); });
@@ -10,16 +10,17 @@
@@ -10,16 +10,17 @@
},
},
"license": "MIT",
"license": "MIT",
"dependencies": {
"dependencies": {
+ "better-sqlite3": "^11.0.0",
"cookie-parser": "^1.4.7",
"cookie-parser": "^1.4.7",
"cors": "^2.8.5",
"cors": "^2.8.5",
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
- "better-sqlite3": "^11.0.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0",
"jsonwebtoken": "^9.0.2",
"jsonwebtoken": "^9.0.2",
"multer": "^1.4.5-lts.1",
"multer": "^1.4.5-lts.1",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.7.0",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.0.0",
"sharp": "^0.33.5",
"sharp": "^0.33.5",
"to-ico": "^1.1.5",
"to-ico": "^1.1.5",
"uuid": "^11.1.0"
"uuid": "^11.1.0"
@@ -172,6 +172,7 @@
@@ -172,6 +172,7 @@
<script src="https://openvibe.network/shared/footer.js"></script>
<script src="https://openvibe.network/shared/footer.js"></script>
<script src="https://openvibe.network/shared/notification-ui.js"></script>
<script src="https://openvibe.network/shared/notification-ui.js"></script>
<script src="https://openvibe.network/shared/account-switcher.js"></script>
<script src="https://openvibe.network/shared/account-switcher.js"></script>
+ <script src="/js/ov-jobs.js"></script>
<script src="/js/app.js"></script>
<script src="/js/app.js"></script>
</body>
</body>
</html>
</html>
@@ -64,6 +64,7 @@
@@ -64,6 +64,7 @@
initOptions();
initOptions();
initSubdomains();
initSubdomains();
initResult();
initResult();
+ reattach();
}
}
// ── Branding ─────────────────────────────────────────────
// ── Branding ─────────────────────────────────────────────
@@ -356,6 +357,21 @@
@@ -356,6 +357,21 @@
}
}
}
}
+ // Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).
+ if (window.OVJobs) {
+ const input = {};
+ for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;
+ try {
+ const job = await OVJobs.submit({ type: 'img.process', input, files: [selectedFile] });
+ OVJobs.remember(job.id);
+ followJob(job);
+ } catch (err) {
+ failProcessing(err.message);
+ }
+ return;
+ }
+
+ // Without the jobs helper: the synchronous endpoint, as before.
try {
try {
const token = getCookie('ov_token') || localStorage.getItem('ov_token');
const token = getCookie('ov_token') || localStorage.getItem('ov_token');
const headers = {};
const headers = {};
@@ -378,9 +394,82 @@
@@ -378,9 +394,82 @@
}
}
}
}
+ // ── Jobs ─────────────────────────────────────────────────
+ let watcher = null;
+ let currentJob = null;
+
+ function processingText(job) {
+ const pct = job.progress && job.progress.percent;
+ if (job.cancel_requested) return 'Cancelling...';
+ if (job.state === 'queued') return 'Waiting for a free slot...';
+ const label = (job.progress && job.progress.message) || 'Processing your image';
+ return pct != null ? `${label}... ${Math.round(pct)}%` : `${label}...`;
+ }
+
+ function cancelButton() {
+ let btn = processingEl.querySelector('.job-cancel');
+ if (!btn) {
+ btn = document.createElement('button');
+ btn.type = 'button';
+ btn.className = 'btn-secondary job-cancel';
+ btn.textContent = 'Cancel';
+ btn.addEventListener('click', () => { if (currentJob) OVJobs.cancel(currentJob).catch(() => {}); });
+ processingEl.appendChild(btn);
+ }
+ return btn;
+ }
+
+ function followJob(job) {
+ currentJob = job.id;
+ optionsPanel.style.display = 'none';
+ resultPanel.style.display = 'none';
+ processingEl.style.display = '';
+ processBtn.disabled = true;
+ const label = processingEl.querySelector('p');
+ const update = (j) => { if (label) label.textContent = processingText(j); };
+ cancelButton().style.display = '';
+ update(job);
+ if (watcher) watcher.close();
+ watcher = OVJobs.watch(job.id, {
+ onUpdate: update,
+ onDone(done) {
+ watcher = null;
+ cancelButton().style.display = 'none';
+ if (done.state === 'succeeded') return showResult(jobResult(done));
+ OVJobs.forget();
+ failProcessing(done.state === 'cancelled' ? null : (done.error && done.error.detail) || 'Processing failed');
+ },
+ onGone() { OVJobs.forget(); failProcessing(null); },
+ });
+ }
+
+ /** A finished job in the shape the synchronous endpoint answers with. */
+ function jobResult(job) {
+ const f = (job.result && job.result.files && job.result.files[0]) || {};
+ return {
+ ...(job.result && job.result.data),
+ download: { id: job.id, downloadUrl: f.url, previewUrl: f.url ? `${f.url}?inline=1` : null, filename: f.name, expiresIn: OVJobs.expiresIn(job) },
+ };
+ }
+
+ function failProcessing(message) {
+ processingEl.style.display = 'none';
+ optionsPanel.style.display = selectedFile ? '' : 'none';
+ processBtn.disabled = !selectedFile;
+ if (message) alert(`Error: ${message}`);
+ }
+
+ /** After a reload: pick the job up again from ?job=<id> or this tab's session. */
+ function reattach() {
+ const id = window.OVJobs && OVJobs.recall();
+ if (!id) return;
+ OVJobs.get(id).then(followJob).catch(() => OVJobs.forget());
+ }
+
// ── Result ───────────────────────────────────────────────
// ── Result ───────────────────────────────────────────────
function initResult() {
function initResult() {
anotherBtn.addEventListener('click', () => {
anotherBtn.addEventListener('click', () => {
+ if (window.OVJobs) OVJobs.forget();
resetUpload();
resetUpload();
toolTabs.scrollIntoView({ behavior: 'smooth' });
toolTabs.scrollIntoView({ behavior: 'smooth' });
});
});
@@ -393,7 +482,10 @@
@@ -393,7 +482,10 @@
// Preview image (if it's a previewable type)
// Preview image (if it's a previewable type)
const previewable = ['image/png', 'image/jpeg', 'image/webp', 'image/gif', 'image/bmp', 'image/avif'];
const previewable = ['image/png', 'image/jpeg', 'image/webp', 'image/gif', 'image/bmp', 'image/avif'];
if (previewable.includes(data.output.mime)) {
if (previewable.includes(data.output.mime)) {
- resultPreview.innerHTML = `<img src="${data.download.downloadUrl}" alt="Result">`;
+ const img = document.createElement('img');
+ img.src = data.download.previewUrl || data.download.downloadUrl;
+ img.alt = 'Result';
+ resultPreview.replaceChildren(img);
} else {
} else {
resultPreview.innerHTML = `<div style="font-size:48px;color:var(--accent);padding:24px"><i class="fa-solid fa-file-image"></i></div>`;
resultPreview.innerHTML = `<div style="font-size:48px;color:var(--accent);padding:24px"><i class="fa-solid fa-file-image"></i></div>`;
}
}
@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {
@@ -114,4 +114,7 @@ function requireAuth(req, res, next) {
next();
next();
}
}
-module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey };
+/** The Network public key (PEM) once loaded; the job routes verify service tokens with it. */
+function getPublicKey() { return publicKey; }
+
+module.exports = { extractToken, verifyToken, optionalAuth, requireAuth, loadPublicKey, getPublicKey, ISSUER };
@@ -14,18 +14,23 @@ const path = require('path');
@@ -14,18 +14,23 @@ const path = require('path');
const fs = require('fs');
const fs = require('fs');
const config = require('./config');
const config = require('./config');
-const { optionalAuth } = require('./auth');
-const { resolveContext } = require('./domain-map');
+const auth = require('./auth');
+const { optionalAuth } = auth;
+const { resolveContext, DOMAIN_MAP } = require('./domain-map');
const { getTool, listTools } = require('./tools');
const { getTool, listTools } = require('./tools');
const { uploadSingle } = require('./middleware/upload');
const { uploadSingle } = require('./middleware/upload');
const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');
const { apiLimiter, processLimiter, burstLimiter } = require('./middleware/rate-limit');
const retention = require('./retention/manager');
const retention = require('./retention/manager');
+const { buildOptions, describe, processBuffer, defineJobs } = require('./process');
+const { hostGuard, ownHost } = require('../../_shared/host-role');
+const jobsRuntime = require('../../_shared/jobs');
+const contracts = require('openvibe-contracts');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
const { AnalyticsTracker } = require('openvibe-shared/analytics');
const { AnalyticsTracker } = require('openvibe-shared/analytics');
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
-const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
+const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
@@ -45,7 +50,7 @@ app.use(helmet({
@@ -45,7 +50,7 @@ app.use(helmet({
scriptSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://openvibe.network"],
scriptSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://openvibe.network"],
styleSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://fonts.gstatic.com"],
styleSrc: ["'self'", "'unsafe-inline'", "https://cdnjs.cloudflare.com", "https://fonts.googleapis.com", "https://fonts.gstatic.com"],
fontSrc: ["'self'", "https://fonts.gstatic.com", "https://cdnjs.cloudflare.com"],
fontSrc: ["'self'", "https://fonts.gstatic.com", "https://cdnjs.cloudflare.com"],
- imgSrc: ["'self'", "data:", "blob:"],
+ imgSrc: ["'self'", "data:", "blob:", jobsRuntime.mediaOrigin()], // job result previews are 302s to Media
connectSrc: ["'self'", "https://openvibe.network", "https://*.openvibe.tools"],
connectSrc: ["'self'", "https://openvibe.network", "https://*.openvibe.tools"],
workerSrc: ["'self'", "blob:"],
workerSrc: ["'self'", "blob:"],
scriptSrcAttr: ["'unsafe-inline'"],
scriptSrcAttr: ["'unsafe-inline'"],
@@ -55,6 +60,7 @@ app.use(helmet({
@@ -55,6 +60,7 @@ app.use(helmet({
}));
}));
app.use(cookieParser());
app.use(cookieParser());
app.use(express.json({ limit: '1mb' }));
app.use(express.json({ limit: '1mb' }));
+app.use(contracts.http.middleware()); // traceparent + X-OpenVibe-Request-Id on every response
// ── CORS ─────────────────────────────────────────────────────
// ── CORS ─────────────────────────────────────────────────────
app.use(cors({
app.use(cors({
@@ -77,9 +83,16 @@ app.use(analytics.middleware());
@@ -77,9 +83,16 @@ app.use(analytics.middleware());
// ── Auth (optional on all routes) ────────────────────────────
// ── Auth (optional on all routes) ────────────────────────────
app.use(optionalAuth);
app.use(optionalAuth);
+// ── Hosts ────────────────────────────────────────────────────
+// Through the gateway the X-OV-* headers name the tool and its canonical host (a custom domain
+// included); aliases the gateway missed are redirected. Hosts this app does not serve go to the
+// tools index. API paths are never redirected.
+const knowsHost = (h) => !!DOMAIN_MAP[h];
+app.use(hostGuard({ knows: knowsHost }));
+
// ── Attach domain context to every request ───────────────────
// ── Attach domain context to every request ───────────────────
app.use((req, _res, next) => {
app.use((req, _res, next) => {
- req.ctx = resolveContext(req.headers.host);
+ req.ctx = resolveContext(ownHost(req, knowsHost));
next();
next();
});
});
@@ -88,7 +101,7 @@ app.use((req, _res, next) => {
@@ -88,7 +101,7 @@ app.use((req, _res, next) => {
// Health check
// Health check
app.get('/api/health', (_req, res) => {
app.get('/api/health', (_req, res) => {
const stats = retention.getStats();
const stats = retention.getStats();
- res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats });
+ res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats, jobs: jobs.stats() });
});
});
// Domain context (frontend calls this on load to get branding)
// Domain context (frontend calls this on load to get branding)
@@ -122,23 +135,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
@@ -122,23 +135,8 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
return res.status(400).json({ error: `Unknown tool: ${toolId}` });
return res.status(400).json({ error: `Unknown tool: ${toolId}` });
}
}
- // Build options from body params
- const options = {
- format: req.body.format || req.ctx.defaultFormat || undefined,
- quality: req.body.quality || undefined,
- width: req.body.width || undefined,
- height: req.body.height || undefined,
- percentage: req.body.percentage || undefined,
- fit: req.body.fit || undefined,
- background: req.body.background || undefined,
- left: req.body.left || undefined,
- top: req.body.top || undefined,
- aspect: req.body.aspect || undefined,
- inputFormat: req.file.mimetype.split('/')[1],
- };
-
- // Execute the tool
- const result = await tool.handler(req.file.buffer, options);
+ // Execute the tool (the same code the img.process job runs)
+ const result = await processBuffer(req.file.buffer, toolId, buildOptions(req.body, req.ctx, req.file.mimetype));
// Save to retention
// Save to retention
const saved = retention.saveOutput(
const saved = retention.saveOutput(
@@ -149,21 +147,7 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
@@ -149,21 +147,7 @@ app.post('/api/process', burstLimiter, processLimiter, uploadSingle, async (req,
req.file.originalname,
req.file.originalname,
);
);
- res.json({
- success: true,
- tool: toolId,
- download: saved,
- output: {
- mime: result.mime,
- ext: result.ext,
- size: result.buffer.length,
- sizeKB: Math.round(result.buffer.length / 1024 * 10) / 10,
- },
- // Include tool-specific metadata
- ...(result.savings && { savings: result.savings }),
- ...(result.dimensions && { dimensions: result.dimensions }),
- ...(result.crop && { crop: result.crop }),
- });
+ res.json({ success: true, download: saved, ...describe(toolId, result) });
} catch (err) {
} catch (err) {
console.error('[Process] Error:', err.message);
console.error('[Process] Error:', err.message);
res.status(422).json({ error: err.message || 'Image processing failed' });
res.status(422).json({ error: err.message || 'Image processing failed' });
@@ -177,20 +161,7 @@ app.post('/api/process/direct', burstLimiter, processLimiter, uploadSingle, asyn
@@ -177,20 +161,7 @@ app.post('/api/process/direct', burstLimiter, processLimiter, uploadSingle, asyn
const tool = getTool(toolId);
const tool = getTool(toolId);
if (!tool) return res.status(400).json({ error: `Unknown tool: ${toolId}` });
if (!tool) return res.status(400).json({ error: `Unknown tool: ${toolId}` });
- const options = {
- format: req.body.format || req.ctx.defaultFormat || undefined,
- quality: req.body.quality || undefined,
- width: req.body.width || undefined,
- height: req.body.height || undefined,
- percentage: req.body.percentage || undefined,
- fit: req.body.fit || undefined,
- left: req.body.left || undefined,
- top: req.body.top || undefined,
- aspect: req.body.aspect || undefined,
- inputFormat: req.file.mimetype.split('/')[1],
- };
-
- const result = await tool.handler(req.file.buffer, options);
+ const result = await processBuffer(req.file.buffer, toolId, buildOptions(req.body, req.ctx, req.file.mimetype));
const baseName = path.basename(req.file.originalname, path.extname(req.file.originalname));
const baseName = path.basename(req.file.originalname, path.extname(req.file.originalname));
res.set({
res.set({
@@ -205,6 +176,23 @@ app.post('/api/process/direct', burstLimiter, processLimiter, uploadSingle, asyn
@@ -205,6 +176,23 @@ app.post('/api/process/direct', burstLimiter, processLimiter, uploadSingle, asyn
}
}
});
});
+// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
+// The same operation as /api/process, asynchronous and durable: accepted into data/jobs.db,
+// followed over SSE, reattachable by id after a reload or a restart (apps/_shared/jobs).
+const jobs = jobsRuntime.setupJobs({
+ app, service: 'img', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts,
+ getPublicKey: auth.getPublicKey, issuer: auth.ISSUER,
+ define: defineJobs,
+ receive: uploadSingle,
+ limiters: [burstLimiter, processLimiter],
+ defaults(req, input) {
+ const out = { ...input };
+ if (!out.tool) out.tool = req.ctx.defaultOp || 'convert';
+ if (!out.format && req.ctx.defaultFormat) out.format = req.ctx.defaultFormat;
+ return out;
+ },
+});
+
// ── File Download ────────────────────────────────────────────
// ── File Download ────────────────────────────────────────────
app.get('/api/download/:id', (req, res) => {
app.get('/api/download/:id', (req, res) => {
const entry = retention.getFile(req.params.id);
const entry = retention.getFile(req.params.id);
@@ -274,6 +262,7 @@ function shutdown() {
@@ -274,6 +262,7 @@ function shutdown() {
console.log('[Img.OpenVibe] Shutting down...');
console.log('[Img.OpenVibe] Shutting down...');
analytics.destroy();
analytics.destroy();
analyticsDb.close();
analyticsDb.close();
+ jobs.close(); // running jobs stay 'running' in data/jobs.db; the next boot re-queues them
server.close(() => process.exit(0));
server.close(() => process.exit(0));
setTimeout(() => process.exit(1), 5000);
setTimeout(() => process.exit(1), 5000);
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,75 @@
@@ -0,0 +1,75 @@
+'use strict';
+
+// ═══════════════════════════════════════════════════════════════
+// Img.OpenVibe — one image operation, shared by the synchronous endpoints
+// (/api/process, /api/process/direct) and the img.process job (server/jobs.js).
+// ═══════════════════════════════════════════════════════════════
+
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const { getTool, listTools } = require('./tools');
+
+const OPTION_KEYS = ['format', 'quality', 'width', 'height', 'percentage', 'fit', 'background', 'left', 'top', 'aspect'];
+
+/** Tool options from a request body or a job input, with the host's defaults. */
+function buildOptions(src, ctx, inputMime) {
+ const options = {};
+ for (const k of OPTION_KEYS) options[k] = src[k] != null && src[k] !== '' ? src[k] : undefined;
+ if (!options.format && ctx && ctx.defaultFormat) options.format = ctx.defaultFormat;
+ options.inputFormat = String(inputMime || '').split('/')[1];
+ return options;
+}
+
+/** The JSON the synchronous endpoint has always answered with, minus the download block. */
+function describe(toolId, result) {
+ return {
+ tool: toolId,
+ output: { mime: result.mime, ext: result.ext, size: result.buffer.length, sizeKB: Math.round(result.buffer.length / 1024 * 10) / 10 },
+ ...(result.savings && { savings: result.savings }),
+ ...(result.dimensions && { dimensions: result.dimensions }),
+ ...(result.crop && { crop: result.crop }),
+ };
+}
+
+async function processBuffer(buffer, toolId, options) {
+ const tool = getTool(toolId);
+ if (!tool) throw Object.assign(new Error(`Unknown tool: ${toolId}`), { code: 'tools.job.invalid', status: 400 });
+ return tool.handler(buffer, options);
+}
+
+/** Job type img.process: input { tool, format?, quality?, width?, … } + one image file. */
+function defineJobs(system) {
+ const known = new Set(listTools().map(t => t.id));
+ system.define({
+ type: 'img.process',
+ version: 1,
+ minFiles: 1,
+ maxFiles: 1,
+ maxAttempts: 3,
+ onRestart: 'requeue', // a pure function of the stored input: safe to run again
+ timeoutMs: 5 * 60 * 1000,
+ validate(input) {
+ if (!known.has(String(input.tool || 'convert'))) return `Unknown tool: ${input.tool}. Known: ${[...known].join(', ')}`;
+ for (const k of Object.keys(input)) if (k !== 'tool' && !OPTION_KEYS.includes(k)) return `Unknown option: ${k}`;
+ return null;
+ },
+ async run({ input, files, outDir, progress, signal }) {
+ const toolId = String(input.tool || 'convert');
+ const src = files[0];
+ progress(5, 'Reading the image');
+ const buffer = await fsp.readFile(src.path);
+ if (signal.aborted) throw new Error('cancelled');
+ progress(20, 'Processing');
+ const result = await processBuffer(buffer, toolId, buildOptions(input, null, src.mime));
+ progress(90, 'Saving');
+ const base = path.basename(src.name || 'image', path.extname(src.name || '')) || 'openvibeimg-output';
+ const name = `${base}.${result.ext}`;
+ const out = path.join(outDir, `result.${result.ext}`);
+ fs.writeFileSync(out, result.buffer);
+ return { files: [{ path: out, name, mime: result.mime }], data: describe(toolId, result) };
+ },
+ });
+}
+
+module.exports = { buildOptions, describe, processBuffer, defineJobs, OPTION_KEYS };
@@ -8,10 +8,15 @@
@@ -8,10 +8,15 @@
// hosts as duplicates of each other and social shares have no card. This stamps each host's
// hosts as duplicates of each other and social shares have no card. This stamps each host's
// own title, description, canonical, Open Graph / Twitter tags and structured data into the
// own title, description, canonical, Open Graph / Twitter tags and structured data into the
// HTML on the way out. Rendered variants are cached per host; the file itself is read once.
// HTML on the way out. Rendered variants are cached per host; the file itself is read once.
+//
+// Canonical host: through the gateway (X-OV-* headers) the canonical, og:url and JSON-LD use the
+// host it names — a descriptive host or a custom domain; reached directly, the host itself.
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
const fs = require('fs');
const fs = require('fs');
const path = require('path');
const path = require('path');
+const { canonicalHostFor, ownHost } = require('../../_shared/host-role');
+const { DOMAIN_MAP } = require('./domain-map');
const PUBLIC = path.join(__dirname, '..', 'public');
const PUBLIC = path.join(__dirname, '..', 'public');
const HUB_HOST = 'img.openvibe.tools';
const HUB_HOST = 'img.openvibe.tools';
@@ -30,16 +35,18 @@ function baseHtml() {
@@ -30,16 +35,18 @@ function baseHtml() {
/**
/**
* Render index.html for one host with that host's own metadata.
* Render index.html for one host with that host's own metadata.
* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)
* @param {object} ctx domain-map context (brandName, seoTitle, seoDescription, toolId)
- * @param {string} host request hostname, e.g. 'png.openvibe.tools'
+ * @param {string} host the satellite's own hostname for this tool, e.g. 'png.openvibe.tools'
+ * @param {string} [canonicalHost] the host canonical URLs use (the gateway's; defaults to `host`)
*/
*/
-function renderIndex(ctx, host) {
- const key = host;
+function renderIndex(ctx, host, canonicalHost) {
+ const canon = canonicalHost || host;
+ const key = `${host}|${canon}`;
if (_cache.has(key)) return _cache.get(key);
if (_cache.has(key)) return _cache.get(key);
- const url = `https://${host}/`;
+ const url = `https://${canon}/`;
const title = ctx.seoTitle || ctx.brandName || HUB_NAME;
const title = ctx.seoTitle || ctx.brandName || HUB_NAME;
const desc = ctx.seoDescription || '';
const desc = ctx.seoDescription || '';
- const image = `https://${host}/og.png`;
+ const image = `https://${canon}/og.png`;
let html = baseHtml();
let html = baseHtml();
// Replace what the file already sets, so there is never a second competing value.
// Replace what the file already sets, so there is never a second competing value.
@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {
@@ -93,11 +100,13 @@ function renderIndex(ctx, host) {
/** Express handler: the SPA with this host's SEO tags applied. */
/** Express handler: the SPA with this host's SEO tags applied. */
function sendIndex(req, res) {
function sendIndex(req, res) {
- const host = String(req.headers.host || HUB_HOST).split(':')[0].toLowerCase();
+ const own = ownHost(req, h => !!DOMAIN_MAP[h]);
+ const host = DOMAIN_MAP[own] ? own : HUB_HOST;
try {
try {
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Cache-Control', 'no-cache');
- return res.send(renderIndex(req.ctx || {}, host));
+ res.setHeader('Vary', 'X-OV-Canonical-Host');
+ return res.send(renderIndex(req.ctx || {}, host, canonicalHostFor(req, host)));
} catch (err) {
} catch (err) {
console.error('[SEO] render failed:', err.message);
console.error('[SEO] render failed:', err.message);
return res.sendFile(path.join(PUBLIC, 'index.html'));
return res.sendFile(path.join(PUBLIC, 'index.html'));
new file mode 100644
new file mode 100644
@@ -0,0 +1,105 @@
@@ -0,0 +1,105 @@
+'use strict';
+// Img.OpenVibe as a real process: an image job accepted by one process is finished by the next one
+// after a SIGKILL; the synchronous endpoint keeps working; the page honours X-OV-Canonical-Host.
+const assert = require('assert');
+const fs = require('fs');
+const http = require('http');
+const os = require('os');
+const path = require('path');
+const sharp = require('sharp');
+const { startApp } = require('../../_shared/test/spawn');
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+(async () => {
+ const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-img-'));
+ const env = { DATA_DIR: data, UPLOADS_DIR: path.join(data, 'uploads'), OUTPUT_DIR: path.join(data, 'output') };
+ const png = await sharp({ create: { width: 64, height: 48, channels: 3, background: { r: 200, g: 30, b: 90 } } }).png().toBuffer();
+ let app;
+ try {
+ // Accepts but runs nothing, so the job is still queued when the process dies.
+ app = await startApp('img', { ...env, TOOLS_JOBS_CONCURRENCY_IMG: '0' });
+ const fd = new FormData();
+ fd.append('type', 'img.process');
+ fd.append('input', JSON.stringify({ tool: 'convert', format: 'webp', quality: '80' }));
+ fd.append('file', new Blob([png], { type: 'image/png' }), 'red.png');
+ let r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd, headers: { 'Idempotency-Key': 'img-test-0001' } });
+ assert.strictEqual(r.status, 202, await r.clone().text());
+ const cookie = r.headers.get('set-cookie').split(';')[0];
+ const job = await r.json();
+ assert.strictEqual(job.type, 'img.process');
+ assert.strictEqual(job.state, 'queued');
+ await app.kill('SIGKILL');
+
+ app = await startApp('img', env, app.port);
+ let done;
+ for (let i = 0; i < 100; i++) {
+ done = await (await fetch(`${app.base}/api/v1/jobs/${job.id}`, { headers: { cookie } })).json();
+ if (done.state === 'succeeded' || done.state === 'failed') break;
+ await sleep(100);
+ }
+ assert.strictEqual(done.state, 'succeeded', JSON.stringify(done.error));
+ assert.strictEqual(done.result.data.output.ext, 'webp');
+ assert.strictEqual(done.result.files[0].name, 'red.webp');
+ r = await fetch(`${app.base}${done.result.files[0].url}`, { headers: { cookie } });
+ const bytes = Buffer.from(await r.arrayBuffer());
+ assert.strictEqual(bytes.slice(8, 12).toString(), 'WEBP', 'the result is a WebP');
+ assert.strictEqual((await sharp(bytes).metadata()).width, 64);
+ r = await fetch(`${app.base}/api/v1/jobs/${job.id}`);
+ assert.strictEqual(r.status, 404, 'another browser cannot see it');
+
+ // A format host fills in its format: through the gateway, the jpg tool converts to JPG.
+ const fd2 = new FormData();
+ fd2.append('type', 'img.process');
+ fd2.append('file', new Blob([png], { type: 'image/png' }), 'x.png');
+ r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: fd2, headers: { cookie, 'X-OV-Tool': 'jpg' } });
+ const hosted = await r.json();
+ for (let i = 0; i < 100 && hosted.state !== 'succeeded'; i++) { await sleep(100); Object.assign(hosted, await (await fetch(`${app.base}/api/v1/jobs/${hosted.id}`, { headers: { cookie } })).json()); }
+ assert.strictEqual(hosted.result.data.output.ext, 'jpg');
+
+ // Errors on the job API are problem+json, including the upload middleware's.
+ r = await fetch(`${app.base}/api/v1/jobs`, { method: 'POST', body: (() => { const f = new FormData(); f.append('type', 'img.process'); return f; })() });
+ assert.strictEqual(r.status, 400);
+ assert.match(r.headers.get('content-type'), /problem\+json/);
+ assert.strictEqual((await r.json()).code, 'tools.job.invalid');
+
+ // The synchronous endpoint is unchanged.
+ const sync = new FormData();
+ sync.append('tool', 'resize'); sync.append('width', '32');
+ sync.append('file', new Blob([png], { type: 'image/png' }), 'red.png');
+ r = await fetch(`${app.base}/api/process`, { method: 'POST', body: sync });
+ const body = await r.json();
+ assert.strictEqual(body.success, true);
+ assert.match(body.download.downloadUrl, /^\/api\/download\/[a-f0-9]{32}$/);
+ assert.strictEqual(body.dimensions.resized.width, 32);
+
+ // Canonical host: through the gateway the page names the host the gateway says.
+ // (fetch cannot set Host, so pages are requested with http.request)
+ const page = (headers) => new Promise((resolve, reject) => {
+ http.get({ host: '127.0.0.1', port: app.port, path: '/', headers }, (res) => {
+ let text = ''; res.setEncoding('utf8'); res.on('data', c => { text += c; });
+ res.on('end', () => resolve({ status: res.statusCode, headers: { get: (k) => res.headers[k.toLowerCase()] }, text: async () => text }));
+ }).on('error', reject);
+ });
+ let html = await (await page({ Host: 'png.openvibe.tools', 'X-OV-Tool': 'png', 'X-OV-Host-Role': 'short', 'X-OV-Canonical-Host': 'png-converter.example.com', 'X-OV-Short-Host': 'png.openvibe.tools' })).text();
+ assert.ok(html.includes('<link rel="canonical" href="https://png-converter.example.com/">'), 'canonical follows X-OV-Canonical-Host');
+ assert.ok(html.includes('"url":"https://png-converter.example.com/"'), 'so does the JSON-LD');
+ assert.ok(/OpenVibePNG/.test(html), 'and the page is still the PNG tool');
+ html = await (await page({ Host: 'png-converter.example.com', 'X-OV-Tool': 'png', 'X-OV-Host-Role': 'canonical', 'X-OV-Canonical-Host': 'png-converter.example.com' })).text();
+ assert.ok(/OpenVibePNG/.test(html), 'a custom domain gets its tool through X-OV-Tool');
+ html = await (await page({ Host: 'webp.openvibe.tools' })).text();
+ assert.ok(html.includes('<link rel="canonical" href="https://webp.openvibe.tools/">'), 'reached directly, each host is its own canonical');
+ r = await page({ Host: 'pngs.openvibe.tools', 'X-OV-Tool': 'png', 'X-OV-Host-Role': 'alias', 'X-OV-Canonical-Host': 'png-converter.example.com', 'X-OV-Short-Host': 'png.openvibe.tools' });
+ assert.strictEqual(r.status, 301); assert.strictEqual(r.headers.get('location'), 'https://png.openvibe.tools/');
+ r = await page({ Host: 'made-up.example.org' });
+ assert.strictEqual(r.status, 301, 'a host this app does not serve goes to the tools index');
+
+ await app.kill('SIGTERM');
+ console.log('img jobs + canonical host: all checks passed');
+ } catch (err) {
+ if (app) { console.error(app.output()); await app.kill('SIGKILL'); }
+ throw err;
+ } finally {
+ fs.rmSync(data, { recursive: true, force: true });
+ }
+})().catch((err) => { console.error(err); process.exit(1); });
@@ -21,6 +21,13 @@ for app in apps/*/; do
@@ -21,6 +21,13 @@ for app in apps/*/; do
fi
fi
done
done
done
done
+# Apps that run jobs (apps/_shared/jobs, required by relative path) hand it their own better-sqlite3 and
+# openvibe-contracts: check both load under this Node (a native-module ABI mismatch shows up here, not in
+# a crash loop) and that the shared runtime itself loads.
+for app in img audio docs; do
+ (cd "apps/$app" && node -e "const D=require('better-sqlite3'); new D(':memory:').close(); require('openvibe-contracts'); require('../_shared/jobs')") \
+ || { echo "ABORT: apps/$app cannot load the jobs runtime; nothing restarted" >&2; exit 1; }
+done
UNITS=$(systemctl list-unit-files 'openvibe-tools*' --no-legend | awk '{print $1}')
UNITS=$(systemctl list-unit-files 'openvibe-tools*' --no-legend | awk '{print $1}')
sudo systemctl restart $UNITS
sudo systemctl restart $UNITS
sleep 5
sleep 5
@@ -2,8 +2,9 @@
@@ -2,8 +2,9 @@
"name": "openvibe-tools",
"name": "openvibe-tools",
"version": "1.0.0",
"version": "1.0.0",
"private": true,
"private": true,
- "description": "OpenVibe.Tools — the tools suite on openvibe.tools and *.openvibe.tools",
+ "description": "OpenVibe.Tools \u2014 the tools suite on openvibe.tools and *.openvibe.tools",
"scripts": {
"scripts": {
+ "test": "node test/run.js",
"install:all": "npm --prefix apps/gateway install && npm --prefix apps/maps install && npm --prefix apps/food install && npm --prefix apps/img install && npm --prefix apps/yt install && npm --prefix apps/audio install && npm --prefix apps/text install && npm --prefix apps/docs install",
"install:all": "npm --prefix apps/gateway install && npm --prefix apps/maps install && npm --prefix apps/food install && npm --prefix apps/img install && npm --prefix apps/yt install && npm --prefix apps/audio install && npm --prefix apps/text install && npm --prefix apps/docs install",
"start:gateway": "npm --prefix apps/gateway start",
"start:gateway": "npm --prefix apps/gateway start",
"start:maps": "npm --prefix apps/maps start",
"start:maps": "npm --prefix apps/maps start",
@@ -14,5 +15,8 @@
@@ -14,5 +15,8 @@
"start:text": "npm --prefix apps/text start",
"start:text": "npm --prefix apps/text start",
"start:docs": "npm --prefix apps/docs start"
"start:docs": "npm --prefix apps/docs start"
},
},
- "license": "MIT"
+ "license": "MIT",
+ "engines": {
+ "node": ">=22"
+ }
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,53 @@
@@ -0,0 +1,53 @@
+'use strict';
+// npm test — every app's tests (apps/*/test/*.test.js, apps/_shared/test/*.test.js), each in its own
+// process from its app directory, after a syntax check of every server file. Run on Node 22.
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+const APPS = path.join(ROOT, 'apps');
+const only = process.argv.slice(2);
+
+function walk(dir, out = []) {
+ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+ if (e.name === 'node_modules' || e.name === 'data' || e.name.startsWith('.')) continue;
+ const p = path.join(dir, e.name);
+ if (e.isDirectory()) walk(p, out); else if (e.name.endsWith('.js')) out.push(p);
+ }
+ return out;
+}
+
+const apps = fs.readdirSync(APPS).filter(a => fs.statSync(path.join(APPS, a)).isDirectory()).sort();
+
+// 1. Syntax: every server-side file and the shared runtime.
+let bad = 0;
+const sources = apps.flatMap(a => (fs.existsSync(path.join(APPS, a, 'server')) ? walk(path.join(APPS, a, 'server')) : []))
+ .concat(walk(path.join(APPS, '_shared')));
+for (const f of sources) {
+ const r = spawnSync(process.execPath, ['--check', f], { encoding: 'utf8' });
+ if (r.status !== 0) { bad++; console.error(`syntax: ${path.relative(ROOT, f)}\n${r.stderr}`); }
+}
+console.log(`syntax: ${sources.length - bad}/${sources.length} files ok`);
+
+// 2. Tests.
+const tests = apps.flatMap(a => {
+ const dir = path.join(APPS, a, 'test');
+ return fs.existsSync(dir) ? fs.readdirSync(dir).filter(f => f.endsWith('.test.js')).sort().map(f => ({ app: a, file: path.join(dir, f) })) : [];
+}).filter(t => !only.length || only.some(o => t.file.includes(o)));
+
+const failed = [];
+for (const t of tests) {
+ const name = path.relative(ROOT, t.file);
+ const t0 = Date.now();
+ const r = spawnSync(process.execPath, [t.file], { cwd: path.join(APPS, t.app), encoding: 'utf8', timeout: 180_000, env: { ...process.env, NODE_ENV: 'test' } });
+ const ms = Date.now() - t0;
+ if (r.status === 0) {
+ console.log(`ok ${name} (${ms} ms)${r.stdout.trim() ? ` — ${r.stdout.trim().split('\n').pop()}` : ''}`);
+ } else {
+ failed.push(name);
+ console.error(`FAIL ${name} (${ms} ms)\n${r.stdout}${r.stderr}${r.error ? r.error.message : ''}`);
+ }
+}
+console.log(`\n${tests.length - failed.length}/${tests.length} test files passed${failed.length ? `; failed: ${failed.join(', ')}` : ''}`);
+process.exit(failed.length || bad ? 1 : 0);