openvibers/openvibe.tools Commits
Tools job store on PostgreSQL (plan T8 phases 2-3; not deployed yet): apps/_shared/jobs on openvibe-sdk/db with async ambient transactions (finish, the usage rollup and the outbox in one), every app sharing the one tools database, ready.postgres, better-sqlite3 gone with a guard test; job reads scoped by app (a shared database let one app read another's job by id); a cancel that lands while a job prepares ends it. npm test and test:pg 58/58.
Changes
@@@ -3,9 +3,9 @@// The Tools guard (apps/_shared/guard): anti-abuse for the gateway and every satellite, driven by// the tools' descriptors (tools.tool@1: quotaClass, cost, auth, files.accept, limits) and the host-wide// numbers in ./limits.js. Like the rest of apps/_shared it has no dependencies: the app passes its--// better-sqlite3 class, openvibe-contracts (when it has it) and its descriptor specs.++// openvibe-sdk/db handle, its Valkey and openvibe-contracts (when it has it) and its descriptor specs.//--// const guard = createGuard({ app: 'img', dataDir, Database, contracts, specs, networkUrl, … });++// const guard = createGuard({ app: 'img', dataDir, db, valkey, contracts, specs, networkUrl, … });// app.set('trust proxy', TRUST_PROXY); // ./ip.js: one loopback hop// app.use(guard.identify); // req.user (Network sign-in, aud checked)// app.use('/api/', legacyLimiter, guard.apiQuota); // every /api/ request (tools-api)@@@ -82,8 +82,9 @@ function discardUploads(req) {/*** @param {object} o* @param {string} o.app 'gateway' | 'img' | … (logs, metrics)-- * @param {string} [o.dataDir] guard.db goes here (with o.Database)-- * @param {Function} [o.Database] require('better-sqlite3'); without it everything stays in memory++ * @param {string} [o.dataDir] where the app keeps its files (job inputs/results)++ * @param {object} [o.db] openvibe-sdk/db handle (guard_abuse); without it everything stays in memory++ * @param {object} [o.valkey] the shared Valkey (day counters, salt, buckets)* @param {object} [o.contracts] openvibe-contracts, when the app has it* @param {object[]} [o.specs] the app's descriptor specs (server/descriptors.js SPECS)* @param {object} [o.keys] { get(), ensure() } — or networkUrl/networkInternalUrl/publicKeyFiles@@@ -7,12 +7,11 @@// succeeded → tools.job.succeeded priority important actor: service:tools// failed (tool, timeout, boot) → tools.job.failed priority important actor: service:tools//--// Each event is written to the satellite's own `event_outbox` table (openvibe-sdk createOutbox) in--// the SQLite transaction that records the transition, so it exists if and only if the transition--// committed; the relay posts it to OpenVibe.Events afterwards (at least once; Events dedupes on--// event_id). A cancelled job, a progress tick and a requeue after a restart are not announced (a--// requeued job is announced again when it starts). Sandbox jobs (developer-app sandbox tokens) are--// never announced.++// Each event is written to the one tools database's `event_outbox` table (openvibe-sdk createPgOutbox) in++// the SAME transaction that records the transition, so it exists if and only if the transition committed;++// the relay posts it to OpenVibe.Events afterwards (at least once; Events dedupes on event_id). A cancelled++// job, a progress tick and a requeue after a restart are not announced (a requeued job is announced again++// when it starts). Sandbox jobs (developer-app sandbox tokens) are never announced.//// Payloads (openvibe-contracts tools.job.*@1, validated before they are enqueued) carry ids, type,// owner, state, times, where result files are and the error; never the job's input, file names,@@@ -29,7 +28,7 @@ const SERVICE_ACTOR = Object.freeze({ type: 'service', id: SOURCE });const OWNER_TYPES = { user: 'user', svc: 'service', app: 'app', mod: 'mod' };const PRIORITY = { 'tools.job.created': 'low', 'tools.job.started': 'low', 'tools.job.succeeded': 'important', 'tools.job.failed': 'important' };--const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());++const iso = (ms) => (ms == null ? null : new Date(Number(ms)).toISOString());const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };/** 'user:usr_…' | 'svc:<slug>' | 'app:app_…' | 'mod:mod_…' → SubjectRef; anything else (a session) → null. */@@@ -60,11 +59,11 @@ function payloadFor(eventType, row, { contracts, service, referenced = false })job_id: row.id,service,type: row.type,-- type_version: row.type_version,++ type_version: Number(row.type_version),state: row.state,owner: ownerRef(contracts, row.owner),-- attempts: row.attempts,-- max_attempts: row.max_attempts,++ attempts: Number(row.attempts),++ max_attempts: Number(row.max_attempts),created_at: iso(row.created_at),retry_of: row.retry_of || null,};@@@ -103,23 +102,24 @@ function payloadFor(eventType, row, { contracts, service, referenced = false })* @param {object} o* @param {object} o.contracts require('openvibe-contracts') (v0.30.0+ knows tools.job.*@1)* @param {string} o.service the satellite ('img' | 'audio' | 'docs' | …) → payload.service-- * @param {object|null} o.outbox openvibe-sdk createOutbox(db, …) on the jobs database, or null-- * @param {(id) => number} [o.referenceCount]++ * @param {object|null} o.outbox openvibe-sdk createPgOutbox(db, …) on the one tools database, or null++ * @param {(id) => Promise<number>} [o.referenceCount]* @param {object} [o.log]*/--function createJobEvents({ contracts, service, outbox = null, referenceCount = () => 0, log = console }) {++function createJobEvents({ contracts, service, outbox = null, referenceCount = async () => 0, log = console }) {const stats = { queued: 0, invalid: 0, lastInvalid: null };/**-- * Enqueue `eventType` for this row (just read back inside the caller's transaction). A payload that-- * fails its contract is logged and skipped, never thrown: a bug here must not stop jobs. An outbox-- * write error does throw, so the caller's transaction rolls back with it.++ * Enqueue `eventType` for this row inside the caller's transaction `t` (the row was just read back).++ * A payload that fails its contract is logged and skipped, never thrown: a bug here must not stop++ * jobs. An outbox write error does throw, so the caller's transaction rolls back with it.*/-- function announce(eventType, row) {++ async function announce(t, eventType, row) {if (!outbox || !row || row.env === 'sandbox') return null;let payload;try {-- payload = payloadFor(eventType, row, { contracts, service, referenced: eventType !== 'tools.job.created' && eventType !== 'tools.job.started' && referenceCount(row.id) > 0 });++ const referenced = eventType !== 'tools.job.created' && eventType !== 'tools.job.started' && (await referenceCount(row.id)) > 0;++ payload = payloadFor(eventType, row, { contracts, service, referenced });const v = contracts.validate(`${eventType}@1`, payload);if (!v.valid) throw new Error(v.errors.map(e => `${e.path} ${e.message}`).join('; '));} catch (err) {@@@ -128,7 +128,7 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (log.error(`[Jobs] ${eventType} for ${row.id} not announced (payload does not match the contract): ${err.message}`);return null;}-- const env = outbox.enqueue({++ const env = await outbox.enqueue(t, {event_type: eventType,actor: eventType === 'tools.job.created' && payload.owner ? payload.owner : SERVICE_ACTOR,subject: { type: 'job', id: row.id },@@@ -143,14 +143,14 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (return {enabled: !!outbox,-- created: (row) => announce('tools.job.created', row),-- started: (row) => announce('tools.job.started', row),++ created: (t, row) => announce(t, 'tools.job.created', row),++ started: (t, row) => announce(t, 'tools.job.started', row),/** succeeded → tools.job.succeeded, failed → tools.job.failed; cancelled is not announced. */-- finished: (row) => (row && row.state === 'succeeded' ? announce('tools.job.succeeded', row)-- : row && row.state === 'failed' ? announce('tools.job.failed', row) : null),-- status() {++ finished: (t, row) => (row && row.state === 'succeeded' ? announce(t, 'tools.job.succeeded', row)++ : row && row.state === 'failed' ? announce(t, 'tools.job.failed', row) : null),++ async status() {if (!outbox) return { enabled: false };-- return { enabled: true, pending: outbox.pending(), rejected: outbox.rejected(), queued_since_boot: stats.queued, invalid: stats.invalid, last_invalid: stats.lastInvalid };++ return { enabled: true, pending: await outbox.pending(), rejected: await outbox.rejected(), queued_since_boot: stats.queued, invalid: stats.invalid, last_invalid: stats.lastInvalid };},};}@@@ -165,8 +165,11 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (* OV_NETWORK_INTERNAL_URL token endpoint host (default http://127.0.0.1:4000)* EVENTS_RELAY_INTERVAL_MS relay poll (default 2000)*++ * The outbox table is created at boot (createPgOutbox.ensureSchema); outbox.ready is that promise, for a++ * test (or a caller) that must know the table exists.++ ** @param {object} o-- * @param {object} o.db the jobs database (better-sqlite3) — the outbox table lives beside tool_jobs++ * @param {object} o.db openvibe-sdk/db handle — the outbox table lives beside tool_jobs* @param {object} [o.sdk] require('openvibe-sdk')* @param {object} [o.env]* @param {Function} [o.fetch]@@@ -190,7 +193,7 @@ function outboxFromEnv({ db, sdk, env = process.env, fetch: fetchImpl, log = conconst client = sdk.createClient({ baseUrls: { events: url }, tokenProvider: tokens, retries: 0, ...(fetchImpl ? { fetch: fetchImpl } : {}) });let lastError = null;const interval = parseInt(env.EVENTS_RELAY_INTERVAL_MS, 10);-- const outbox = sdk.events.createOutbox(db, {++ const outbox = sdk.events.createPgOutbox(db, {events: sdk.events.createEventsClient(client, { source: SOURCE }),intervalMs: Number.isFinite(interval) && interval >= 100 ? interval : 2000,onError: (err) => {@@@ -199,8 +202,8 @@ function outboxFromEnv({ db, sdk, env = process.env, fetch: fetchImpl, log = conlastError = msg;},});-- outbox.ensureSchema();outbox.url = url;++ outbox.ready = outbox.ensureSchema();return { outbox, reason: null };}@@@ -95,11 +95,11 @@ function mountJobRoutes(app, o) {return w || { owner: null };}/** The job, if it exists AND belongs to the caller; otherwise the same 404 either way. */-- function load(req, res, action) {++ async function load(req, res, action) {const w = who(req, res, { action });if (!w) return null;const id = String(req.params.id || '');-- const row = ID_RE.test(id) ? system.get(id) : null;++ const row = ID_RE.test(id) ? await system.get(id) : null;if (!row || !w.owner || row.owner !== w.owner) { send(res, 404, 'tools.job.not_found', 'No such job'); return null; }return row;}@@@ -109,11 +109,11 @@ function mountJobRoutes(app, o) {// Which satellite holds a job? Only for a caller on this host with no proxy in between (the// gateway's facade); through nginx or the gateway's own proxy it is not found. Says nothing but// whether the id exists here.-- app.get('/api/internal/jobs/:id', (req, res) => {++ app.get('/api/internal/jobs/:id', async (req, res) => {res.set('Cache-Control', 'no-store');const direct = isLoopback(req.socket && req.socket.remoteAddress) && !req.headers['x-forwarded-for'] && !req.headers['x-real-ip'];const id = String(req.params.id || '');-- if (!direct || !ID_RE.test(id) || !system.get(id)) return send(res, 404, 'tools.job.not_found', 'No such job');++ if (!direct || !ID_RE.test(id) || !(await system.get(id))) return send(res, 404, 'tools.job.not_found', 'No such job');return res.json({ id, service: system.service || null });});@@@ -136,8 +136,8 @@ function mountJobRoutes(app, o) {};// Over the store's bounds (queued jobs, disk): answer before an upload is accepted.-- const notBusy = (req, res, next) => {-- const busy = system.busy ? system.busy() : null;++ const notBusy = async (req, res, next) => {++ const busy = system.busy ? await system.busy() : null;if (!busy) return next();if (o.onBusy) return o.onBusy(req, res, busy) ? next() : undefined;res.set('Retry-After', String(busy.retryAfter || 30));@@@ -160,7 +160,7 @@ function mountJobRoutes(app, o) {const files = list.map(f => ({ path: f.path, buffer: f.path ? undefined : f.buffer, name: f.originalname, mime: f.mimetype, size: f.size }));const key = req.headers['idempotency-key'] != null ? String(req.headers['idempotency-key']) : (b.idempotency_key != null ? String(b.idempotency_key) : null);// Sessions from one address share a bound on unfinished jobs: a new cookie is no new allowance.-- const full = w.caller && system.addressFull ? system.addressFull(w.owner, w.caller.ipKey) : null;++ const full = w.caller && system.addressFull ? await system.addressFull(w.owner, w.caller.ipKey) : null;if (full) {const goOn = o.onAddressFull ? o.onAddressFull(req, res, full) : (send(res, 429, 'tools.job.too_many_active', `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish`), false);if (!goOn) { for (const f of files) if (f.path) fs.unlink(f.path, () => {}); return; }@@@ -177,7 +177,7 @@ function mountJobRoutes(app, o) {const { job, replayed } = await system.submit({ owner: w.owner, type: String(b.type || ''), input, files, idempotencyKey: key, ttlMs: ttlMs(req, w), env: w.env || 'production', ipKey: w.caller ? w.caller.ipKey : null, project: w.kind === 'principal' && w.claims && w.claims.actor_type === 'app' ? w.claims.project_id : null, traceId: req.ov && req.ov.traceId });res.set('Location', `/api/v1/jobs/${job.id}`);if (replayed) res.set('Idempotent-Replayed', 'true');-- return res.status(replayed ? 200 : 202).json(system.view(job));++ return res.status(replayed ? 200 : 202).json(await system.view(job));} catch (err) {if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);console.error('[Jobs] submit failed:', err.message);@@@ -185,31 +185,31 @@ function mountJobRoutes(app, o) {}});-- app.get('/api/v1/jobs/:id', (req, res) => {++ app.get('/api/v1/jobs/:id', async (req, res) => {noStore(res);-- const row = load(req, res, 'read');-- if (row) res.json(system.view(row));++ const row = await load(req, res, 'read');++ if (row) res.json(await system.view(row));});-- app.delete('/api/v1/jobs/:id', origin, (req, res) => {++ app.delete('/api/v1/jobs/:id', origin, async (req, res) => {noStore(res);-- const row = load(req, res, 'cancel');++ const row = await load(req, res, 'cancel');if (!row) return;if (row.state === 'succeeded' || row.state === 'failed') return send(res, 409, 'tools.job.already_finished', `The job already ${row.state}`, { state: row.state });-- const { job } = system.cancel(row.id);-- return res.status(job.state === 'cancelled' ? 200 : 202).json(system.view(job));++ const { job } = await system.cancel(row.id);++ return res.status(job.state === 'cancelled' ? 200 : 202).json(await system.view(job));});// Retry and references are writes: a principal needs tools.job.create, like a submit.-- app.post('/api/v1/jobs/:id/retry', origin, ...(o.limiters || []), notBusy, (req, res) => {++ app.post('/api/v1/jobs/:id/retry', origin, ...(o.limiters || []), notBusy, async (req, res) => {noStore(res);-- const row = load(req, res, 'create');++ const row = await load(req, res, 'create');if (!row) return;try {-- const { job, replayed } = system.retry(row.id);++ const { job, replayed } = await system.retry(row.id);res.set('Location', `/api/v1/jobs/${job.id}`);if (replayed) res.set('Idempotent-Replayed', 'true');-- return res.status(replayed ? 200 : 202).json(system.view(job));++ return res.status(replayed ? 200 : 202).json(await system.view(job));} catch (err) {if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);console.error('[Jobs] retry failed:', err.message);@@@ -218,13 +218,13 @@ function mountJobRoutes(app, o) {});function referenceRoute(fn) {-- return (req, res) => {++ return async (req, res) => {noStore(res);-- const row = load(req, res, 'create');++ const row = await load(req, res, 'create');if (!row) return;try {-- const r = fn(row.id, String(req.params.ref || ''));-- return res.status(r.created ? 201 : 200).json(system.view(r.job));++ const r = await fn(row.id, String(req.params.ref || ''));++ return res.status(r.created ? 201 : 200).json(await system.view(r.job));} catch (err) {if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);console.error('[Jobs] reference failed:', err.message);@@@ -235,12 +235,12 @@ function mountJobRoutes(app, o) {app.put('/api/v1/jobs/:id/references/:ref', origin, referenceRoute((id, ref) => system.reference(id, ref)));app.delete('/api/v1/jobs/:id/references/:ref', origin, referenceRoute((id, ref) => system.unreference(id, ref)));-- app.get('/api/v1/jobs/:id/events', (req, res) => {-- const row = load(req, res, 'read');++ app.get('/api/v1/jobs/:id/events', async (req, res) => {++ const row = await load(req, res, 'read');if (!row) return;const raw = req.headers['last-event-id'] != null ? req.headers['last-event-id'] : req.query.last_event_id;const after = Math.max(0, parseInt(raw, 10) || 0);-- const pending = system.eventsAfter(row.id, after);++ const pending = await system.eventsAfter(row.id, after);// Finished and nothing new since the client's last event: 204 tells EventSource to stop reconnecting.if (TERMINAL.has(row.state) && !pending.length) return res.status(204).set('Cache-Control', 'no-store').end();@@@ -264,7 +264,7 @@ function mountJobRoutes(app, o) {};// Subscribe before replaying so nothing emitted in between is lost; the seq check drops duplicates.const unsubscribe = system.subscribe(row.id, (e) => (replaying ? queue.push(e) : write(e)));-- for (const e of system.eventsAfter(row.id, after)) write(e);++ for (const e of await system.eventsAfter(row.id, after)) write(e);replaying = false;queue.forEach(write);const keepAlive = setInterval(() => { if (!closed) res.write(': keep-alive\n\n'); }, 15000);@@@ -273,7 +273,7 @@ function mountJobRoutes(app, o) {});app.get('/api/v1/jobs/:id/files/:n', async (req, res) => {-- const row = load(req, res, 'read');++ const row = await load(req, res, 'read');if (!row) return;if (row.state !== 'succeeded') return send(res, 409, 'tools.job.not_ready', `The job is ${row.state}`);const f = system.resultFile(row, parseInt(req.params.n, 10));@@@ -1,10 +1,12 @@'use strict';// ═══════════════════════════════════════════════════════════════// Tools job runtime (roadmap Wave 11). Shared by the satellites the way apps/_shared always is:--// required by relative path, no dependencies of its own — each app passes its better-sqlite3,--// openvibe-contracts and openvibe-sdk. See system.js (lifecycle), http.js (routes), media.js (results--// in Media), events.js (tools.job.* to OpenVibe.Events), usage.js (tools.usage.recorded, a developer--// project's jobs per hour), client.js (browser helper).++// required by relative path, no dependencies of its own — each app passes the one openvibe-sdk/db++// handle (the shared `tools` database, plan T8), openvibe-contracts and openvibe-sdk. See system.js++// (lifecycle), http.js (routes), media.js (results in Media), events.js (tools.job.* to OpenVibe.Events),++// usage.js (tools.usage.recorded, a developer project's jobs per hour), client.js (browser helper).++//++// The job store is part of every app's one `tools` database: no per-satellite jobs.db.//// Environment (one /etc/openvibe/tools.env for every unit):// TOOLS_JOBS_CONCURRENCY jobs running at once per satellite (default 2)@@@ -65,14 +67,14 @@ function mediaOrigin(env = process.env) {}/**-- * One call per satellite: open <dataDir>/jobs.db, build the system, let the app define its job-- * types, mount the routes and start (boot recovery happens here).++ * One call per satellite: build the system on the app's shared `tools` database, let the app define its++ * job types, mount the routes and start (boot recovery happens here).** @param {object} o* @param {object} o.app Express app* @param {string} o.service 'img' | 'audio' | 'docs'* @param {string} o.dataDir-- * @param {Function} o.Database require('better-sqlite3')++ * @param {object} o.db the app's shared openvibe-sdk/db handle (one `tools` database)* @param {object} o.contracts require('openvibe-contracts')* @param {object} [o.sdk] require('openvibe-sdk') — the outbox relay to OpenVibe.Events* @param {() => string|null} o.getPublicKey@@@ -89,12 +91,14 @@ function mediaOrigin(env = process.env) {*/function setupJobs(o) {fs.mkdirSync(o.dataDir, { recursive: true });-- const db = new o.Database(path.join(o.dataDir, 'jobs.db'));++ const db = o.db;++ const log = o.log || console;const results = mediaFromEnv(o.contracts);if (results.reason && String(process.env.TOOLS_JOB_RESULTS || '').toLowerCase() === 'media') console.warn(`[Jobs] ${o.service}: results stay local — ${results.reason}`);-- // tools.job.* → OpenVibe.Events through an outbox table in this same jobs.db (inert without EVENTS_URL).-- const events = outboxFromEnv({ db, sdk: o.sdk });++ // tools.job.* → OpenVibe.Events through the `event_outbox` table in this same database (inert without EVENTS_URL).++ const events = outboxFromEnv({ db, sdk: o.sdk, log });if (events.reason && process.env.EVENTS_URL) console.warn(`[Jobs] ${o.service}: job events are not published — ${events.reason}`);++ if (events.outbox) events.outbox.ready.catch((err) => log.error(`[Jobs] ${o.service}: could not create the event outbox:`, err.message));const system = createJobSystem({db, contracts: o.contracts, service: o.service, dataDir: o.dataDir,concurrency: envInt(`TOOLS_JOBS_CONCURRENCY_${o.service.toUpperCase()}`, envInt('TOOLS_JOBS_CONCURRENCY', 2)),@@@ -104,6 +108,7 @@ function setupJobs(o) {diskBudgetBytes: envInt('TOOLS_DISK_BUDGET_MB', 8192) * 1024 * 1024,media: results.media,outbox: events.outbox,++ log,});o.define(system);const g = o.guard || null;@@@ -117,19 +122,19 @@ function setupJobs(o) {onAddressFull: (req, res, full) => !g.refuse(req, res, { status: 429, code: 'tools.job.too_many_active', reason: 'jobs.address', tool: o.jobTool ? o.jobTool(req, null, {}) : null, retryAfter: 30, detail: `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish.`, extra: { scope: 'address' } }),}),});-- system.start();++ system.start().catch((err) => log.error(`[Jobs] ${o.service}: boot recovery failed:`, err.message));let pruneOutbox = null;if (events.outbox) {events.outbox.start();-- pruneOutbox = setInterval(() => { try { events.outbox.prune(); } catch { /* next time */ } }, 6 * 60 * 60 * 1000);++ pruneOutbox = setInterval(() => { events.outbox.prune().catch(() => { /* next time */ }); }, 6 * 60 * 60 * 1000);if (pruneOutbox.unref) pruneOutbox.unref();-- console.log(`[Jobs] ${o.service}: job events → ${events.outbox.url} (${events.outbox.pending()} pending)`);++ console.log(`[Jobs] ${o.service}: job events → ${events.outbox.url}`);}system.db = db;++ // The `tools` database is shared with the rest of the app and closed there; this only stops the worker.system.close = () => {system.stop();-- if (events.outbox) { events.outbox.stop(); clearInterval(pruneOutbox); }-- try { db.close(); } catch { /* already closed */ }++ if (events.outbox) { events.outbox.stop().catch(() => {}); clearInterval(pruneOutbox); }};return system;}@@@ -1,173 +1,118 @@'use strict';// ═══════════════════════════════════════════════════════════════--// Job store — the satellite's own SQLite (better-sqlite3 handle passed in).++// Job store — the one `tools` PostgreSQL database (plan T8, decision 3), through openvibe-sdk/db.//--// Three tables, created on first use (idempotent):++// Three tables, all created by migrations/0001_tools.sql (not here):// tool_jobs one row per job; the durable truth a restart recovers from--// tool_job_events the ordered event log per job; its AUTOINCREMENT seq is the SSE event id,++// tool_job_events the ordered event log per job; its per-job identity seq is the SSE event id,// so a reconnect with Last-Event-ID replays exactly what was missed// tool_job_references who still points at a job's result (a paste, a project, …); a referenced// job never expires, so its files and Media objects are not pruned++//++// Eight apps share the one database, so every row carries `app` and every read that could see another++// app's rows filters on it. All statements are async (openvibe-sdk/db) and join an ambient transaction++// when one is open, so a store call inside db.tx(fn) commits with it.// ═══════════════════════════════════════════════════════════════const TERMINAL = ['succeeded', 'failed', 'cancelled'];const STATES = ['queued', 'running', ...TERMINAL];--const SCHEMA = `--CREATE TABLE IF NOT EXISTS tool_jobs (-- id TEXT PRIMARY KEY,-- type TEXT NOT NULL,-- type_version INTEGER NOT NULL DEFAULT 1,-- owner TEXT NOT NULL,-- state TEXT NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled')),-- progress REAL,-- progress_message TEXT,-- input_json TEXT NOT NULL DEFAULT '{}',-- files_json TEXT NOT NULL DEFAULT '[]',-- result_json TEXT,-- error_json TEXT,-- retryable INTEGER NOT NULL DEFAULT 0,-- idempotency_key TEXT,-- request_hash TEXT,-- attempts INTEGER NOT NULL DEFAULT 0,-- max_attempts INTEGER NOT NULL DEFAULT 1,-- cancel_requested INTEGER NOT NULL DEFAULT 0,-- ttl_ms INTEGER NOT NULL DEFAULT 3600000,-- created_at INTEGER NOT NULL,-- updated_at INTEGER NOT NULL,-- started_at INTEGER,-- finished_at INTEGER,-- expires_at INTEGER,-- env TEXT NOT NULL DEFAULT 'production' CHECK (env IN ('production','sandbox'))--);--CREATE UNIQUE INDEX IF NOT EXISTS tool_jobs_idem ON tool_jobs(owner, idempotency_key) WHERE idempotency_key IS NOT NULL;--CREATE INDEX IF NOT EXISTS tool_jobs_state ON tool_jobs(state, id);--CREATE INDEX IF NOT EXISTS tool_jobs_expiry ON tool_jobs(expires_at) WHERE expires_at IS NOT NULL;--CREATE INDEX IF NOT EXISTS tool_jobs_owner ON tool_jobs(owner, state);--CREATE TABLE IF NOT EXISTS tool_job_events (-- seq INTEGER PRIMARY KEY AUTOINCREMENT,-- job_id TEXT NOT NULL,-- event TEXT NOT NULL,-- data TEXT NOT NULL,-- at INTEGER NOT NULL--);--CREATE INDEX IF NOT EXISTS tool_job_events_job ON tool_job_events(job_id, seq);--CREATE TABLE IF NOT EXISTS tool_job_references (-- job_id TEXT NOT NULL,-- ref TEXT NOT NULL,-- created_at INTEGER NOT NULL,-- PRIMARY KEY (job_id, ref)--);--`;----// Columns added after the first release, each added when missing (idempotent).--const ADDED_COLUMNS = [-- ['env', "TEXT NOT NULL DEFAULT 'production' CHECK (env IN ('production','sandbox'))"], // developer-app sandboxes-- ['retry_of', 'TEXT'], // the failed job this one retries-- ['retried_by', 'TEXT'], // set on a failed job once it has been retried: the retry's id-- ['ip_key', 'TEXT'], // a browser session's job: the guard's hashed address key (never the address), so-- // unfinished jobs are also bounded per address (a new cookie is no new allowance)-- ['tool', 'TEXT'], // the tool (tools.tool@1 id) whose run created the job (POST /api/v1/tools/:id/run)-- ['project_id', 'TEXT'], // a developer app's job: its project (prj_…), so its results go to Media under the-- // project's namespace, metered per project (WS-L task 5)-- ['trace_id', 'TEXT'], // the trace id of the request that submitted the job, sampled with a developer-- // project's failures in its usage rollups (./usage.js, WS-N task 4)--];--const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };--function createStore(db) {-- db.pragma('journal_mode = WAL');-- db.pragma('busy_timeout = 5000');-- db.exec(SCHEMA);-- const have = new Set(db.prepare('PRAGMA table_info(tool_jobs)').all().map((c) => c.name));-- for (const [name, decl] of ADDED_COLUMNS) if (!have.has(name)) db.exec(`ALTER TABLE tool_jobs ADD COLUMN ${name} ${decl}`);-- db.exec("CREATE INDEX IF NOT EXISTS tool_jobs_ip ON tool_jobs(ip_key, state) WHERE ip_key IS NOT NULL");--++/**++ * @param {object} db openvibe-sdk/db handle (or the gated one apps/_shared/db.js returns)++ * @param {object} o++ * @param {string} o.app the owning app ('img' | 'audio' | 'docs' | …): every row this store writes or reads++ */++function createStore(db, { app } = {}) {++ if (!db || typeof db.prepare !== 'function') throw new TypeError('createStore needs an openvibe-sdk/db handle');++ if (!app) throw new TypeError('createStore needs app');const q = {-- insert: db.prepare(`INSERT INTO tool_jobs (id, type, type_version, owner, state, input_json, files_json, idempotency_key, request_hash, max_attempts, ttl_ms, created_at, updated_at, env, retry_of, ip_key, tool, project_id, trace_id)-- VALUES (@id, @type, @type_version, @owner, 'queued', @input_json, @files_json, @idempotency_key, @request_hash, @max_attempts, @ttl_ms, @now, @now, @env, @retry_of, @ip_key, @tool, @project_id, @trace_id)`),++ insert: db.prepare(`INSERT INTO tool_jobs (id, app, type, type_version, owner, state, input_json, files_json, idempotency_key, request_hash, max_attempts, ttl_ms, created_at, updated_at, env, retry_of, ip_key, tool, project_id, trace_id)++ VALUES (@id, @app, @type, @type_version, @owner, 'queued', @input_json, @files_json, @idempotency_key, @request_hash, @max_attempts, @ttl_ms, @now, @now, @env, @retry_of, @ip_key, @tool, @project_id, @trace_id)`),markRetried: db.prepare("UPDATE tool_jobs SET retried_by = @next, updated_at = @now WHERE id = @id AND state = 'failed' AND retried_by IS NULL"),-- get: db.prepare('SELECT * FROM tool_jobs WHERE id = ?'),-- byIdem: db.prepare('SELECT * FROM tool_jobs WHERE owner = ? AND idempotency_key = ?'),++ get: db.prepare('SELECT * FROM tool_jobs WHERE id = ? AND app = ?'),++ byIdem: db.prepare('SELECT * FROM tool_jobs WHERE app = @app AND owner = @owner AND idempotency_key = @idempotency_key'),// A result file stored in Media, by its media_id (a run's { media_id } reference); the owner's only.-- byResultMedia: db.prepare("SELECT * FROM tool_jobs WHERE owner = ? AND state = 'succeeded' AND result_json LIKE ? ORDER BY id DESC LIMIT 5"),-- nextQueued: db.prepare("SELECT * FROM tool_jobs WHERE state = 'queued' ORDER BY id LIMIT ?"),++ byResultMedia: db.prepare("SELECT * FROM tool_jobs WHERE app = @app AND owner = @owner AND state = 'succeeded' AND result_json ILIKE @like ORDER BY id DESC LIMIT 5"),++ nextQueued: db.prepare("SELECT * FROM tool_jobs WHERE app = @app AND state = 'queued' ORDER BY id LIMIT @limit"),claim: db.prepare("UPDATE tool_jobs SET state = 'running', attempts = attempts + 1, started_at = @now, updated_at = @now, progress = NULL, progress_message = NULL WHERE id = @id AND state = 'queued'"),-- running: db.prepare("SELECT * FROM tool_jobs WHERE state = 'running'"),-- activeForOwner: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE owner = ? AND state IN ('queued','running')"),-- activeForIp: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE ip_key = ? AND state IN ('queued','running')"),++ running: db.prepare("SELECT * FROM tool_jobs WHERE app = @app AND state = 'running'"),++ activeForOwner: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE app = @app AND owner = @owner AND state IN ('queued','running')"),++ activeForIp: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE app = @app AND ip_key = @ip_key AND state IN ('queued','running')"),progress: db.prepare('UPDATE tool_jobs SET progress = @progress, progress_message = @message, updated_at = @now WHERE id = @id AND state = \'running\''),requestCancel: db.prepare("UPDATE tool_jobs SET cancel_requested = 1, updated_at = @now WHERE id = @id AND state IN ('queued','running')"),requeue: db.prepare("UPDATE tool_jobs SET state = 'queued', started_at = NULL, progress = NULL, progress_message = NULL, updated_at = @now WHERE id = @id AND state = 'running'"),finish: db.prepare(`UPDATE tool_jobs SET state = @state, result_json = @result_json, error_json = @error_json, retryable = @retryable,progress = CASE WHEN @state = 'succeeded' THEN 100 ELSE progress END, progress_message = NULL,-- finished_at = @now, updated_at = @now, expires_at = @now + ttl_ms WHERE id = @id AND state IN ('queued','running')`),++ finished_at = @now, updated_at = @now, expires_at = @now + ttl_ms::bigint WHERE id = @id AND state IN ('queued','running')`),setFiles: db.prepare('UPDATE tool_jobs SET files_json = @files_json, updated_at = @now WHERE id = @id'),-- event: db.prepare('INSERT INTO tool_job_events (job_id, event, data, at) VALUES (?, ?, ?, ?)'),++ event: db.prepare('INSERT INTO tool_job_events (job_id, event, data, at) VALUES (@job_id, @event, @data, @at) RETURNING seq'),eventsAfter: db.prepare('SELECT seq, event, data, at FROM tool_job_events WHERE job_id = ? AND seq > ? ORDER BY seq'),lastSeq: db.prepare('SELECT MAX(seq) AS seq FROM tool_job_events WHERE job_id = ?'),// A job something still references is never expired (see tool_job_references).-- expired: db.prepare(`SELECT * FROM tool_jobs j WHERE expires_at IS NOT NULL AND expires_at <= ?++ expired: db.prepare(`SELECT * FROM tool_jobs j WHERE app = @app AND expires_at IS NOT NULL AND expires_at <= @nowAND NOT EXISTS (SELECT 1 FROM tool_job_references r WHERE r.job_id = j.id) LIMIT 500`),deferExpiry: db.prepare('UPDATE tool_jobs SET expires_at = @until, updated_at = @now WHERE id = @id'),del: db.prepare('DELETE FROM tool_jobs WHERE id = ?'),delEvents: db.prepare('DELETE FROM tool_job_events WHERE job_id = ?'),delRefs: db.prepare('DELETE FROM tool_job_references WHERE job_id = ?'),-- addRef: db.prepare('INSERT OR IGNORE INTO tool_job_references (job_id, ref, created_at) VALUES (?, ?, ?)'),++ addRef: db.prepare('INSERT INTO tool_job_references (job_id, ref, created_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING'),dropRef: db.prepare('DELETE FROM tool_job_references WHERE job_id = ? AND ref = ?'),refs: db.prepare('SELECT ref, created_at FROM tool_job_references WHERE job_id = ? ORDER BY created_at, ref'),refCount: db.prepare('SELECT COUNT(*) AS n FROM tool_job_references WHERE job_id = ?'),// After the last reference goes, the result is kept at least one more ttl.-- releaseExpiry: db.prepare('UPDATE tool_jobs SET expires_at = MAX(COALESCE(expires_at, 0), @now + ttl_ms), updated_at = @now WHERE id = @id AND finished_at IS NOT NULL'),-- counts: db.prepare('SELECT state, COUNT(*) AS n FROM tool_jobs GROUP BY state'),++ releaseExpiry: db.prepare('UPDATE tool_jobs SET expires_at = GREATEST(COALESCE(expires_at, 0), @now + ttl_ms::bigint), updated_at = @now WHERE id = @id AND finished_at IS NOT NULL'),++ counts: db.prepare('SELECT state, COUNT(*) AS n FROM tool_jobs WHERE app = @app GROUP BY state'),};/** Append one event; returns its seq (the SSE id). */-- function appendEvent(jobId, event, data, now = Date.now()) {-- return Number(q.event.run(jobId, event, JSON.stringify(data), now).lastInsertRowid);++ async function appendEvent(jobId, event, data, now = Date.now()) {++ const r = await q.event.run({ job_id: jobId, event, data: JSON.stringify(data), at: now });++ return Number(r.lastInsertRowid);}return {db,++ app,TERMINAL, STATES,-- insert(row) { q.insert.run({ retry_of: null, ip_key: null, tool: null, project_id: null, trace_id: null, ...row }); },-- byResultMedia: (owner, mediaId) => q.byResultMedia.all(owner, `%"media_id":"${String(mediaId).replace(/[%"\\]/g, '')}"%`),-- markRetried: (id, next, now = Date.now()) => q.markRetried.run({ id, next, now }).changes === 1,-- get: (id) => q.get.get(id) || null,-- byIdempotencyKey: (owner, key) => q.byIdem.get(owner, key) || null,-- nextQueued: (limit) => q.nextQueued.all(limit),-- claim: (id, now = Date.now()) => q.claim.run({ id, now }).changes === 1,-- running: () => q.running.all(),-- activeForOwner: (owner) => q.activeForOwner.get(owner).n,-- activeForIp: (ipKey) => (ipKey ? q.activeForIp.get(ipKey).n : 0),-- setProgress: (id, progress, message, now = Date.now()) => q.progress.run({ id, progress, message, now }).changes === 1,-- requestCancel: (id, now = Date.now()) => q.requestCancel.run({ id, now }).changes === 1,-- requeue: (id, now = Date.now()) => q.requeue.run({ id, now }).changes === 1,-- finish: (id, { state, result = null, error = null, retryable = false }, now = Date.now()) => q.finish.run({++ async insert(row) { await q.insert.run({ app, retry_of: null, ip_key: null, tool: null, project_id: null, trace_id: null, ...row }); },++ byResultMedia: async (owner, mediaId) => q.byResultMedia.all({ app, owner, like: `%"media_id":"${String(mediaId).replace(/[%"\\]/g, '')}"%` }),++ markRetried: async (id, next, now = Date.now()) => (await q.markRetried.run({ id, next, now })).changes === 1,++ get: async (id) => (await q.get.get(id, app)) || null,++ byIdempotencyKey: async (owner, key) => (await q.byIdem.get({ app, owner, idempotency_key: key })) || null,++ nextQueued: (limit) => q.nextQueued.all({ app, limit }),++ claim: async (id, now = Date.now()) => (await q.claim.run({ id, now })).changes === 1,++ running: () => q.running.all({ app }),++ activeForOwner: async (owner) => (await q.activeForOwner.get({ app, owner })).n,++ activeForIp: async (ipKey) => (ipKey ? (await q.activeForIp.get({ app, ip_key: ipKey })).n : 0),++ setProgress: async (id, progress, message, now = Date.now()) => (await q.progress.run({ id, progress, message, now })).changes === 1,++ requestCancel: async (id, now = Date.now()) => (await q.requestCancel.run({ id, now })).changes === 1,++ requeue: async (id, now = Date.now()) => (await q.requeue.run({ id, now })).changes === 1,++ finish: async (id, { state, result = null, error = null, retryable = false }, now = Date.now()) => (await q.finish.run({id, state, now, retryable: retryable ? 1 : 0,result_json: result == null ? null : JSON.stringify(result),error_json: error == null ? null : JSON.stringify(error),-- }).changes === 1,++ })).changes === 1,setFiles: (id, files, now = Date.now()) => q.setFiles.run({ id, files_json: JSON.stringify(files), now }),appendEvent,-- eventsAfter: (jobId, seq) => q.eventsAfter.all(jobId, seq).map(e => ({ seq: e.seq, event: e.event, data: parse(e.data, {}), at: e.at })),-- lastSeq: (jobId) => q.lastSeq.get(jobId).seq || 0,-- expired: (now = Date.now()) => q.expired.all(now),++ eventsAfter: async (jobId, seq) => (await q.eventsAfter.all(jobId, seq)).map(e => ({ seq: Number(e.seq), event: e.event, data: parse(e.data, {}), at: Number(e.at) })),++ lastSeq: async (jobId) => Number((await q.lastSeq.get(jobId)).seq) || 0,++ expired: (now = Date.now()) => q.expired.all({ app, now }),deferExpiry: (id, until, now = Date.now()) => q.deferExpiry.run({ id, until, now }),-- remove: db.transaction((id) => { q.delEvents.run(id); q.delRefs.run(id); q.del.run(id); }),++ remove: (id) => db.tx(async () => { await q.delEvents.run(id); await q.delRefs.run(id); await q.del.run(id); }),/** → true when the reference is new. */-- addReference: (id, ref, now = Date.now()) => q.addRef.run(id, ref, now).changes === 1,++ addReference: async (id, ref, now = Date.now()) => (await q.addRef.run(id, ref, now)).changes === 1,/** → true when it existed; dropping the last one restarts the job's expiry clock. */-- dropReference: db.transaction((id, ref, now = Date.now()) => {-- const gone = q.dropRef.run(id, ref).changes === 1;-- if (gone && q.refCount.get(id).n === 0) q.releaseExpiry.run({ id, now });++ dropReference: (id, ref, now = Date.now()) => db.tx(async () => {++ const gone = (await q.dropRef.run(id, ref)).changes === 1;++ if (gone && (await q.refCount.get(id)).n === 0) await q.releaseExpiry.run({ id, now });return gone;}),references: (id) => q.refs.all(id),-- referenceCount: (id) => q.refCount.get(id).n,-- counts() { const out = Object.fromEntries(STATES.map(s => [s, 0])); for (const r of q.counts.all()) out[r.state] = r.n; return out; },-- transaction: (fn) => db.transaction(fn),++ referenceCount: async (id) => (await q.refCount.get(id)).n,++ counts: async () => { const out = Object.fromEntries(STATES.map(s => [s, 0])); for (const r of await q.counts.all({ app })) out[r.state] = Number(r.n); return out; },++ tx: (fn) => db.tx(fn),parse,};}@@@ -4,34 +4,34 @@//// accepted (202) → queued → running → succeeded | failed | cancelled//--// Durable: a job is written to the satellite's SQLite, and its input files moved under--// <dataDir>/jobs/<id>/in/, BEFORE the 202 goes out. On boot, rows left 'running' by a restart--// are re-queued (onRestart: 'requeue', while attempts remain) or failed with retryable = true--// (onRestart: 'fail'), per job type. Queued rows simply start again.++// Durable: a job is written to the one `tools` PostgreSQL database (plan T8), and its input files moved++// under <dataDir>/jobs/<id>/in/, BEFORE the 202 goes out. On boot, rows left 'running' by a restart are++// re-queued (onRestart: 'requeue', while attempts remain) or failed with retryable = true (onRestart:++// 'fail'), per job type. Queued rows simply start again.//--// Bounded: at most `concurrency` jobs run at once in this process, and each owner may have at--// most `maxActivePerOwner` queued + running jobs. busy() says when the whole store is over its--// bounds — `maxQueued` jobs waiting (all owners), or `diskBudgetBytes` under <dataDir>/jobs — so the--// HTTP layer can answer 503 tools.busy with Retry-After before it accepts an upload (through the--// guard: refused in enforce mode, recorded in report mode). Finished jobs expire (ttl chosen at submit),--// and the pruner deletes the row, its events, its files and any Media objects it made — except--// while something references the result (reference(), e.g. a paste or a project that points at the--// Media object), and never while Media keeps an object (a retention hold, or Media unreachable):--// then the job is kept and looked at again later, so no referenced result loses its record.++// Bounded: at most `concurrency` jobs run at once in this process, and each owner may have at most++// `maxActivePerOwner` queued + running jobs. busy() says when the whole store is over its bounds —++// `maxQueued` jobs waiting (all owners), or `diskBudgetBytes` under <dataDir>/jobs — so the HTTP layer can++// answer 503 tools.busy with Retry-After before it accepts an upload (through the guard: refused in++// enforce mode, recorded in report mode). Finished jobs expire (ttl chosen at submit), and the pruner++// deletes the row, its events, its files and any Media objects it made — except while something++// references the result (reference(), e.g. a paste or a project that points at the Media object), and++// never while Media keeps an object (a retention hold, or Media unreachable): then the job is kept and++// looked at again later, so no referenced result loses its record.//// Retry: a failed job keeps its input files until it expires; retry(id) moves them to a new job--// (retry_of → the failed one, which records retried_by). Retrying the same failed job again--// returns that same new job, so the call is idempotent.++// (retry_of → the failed one, which records retried_by). Retrying the same failed job again returns++// that same new job, so the call is idempotent.//--// Events: with an outbox (./events, openvibe-sdk createOutbox on this same database), created,--// started, succeeded and failed are also announced to OpenVibe.Events as tools.job.*, each written--// in the transaction that records the transition. Without one, nothing is announced.++// Events: with an outbox (./events, openvibe-sdk createPgOutbox on this same database), created,++// started, succeeded and failed are also announced to OpenVibe.Events as tools.job.*, each written in++// the transaction that records the transition. Without one, nothing is announced.//// Usage (WS-N task 4): with an outbox, a developer project's job that ends is also counted in the// transaction that records its end (./usage.js), and each closed hour goes out as tools.usage.recorded// from the pruner's timer.//--// No dependencies of its own: the app passes its better-sqlite3 handle and openvibe-contracts.++// No dependencies of its own: the app passes the shared openvibe-sdk/db handle and openvibe-contracts.// ═══════════════════════════════════════════════════════════════const fs = require('fs');@@@ -86,9 +86,9 @@ const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]')/*** @param {object} o-- * @param {object} o.db better-sqlite3 Database (the satellite's own jobs database)++ * @param {object} o.db openvibe-sdk/db handle (the one tools database)++ * @param {string} o.service 'img' | 'audio' | 'docs' | … (also the store's `app`)* @param {object} o.contracts require('openvibe-contracts')-- * @param {string} o.service 'img' | 'audio' | 'docs' | …* @param {string} o.dataDir where job files live (<dataDir>/jobs/<id>/…)* @param {number} [o.concurrency=2]* @param {number} [o.maxActivePerOwner=10]@@@ -104,7 +104,7 @@ const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]')function createJobSystem(o) {if (!o || !o.db || !o.contracts || !o.dataDir) throw new TypeError('createJobSystem needs db, contracts and dataDir');const { contracts } = o;-- const store = createStore(o.db);++ const store = createStore(o.db, { app: o.service });const log = o.log || console;const concurrency = Math.max(0, Number.isFinite(o.concurrency) ? o.concurrency : 2);const maxActivePerOwner = Math.max(1, o.maxActivePerOwner || 10);@@@ -117,17 +117,18 @@ function createJobSystem(o) {const diskBudget = o.diskBudgetBytes > 0 ? o.diskBudgetBytes : 0;const disk = { bytes: 0, at: 0, scanning: null };const announce = createJobEvents({ contracts, service: o.service, outbox: o.outbox || null, referenceCount: (id) => store.referenceCount(id), log });-- const usage = createJobUsage({ db: o.db, contracts, outbox: o.outbox || null, log });-- /** Inside the transaction that recorded the end: tools.job.succeeded|failed and the project's usage. */-- function ended(id) {-- const row = store.get(id);-- announce.finished(row);-- usage.finished(row);++ const usage = createJobUsage({ db: o.db, app: o.service, contracts, outbox: o.outbox || null, log });++ /** Inside the transaction `t` that recorded the end: tools.job.succeeded|failed and the project's usage. */++ async function ended(t, id) {++ const row = await store.get(id);++ await announce.finished(t, row);++ await usage.finished(row);}const types = new Map();const active = new Map(); // id → { ctrl, reason, last: { pct, msg, at } }const listeners = new Map(); // id → Set<fn>++ const emitChains = new Map(); // id → the tail of this job's event appends (strict per-job order)const stopHooks = new Set(); // e.g. open SSE streams, ended on stop()let started = false, stopped = false, pruneTimer = null, kicking = false;@@@ -151,7 +152,7 @@ function createJobSystem(o) {}// ── Public representation ─────────────────────────────────-- const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());++ const iso = (ms) => (ms == null ? null : new Date(Number(ms)).toISOString());function publicResult(result, id) {if (!result) return null;return {@@@ -164,21 +165,21 @@ function createJobSystem(o) {data: result.data || {},};}-- function view(row) {++ async function view(row) {if (!row) return null;const live = row.state === 'queued' || row.state === 'running';-- const references = store.references(row.id).map(r => ({ ref: r.ref, created_at: iso(r.created_at) }));++ const references = (await store.references(row.id)).map(r => ({ ref: r.ref, created_at: iso(r.created_at) }));return {id: row.id,object: 'tools.job',service: o.service,...(row.tool && { tool: row.tool }),type: row.type,-- type_version: row.type_version,++ type_version: Number(row.type_version),state: row.state,-- progress: { percent: row.progress == null ? null : Math.round(row.progress * 10) / 10, message: row.progress_message || null },-- attempts: row.attempts,-- max_attempts: row.max_attempts,++ progress: { percent: row.progress == null ? null : Math.round(Number(row.progress) * 10) / 10, message: row.progress_message || null },++ attempts: Number(row.attempts),++ max_attempts: Number(row.max_attempts),cancel_requested: !!row.cancel_requested,created_at: iso(row.created_at),started_at: iso(row.started_at),@@@ -202,13 +203,20 @@ function createJobSystem(o) {}// ── Events ─────────────────────────────────────────────────++ /** Append this job's next SSE event and tell its listeners; appends are serialized per job. */function emit(id, event) {-- if (stopped) return;-- const data = view(store.get(id));-- if (!data) return;-- const seq = store.appendEvent(id, event, data);-- const set = listeners.get(id);-- if (set) for (const fn of [...set]) { try { fn({ seq, event, data }); } catch (err) { log.error('[Jobs] listener error:', err.message); } }++ const prev = emitChains.get(id) || Promise.resolve();++ const next = prev.then(async () => {++ if (stopped) return;++ const data = await view(await store.get(id));++ if (!data) return;++ const seq = await store.appendEvent(id, event, data);++ const set = listeners.get(id);++ if (set) for (const fn of [...set]) { try { fn({ seq, event, data }); } catch (err) { log.error('[Jobs] listener error:', err.message); } }++ }).catch(err => log.error('[Jobs] could not record the event:', err.message))++ .finally(() => { if (emitChains.get(id) === next) emitChains.delete(id); });++ emitChains.set(id, next);++ return next;}function subscribe(id, fn) {if (!listeners.has(id)) listeners.set(id, new Set());@@@ -247,7 +255,7 @@ function createJobSystem(o) {const requestHash = crypto.createHash('sha256').update(canonical({ type, input, files: fileHashes })).digest('hex');if (idempotencyKey != null) {-- const prior = store.byIdempotencyKey(owner, String(idempotencyKey));++ const prior = await store.byIdempotencyKey(owner, String(idempotencyKey));if (prior) {await discard();if (prior.request_hash !== requestHash) throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request', { job_id: prior.id });@@@ -255,7 +263,7 @@ function createJobSystem(o) {}}const limit = env === 'sandbox' ? Math.min(SANDBOX_MAX_ACTIVE, maxActivePerOwner) : maxActivePerOwner;-- if (store.activeForOwner(owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);++ if (await store.activeForOwner(owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);const id = `job_${contracts.ids.ulid()}`;const inDir = path.join(jobDir(id), 'in');@@@ -265,11 +273,11 @@ function createJobSystem(o) {const f = files[i];const dest = path.join(inDir, `${i}-${SAFE_NAME(f.name)}`);if (f.buffer) await fsp.writeFile(dest, f.buffer); else await moveFile(f.path, dest);-- stored.push({ name: String(f.name || `file-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size: f.size != null ? f.size : fs.statSync(dest).size, sha256: fileHashes[i], _path: dest });++ stored.push({ name: String(f.name || `file-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size: f.size != null ? f.size : (await fsp.stat(dest)).size, sha256: fileHashes[i], _path: dest });}try {-- store.transaction(() => {-- store.insert({++ await store.tx(async (t) => {++ await store.insert({id, type, type_version: def.version, owner, input_json: JSON.stringify(input), files_json: JSON.stringify(stored),idempotency_key: idempotencyKey == null ? null : String(idempotencyKey), request_hash: requestHash,max_attempts: def.maxAttempts, ttl_ms: ttlMs, now: Date.now(), env,@@@ -278,22 +286,22 @@ function createJobSystem(o) {project_id: projectOf(project),trace_id: /^[0-9a-f]{32}$/.test(String(traceId || '')) ? String(traceId) : null,});-- announce.created(store.get(id));-- })();++ await announce.created(t, await store.get(id));++ });} catch (err) {await fsp.rm(jobDir(id), { recursive: true, force: true });// Two submits with one key raced: the other one won, so this is a replay of it.-- if (/UNIQUE/.test(err.message) && idempotencyKey != null) {-- const prior = store.byIdempotencyKey(owner, String(idempotencyKey));++ if (/UNIQUE|duplicate key/i.test(err.message) && idempotencyKey != null) {++ const prior = await store.byIdempotencyKey(owner, String(idempotencyKey));if (prior && prior.request_hash === requestHash) return { job: prior, replayed: true };throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request');}throw err;}disk.bytes += stored.reduce((n, f) => n + (f.size || 0), 0);-- emit(id, 'job.queued');++ await emit(id, 'job.queued');kick();-- return { job: store.get(id), replayed: false };++ return { job: await store.get(id), replayed: false };} catch (err) {await discard();throw err;@@@ -301,21 +309,21 @@ function createJobSystem(o) {}// ── Scheduling ─────────────────────────────────────────────-- function kick() {++ async function kick() {if (!started || stopped || kicking) return;kicking = true;try {while (active.size < concurrency) {-- const [row] = store.nextQueued(1);++ const [row] = await store.nextQueued(1);if (!row) break;// The claim and its tools.job.started event commit together (or neither does).let claimed;try {-- claimed = store.transaction(() => {-- if (!store.claim(row.id)) return false;-- announce.started(store.get(row.id));++ claimed = await store.tx(async (t) => {++ if (!(await store.claim(row.id))) return false;++ await announce.started(t, await store.get(row.id));return true;-- })();++ });} catch (err) {log.error(`[Jobs] could not start ${row.id}:`, err.message);break; // it stays queued; the next kick tries again@@@ -323,7 +331,7 @@ function createJobSystem(o) {if (!claimed) continue;const entry = { ctrl: new AbortController(), reason: null, last: { pct: -1, msg: null, at: 0 } };active.set(row.id, entry);-- emit(row.id, 'job.running');++ await emit(row.id, 'job.running');setImmediate(() => execute(row.id, entry));}} finally { kicking = false; }@@@ -337,12 +345,12 @@ function createJobSystem(o) {const now = Date.now();if (msg === entry.last.msg && (Math.abs(pct - entry.last.pct) < 1 || now - entry.last.at < throttleMs)) return;entry.last = { pct, msg, at: now };-- if (store.setProgress(id, pct, msg, now)) emit(id, 'job.progress');++ store.setProgress(id, pct, msg, now).then((changed) => { if (changed) return emit(id, 'job.progress'); }).catch(() => {});};}async function execute(id, entry) {-- const row = stopped ? null : store.get(id);++ const row = stopped ? null : await store.get(id);if (!row) { active.delete(id); return; }const def = types.get(row.type);const outDir = path.join(jobDir(row.id), 'out');@@@ -351,9 +359,12 @@ function createJobSystem(o) {if (timer.unref) timer.unref();try {await fsp.mkdir(outDir, { recursive: true });++ // A cancel (or a timeout) that arrived while we were preparing must end the job now: the run++ // would not see an abort that fired before it registered its listener.++ if (entry.reason) throw new Error(entry.reason);const files = store.parse(row.files_json, []).map(f => ({ name: f.name, mime: f.mime, size: f.size, sha256: f.sha256, path: f._path }));const out = await def.run({-- job: { id: row.id, type: row.type, attempt: row.attempts, owner: row.owner },++ job: { id: row.id, type: row.type, attempt: Number(row.attempts), owner: row.owner },input: store.parse(row.input_json, {}), files, outDir,signal: entry.ctrl.signal,progress: progressFn(row.id, entry),@@@ -366,7 +377,7 @@ function createJobSystem(o) {outcome = { state: 'succeeded', result };} catch (err) {if (stopped) return;-- const fresh = store.get(row.id) || row;++ const fresh = (await store.get(row.id)) || row;if (entry.reason === 'cancel' || fresh.cancel_requested) outcome = { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') };else if (entry.reason === 'timeout') outcome = { state: 'failed', error: problem(504, 'tools.job.timeout', `The job ran longer than ${Math.round(def.timeoutMs / 1000)} s`), retryable: true };else {@@@ -381,9 +392,9 @@ function createJobSystem(o) {let recorded = false;try {// The end state and its tools.job.succeeded|failed event commit together (or neither does).-- store.transaction(() => { if (store.finish(row.id, outcome)) ended(row.id); })();++ await store.tx(async (t) => { if (await store.finish(row.id, outcome)) await ended(t, row.id); });recorded = true;-- emit(row.id, EVENT_FOR[outcome.state]);++ await emit(row.id, EVENT_FOR[outcome.state]);} catch (err) {// Nothing committed: the row is still 'running' with its inputs, which the next start() recovers.log.error(`[Jobs] could not record the end of ${row.id}:`, err.message);@@@ -409,7 +420,7 @@ function createJobSystem(o) {await moveFile(local, dest);local = dest;}-- const size = fs.statSync(local).size;++ const size = (await fsp.stat(local)).size;const sha256 = await sha256File(local);const rec = { name: String(f.name || `result-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size, sha256, storage: 'local', media: null, _path: local };// A developer app's results (sandbox too) go under its project's namespace (WS-L task 5); other@@@ -442,23 +453,29 @@ function createJobSystem(o) {// ── Cancel ─────────────────────────────────────────────────/** → { job, changed } — queued jobs end at once; running ones are signalled and end as cancelled. */-- function cancel(id) {-- const row = store.get(id);++ async function cancel(id) {++ const row = await store.get(id);if (!row) return null;if (TERMINAL.includes(row.state)) return { job: row, changed: false };if (row.state === 'queued') {-- const done = store.transaction(() => store.requestCancel(id) && store.finish(id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled before it started') }))();++ const done = await store.tx(async () => {++ if (!(await store.requestCancel(id))) return false;++ return store.finish(id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled before it started') });++ });if (done) {-- emit(id, 'job.cancelled');++ await emit(id, 'job.cancelled');fsp.rm(jobDir(id), { recursive: true, force: true }).catch(() => {});}-- return { job: store.get(id), changed: done };++ return { job: await store.get(id), changed: done };}-- store.requestCancel(id);++ await store.requestCancel(id);++ // Read the row (still 'running') before signalling the worker: the HTTP layer decides 200 vs 202++ // from it, and the abort must not race that answer.++ const job = await store.get(id);const entry = active.get(id);if (entry && !entry.reason) { entry.reason = 'cancel'; entry.ctrl.abort(new Error('cancelled')); }-- emit(id, 'job.cancel_requested');-- return { job: store.get(id), changed: true };++ await emit(id, 'job.cancel_requested');++ return { job, changed: true };}// ── Retry ──────────────────────────────────────────────────@@@ -466,13 +483,12 @@ function createJobSystem(o) {* Retry a failed job as a new job with the same type, input, files and lifetime.* → { job, replayed } | null (no such job). Idempotent: a failed job is retried once, and asking* again returns that retry, whatever state it is in now. Only failed jobs can be retried.-- * Everything up to the insert is synchronous, so two concurrent calls cannot both retry it.*/-- function retry(id) {-- const row = store.get(id);++ async function retry(id) {++ const row = await store.get(id);if (!row) return null;if (row.retried_by) {-- const next = store.get(row.retried_by);++ const next = await store.get(row.retried_by);if (next) return { job: next, replayed: true };throw new JobError(410, 'tools.job.retry_gone', 'This job was retried and the retry has since expired', { retried_by: row.retried_by });}@@@ -481,7 +497,7 @@ function createJobSystem(o) {const def = types.get(row.type);if (!def) throw new JobError(409, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`);const limit = row.env === 'sandbox' ? Math.min(SANDBOX_MAX_ACTIVE, maxActivePerOwner) : maxActivePerOwner;-- if (store.activeForOwner(row.owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);++ if (await store.activeForOwner(row.owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);const inputs = store.parse(row.files_json, []);if (inputs.some(f => !f._path || !fs.existsSync(f._path))) throw new JobError(410, 'tools.job.inputs_gone', 'The input files of this job are no longer kept; submit it again');@@@ -496,23 +512,23 @@ function createJobSystem(o) {}const stored = inputs.map(f => ({ ...f, _path: path.join(newIn, path.basename(f._path)) }));try {-- store.transaction(() => {-- if (!store.markRetried(row.id, next)) throw new JobError(409, 'tools.job.not_failed', 'This job cannot be retried any more');-- store.insert({++ await store.tx(async (t) => {++ if (!(await store.markRetried(row.id, next))) throw new JobError(409, 'tools.job.not_failed', 'This job cannot be retried any more');++ await store.insert({id: next, type: row.type, type_version: def.version, owner: row.owner, input_json: row.input_json, files_json: JSON.stringify(stored),idempotency_key: null, request_hash: row.request_hash, max_attempts: def.maxAttempts, ttl_ms: row.ttl_ms, now: Date.now(),env: row.env, retry_of: row.id, ip_key: row.ip_key || null, tool: row.tool || null, project_id: row.project_id || null,});-- announce.created(store.get(next));-- })();++ await announce.created(t, await store.get(next));++ });} catch (err) {if (moved) { try { fs.renameSync(newIn, oldIn); } catch { /* best effort */ } }fs.rmSync(jobDir(next), { recursive: true, force: true });throw err;}-- emit(next, 'job.queued');++ await emit(next, 'job.queued');kick();-- return { job: store.get(next), replayed: false };++ return { job: await store.get(next), replayed: false };}// ── References (results something still points at) ─────────@@@ -525,58 +541,58 @@ function createJobSystem(o) {/*** Keep a succeeded job's result while `ref` points at it. → { job, created }. Idempotent.*/-- function reference(id, ref) {++ async function reference(id, ref) {ref = checkRef(ref);-- const row = store.get(id);++ const row = await store.get(id);if (!row) return null;if (row.state !== 'succeeded') throw new JobError(409, 'tools.job.not_succeeded', `Only a succeeded job's result can be referenced; this one is ${row.state}`, { state: row.state });if (row.env === 'sandbox') throw new JobError(409, 'tools.job.sandbox', 'Sandbox results are kept briefly and cannot be referenced');-- const has = store.references(id).some(r => r.ref === ref);-- if (!has && store.referenceCount(id) >= MAX_REFS) throw new JobError(409, 'tools.job.too_many_references', `At most ${MAX_REFS} references per job`);-- const created = store.addReference(id, ref);-- return { job: store.get(id), created };++ const has = (await store.references(id)).some(r => r.ref === ref);++ if (!has && await store.referenceCount(id) >= MAX_REFS) throw new JobError(409, 'tools.job.too_many_references', `At most ${MAX_REFS} references per job`);++ const created = await store.addReference(id, ref);++ return { job: await store.get(id), created };}/** Stop keeping the result for `ref`. → { job, removed }. Idempotent. */-- function unreference(id, ref) {++ async function unreference(id, ref) {ref = checkRef(ref);-- if (!store.get(id)) return null;-- const removed = store.dropReference(id, ref);-- return { job: store.get(id), removed };++ if (!(await store.get(id))) return null;++ const removed = await store.dropReference(id, ref);++ return { job: await store.get(id), removed };}// ── Boot recovery, pruning ─────────────────────────────────/** A job the restart ended as failed: the row and its tools.job.failed event in one transaction. */-- function failRecovered(id, outcome) {-- store.transaction(() => { if (store.finish(id, outcome)) ended(id); })();++ async function failRecovered(id, outcome) {++ await store.tx(async (t) => { if (await store.finish(id, outcome)) await ended(t, id); });}-- function recover() {++ async function recover() {let requeued = 0, failed = 0;-- for (const row of store.running()) {++ for (const row of await store.running()) {const def = types.get(row.type);if (row.cancel_requested) {-- store.finish(row.id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') });-- emit(row.id, 'job.cancelled');++ await store.finish(row.id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') });++ await emit(row.id, 'job.cancelled');} else if (!def) {-- failRecovered(row.id, { state: 'failed', error: problem(500, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`) });-- emit(row.id, 'job.failed'); failed++;-- } else if (def.onRestart === 'requeue' && row.attempts < row.max_attempts) {-- store.requeue(row.id);-- emit(row.id, 'job.queued'); requeued++;++ await failRecovered(row.id, { state: 'failed', error: problem(500, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`) });++ await emit(row.id, 'job.failed'); failed++;++ } else if (def.onRestart === 'requeue' && Number(row.attempts) < Number(row.max_attempts)) {++ await store.requeue(row.id);++ await emit(row.id, 'job.queued'); requeued++;} else {const retryable = def.onRestart === 'fail';-- failRecovered(row.id, {++ await failRecovered(row.id, {state: 'failed', retryable,error: problem(503, 'tools.job.interrupted', retryable ? 'The service restarted while this job was running; submit it again' : `The service restarted during each of ${row.attempts} attempts`),});-- emit(row.id, 'job.failed'); failed++;++ await emit(row.id, 'job.failed'); failed++;}fs.rmSync(path.join(jobDir(row.id), 'out'), { recursive: true, force: true });}// Directories without a row: a crash between moving the upload and recording the job.// That job was never accepted (no 202 went out), so its files have no owner.for (const name of fs.readdirSync(root)) {-- if (/^job_[0-9A-Z]{26}$/.test(name) && !store.get(name)) fs.rmSync(path.join(root, name), { recursive: true, force: true });++ if (/^job_[0-9A-Z]{26}$/.test(name) && !(await store.get(name))) fs.rmSync(path.join(root, name), { recursive: true, force: true });}return { requeued, failed };}@@@ -590,7 +606,7 @@ function createJobSystem(o) {*/async function prune(now = Date.now()) {let n = 0;-- for (const row of store.expired(now)) {++ for (const row of await store.expired(now)) {const result = store.parse(row.result_json, null);let kept = null;for (const f of (result && result.files) || []) {@@@ -602,11 +618,11 @@ function createJobSystem(o) {}if (kept) {log.warn(`[Jobs] kept expired job ${row.id} (${kept}); checking again later`);-- store.deferExpiry(row.id, now + PRUNE_RECHECK_MS);++ await store.deferExpiry(row.id, now + PRUNE_RECHECK_MS);continue;}await fsp.rm(jobDir(row.id), { recursive: true, force: true }).catch(() => {});-- store.remove(row.id);++ await store.remove(row.id);listeners.delete(row.id);n++;}@@@ -640,9 +656,9 @@ function createJobSystem(o) {* Is the store over its bounds? → null, or { reason: 'queue'|'disk', detail, retryAfter } for a* 503 tools.busy. Cheap: one COUNT and the last disk walk (a stale walk is refreshed in the background).*/-- function busy() {++ async function busy() {if (maxQueued) {-- const queued = store.counts().queued || 0;++ const queued = (await store.counts()).queued || 0;if (queued >= maxQueued) return { reason: 'queue', detail: `${queued} jobs are already waiting on this server; try again in a minute.`, retryAfter: 60, queued, limit: maxQueued };}if (diskBudget) {@@@ -657,26 +673,26 @@ function createJobSystem(o) {* { active, limit } when a new session job from `ipKey` would pass it (the HTTP layer answers 429* tools.job.too_many_active through the guard). Principals and people are bounded per owner only.*/-- function addressFull(owner, ipKey) {++ async function addressFull(owner, ipKey) {if (!ipKey || !String(owner || '').startsWith('session:')) return null;-- const active = store.activeForIp(ipKey);++ const active = await store.activeForIp(ipKey);return active >= maxActivePerAddress ? { active, limit: maxActivePerAddress } : null;}-- function start() {++ async function start() {if (started) return api;started = true;if (diskBudget) refreshDisk();-- const r = recover();++ const r = await recover();if (r.requeued || r.failed) log.log(`[Jobs] ${o.service}: ${r.requeued} job(s) re-queued, ${r.failed} failed after restart`);-- prune().catch(() => {});-- flushUsage();++ await prune().catch(() => {});++ await flushUsage();pruneTimer = setInterval(() => {prune().catch(err => log.error('[Jobs] prune:', err.message));flushUsage();}, o.pruneIntervalMs || 5 * 60 * 1000);if (pruneTimer.unref) pruneTimer.unref();-- kick();++ await kick();return api;}@@@ -685,8 +701,8 @@ function createJobSystem(o) {* crash or a restart leaves behind; the next start() recovers them per type.*//** Closed hours of project usage → the outbox (./usage.js); a failure is logged and retried next tick. */-- function flushUsage(at) {-- try { return usage.flush(at); } catch (err) { log.error('[Jobs] usage flush:', err.message); return { queued: 0, invalid: 0 }; }++ async function flushUsage(at) {++ try { return await usage.flush(at); } catch (err) { log.error('[Jobs] usage flush:', err.message); return { queued: 0, invalid: 0 }; }}function stop() {@@@ -701,8 +717,8 @@ function createJobSystem(o) {function onStop(fn) { stopHooks.add(fn); return () => stopHooks.delete(fn); }/** The owner's succeeded job with a result file stored as Media object `mediaId` → { row, index } or null. */-- function findResultMedia(owner, mediaId) {-- for (const row of store.byResultMedia(owner, mediaId)) {++ async function findResultMedia(owner, mediaId) {++ for (const row of await store.byResultMedia(owner, mediaId)) {const files = (store.parse(row.result_json, null) || {}).files || [];const index = files.findIndex(f => f.media && f.media.media_id === mediaId);if (index >= 0) return { row, index };@@@ -728,7 +744,7 @@ function createJobSystem(o) {types: () => [...types.keys()],media,// running (after the spread) is the store's count of rows in 'running'; executing is this process's.-- stats: () => ({ running: active.size, concurrency, ...store.counts(), executing: active.size, results: media ? 'media' : 'local', events: announce.status(), usage: usage.enabled ? usage.status() : { enabled: false } }),++ stats: async () => ({ running: active.size, concurrency, ...(await store.counts()), executing: active.size, results: media ? 'media' : 'local', events: await announce.status(), usage: usage.enabled ? await usage.status() : { enabled: false } }),usage, flushUsage,outbox: o.outbox || null,/** True between start() and stop(): the worker picks up queued jobs. */@@@ -19,7 +19,8 @@// outbox relay publishes it. A job that ends in an hour already sent (a clock step back) reopens it as// revision + 1. Only counts leave: no owner, session, address, input, file name or output.//--// Inert without an outbox (EVENTS_URL unset, like ./events.js): nothing is counted.++// On PostgreSQL (plan T8): openvibe-sdk/db, ambient transactions, createPgOutbox. Inert without an++// outbox (EVENTS_URL unset, like ./events.js): nothing is counted.// ═══════════════════════════════════════════════════════════════const HOUR_MS = 60 * 60 * 1000;@@@ -32,25 +33,6 @@ const CODE_RE = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+)+$/;const TRACE_RE = /^[0-9a-f]{32}$/;const DIMENSION_RE = /^[a-z0-9][a-z0-9_.:-]{0,79}$/;--const SCHEMA = `--CREATE TABLE IF NOT EXISTS tool_job_usage (-- project_id TEXT NOT NULL,-- env TEXT NOT NULL,-- capability TEXT NOT NULL,-- dimension TEXT NOT NULL,-- window_start INTEGER NOT NULL,-- quantity INTEGER NOT NULL DEFAULT 0,-- errors INTEGER NOT NULL DEFAULT 0,-- error_codes TEXT NOT NULL DEFAULT '{}',-- samples TEXT NOT NULL DEFAULT '[]',-- revision INTEGER NOT NULL DEFAULT 1,-- event_id TEXT,-- emitted_at INTEGER,-- PRIMARY KEY (project_id, env, capability, dimension, window_start)--);--CREATE INDEX IF NOT EXISTS tool_job_usage_open ON tool_job_usage(emitted_at, window_start);--`;--const hourOf = (ms) => Math.floor(ms / HOUR_MS) * HOUR_MS;const parse = (s, d) => { try { return JSON.parse(s); } catch { return d; } };@@@ -69,60 +51,60 @@ function keyOf(row) {/*** @param {object} o-- * @param {object} o.db the jobs database (better-sqlite3), where the outbox lives too++ * @param {object} o.db openvibe-sdk/db handle (the one tools database), where the outbox lives too++ * @param {string} o.app the owning app ('img' | …)* @param {object} o.contracts require('openvibe-contracts') (0.63.0+ knows tools.usage.recorded@1)-- * @param {object|null} o.outbox openvibe-sdk outbox on `db` (./events outboxFromEnv); null = inert++ * @param {object|null} o.outbox openvibe-sdk createPgOutbox on `db` (./events outboxFromEnv); null = inert* @param {() => number} [o.now]* @param {object} [o.log]*/--function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(), log = console }) {-- if (!outbox) return { enabled: false, finished() {}, flush: () => ({ queued: 0, invalid: 0 }), pending: () => 0 };-- db.exec(SCHEMA);++function createJobUsage({ db, app, contracts, outbox = null, now = () => Date.now(), log = console }) {++ if (!outbox) return { enabled: false, finished: async () => {}, flush: async () => ({ queued: 0, invalid: 0 }), pending: async () => 0, status: async () => ({ enabled: false }) };const q = {-- add: db.prepare(`INSERT INTO tool_job_usage (project_id, env, capability, dimension, window_start, quantity, errors)-- VALUES (@project_id, @env, @capability, @dimension, @window_start, 1, @errors)-- ON CONFLICT(project_id, env, capability, dimension, window_start) DO UPDATE SET-- quantity = quantity + 1, errors = errors + excluded.errors,-- revision = revision + (CASE WHEN emitted_at IS NULL THEN 0 ELSE 1 END), emitted_at = NULL`),-- get: db.prepare('SELECT error_codes, samples FROM tool_job_usage WHERE project_id = ? AND env = ? AND capability = ? AND dimension = ? AND window_start = ?'),-- setErrors: db.prepare('UPDATE tool_job_usage SET error_codes = ?, samples = ? WHERE project_id = ? AND env = ? AND capability = ? AND dimension = ? AND window_start = ?'),-- due: db.prepare('SELECT * FROM tool_job_usage WHERE emitted_at IS NULL AND window_start <= ? ORDER BY window_start LIMIT 500'),-- sent: db.prepare('UPDATE tool_job_usage SET emitted_at = ?, event_id = ? WHERE project_id = ? AND env = ? AND capability = ? AND dimension = ? AND window_start = ? AND emitted_at IS NULL'),-- prune: db.prepare('DELETE FROM tool_job_usage WHERE emitted_at IS NOT NULL AND window_start < ?'),-- pending: db.prepare('SELECT COUNT(*) AS n FROM tool_job_usage WHERE emitted_at IS NULL'),++ add: db.prepare(`INSERT INTO tool_job_usage (app, project_id, env, capability, dimension, window_start, quantity, errors)++ VALUES (@app, @project_id, @env, @capability, @dimension, @window_start, 1, @errors)++ ON CONFLICT(app, project_id, env, capability, dimension, window_start) DO UPDATE SET++ quantity = tool_job_usage.quantity + 1, errors = tool_job_usage.errors + excluded.errors,++ revision = tool_job_usage.revision + (CASE WHEN tool_job_usage.emitted_at IS NULL THEN 0 ELSE 1 END), emitted_at = NULL`),++ get: db.prepare('SELECT error_codes, samples FROM tool_job_usage WHERE app = @app AND project_id = @project_id AND env = @env AND capability = @capability AND dimension = @dimension AND window_start = @window_start'),++ setErrors: db.prepare('UPDATE tool_job_usage SET error_codes = @error_codes, samples = @samples WHERE app = @app AND project_id = @project_id AND env = @env AND capability = @capability AND dimension = @dimension AND window_start = @window_start'),++ due: db.prepare('SELECT * FROM tool_job_usage WHERE app = @app AND emitted_at IS NULL AND window_start <= @at ORDER BY window_start LIMIT 500'),++ sent: db.prepare('UPDATE tool_job_usage SET emitted_at = @at, event_id = @event_id WHERE app = @app AND project_id = @project_id AND env = @env AND capability = @capability AND dimension = @dimension AND window_start = @window_start AND emitted_at IS NULL'),++ prune: db.prepare('DELETE FROM tool_job_usage WHERE app = @app AND emitted_at IS NOT NULL AND window_start < @at'),++ pending: db.prepare('SELECT COUNT(*) AS n FROM tool_job_usage WHERE app = @app AND emitted_at IS NULL'),};const stats = { invalid: 0, lastInvalid: null };/** INSIDE the transaction that records the job's end (store.finish), with the row read back. */-- function finished(row) {++ async function finished(row) {const k = keyOf(row);if (!k) return;const failed = row.state === 'failed';-- q.add.run({ ...k, errors: failed ? 1 : 0 });++ await q.add.run({ app, ...k, errors: failed ? 1 : 0 });if (!failed) return;const error = parse(row.error_json, null) || {};const code = CODE_RE.test(String(error.code || '')) ? String(error.code) : 'tools.job.failed';-- const cur = q.get.get(k.project_id, k.env, k.capability, k.dimension, k.window_start);++ const cur = await q.get.get({ app, ...k });const codes = parse(cur.error_codes, {});if (codes[code] || Object.keys(codes).length < MAX_CODES) codes[code] = (codes[code] || 0) + 1;const sample = { at: new Date(row.finished_at || now()).toISOString(), code, ref: row.id };if (Number.isInteger(error.status) && error.status >= 100 && error.status <= 599) sample.status = error.status;if (TRACE_RE.test(String(row.trace_id || ''))) sample.trace_id = row.trace_id;const samples = [sample, ...parse(cur.samples, [])].slice(0, MAX_SAMPLES);-- q.setErrors.run(JSON.stringify(codes), JSON.stringify(samples), k.project_id, k.env, k.capability, k.dimension, k.window_start);++ await q.setErrors.run({ app, ...k, error_codes: JSON.stringify(codes), samples: JSON.stringify(samples) });}function payloadOf(row) {const p = {project_id: row.project_id, env: row.env, capability: row.capability, dimension: row.dimension, unit: 'jobs', window: 'hour',-- window_start: new Date(row.window_start).toISOString(), window_end: new Date(row.window_start + HOUR_MS).toISOString(),-- quantity: row.quantity, errors: row.errors,++ window_start: new Date(Number(row.window_start)).toISOString(), window_end: new Date(Number(row.window_start) + HOUR_MS).toISOString(),++ quantity: Number(row.quantity), errors: Number(row.errors),};const codes = parse(row.error_codes, {});if (Object.keys(codes).length) p.error_codes = codes;const samples = parse(row.samples, []);if (samples.length) p.samples = samples;-- if (row.revision > 1) p.revision = row.revision;++ if (Number(row.revision) > 1) p.revision = Number(row.revision);return p;}@@@ -130,9 +112,9 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),* Write every closed hour's rollups to the outbox (each in the transaction that marks it sent).* A payload that fails its contract is logged and left unsent, never thrown.*/-- function flush(at = now()) {++ async function flush(at = now()) {let queued = 0;-- for (const row of q.due.all(hourOf(at - GRACE_MS) - HOUR_MS)) {++ for (const row of await q.due.all({ app, at: hourOf(at - GRACE_MS) - HOUR_MS })) {const payload = payloadOf(row);const v = contracts.validate('tools.usage.recorded@1', payload);if (!v.valid) {@@@ -141,8 +123,8 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),log.error(`[Jobs] tools.usage.recorded not sent (payload does not match the contract): ${stats.lastInvalid}`);continue;}-- db.transaction(() => {-- const env = outbox.enqueue({++ await db.tx(async (t) => {++ const env = await outbox.enqueue(t, {event_type: 'tools.usage.recorded',actor: { type: 'service', id: 'tools' },subject: { type: 'project', id: row.project_id },@@@ -150,16 +132,20 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),priority: 'low',payload,});-- q.sent.run(at, env.event_id, row.project_id, row.env, row.capability, row.dimension, row.window_start);-- })();++ await q.sent.run({ app, ...row, event_id: env.event_id, at });++ });queued++;}-- q.prune.run(at - KEEP_SENT_MS);++ await q.prune.run({ app, at: at - KEEP_SENT_MS });if (queued) outbox.kick();return { queued, invalid: stats.invalid };}-- return { enabled: true, finished, flush, payloadOf, pending: () => q.pending.get().n, status: () => ({ pending: q.pending.get().n, invalid: stats.invalid, last_invalid: stats.lastInvalid }) };++ return {++ enabled: true, finished, flush, payloadOf,++ pending: async () => (await q.pending.get({ app })).n,++ status: async () => ({ pending: (await q.pending.get({ app })).n, invalid: stats.invalid, last_invalid: stats.lastInvalid }),++ };}module.exports = { createJobUsage, keyOf, HOUR_MS, GRACE_MS };@@@ -52,18 +52,18 @@ function observe(o) {if (o.jobs) {inst.registry.gauge({name: 'tools_jobs', help: 'Jobs in this satellite\'s job store by state', labelNames: ['app', 'state'],-- collect: () => {++ collect: async () => {const sys = o.jobs();if (!sys) return null;-- return Object.entries(sys.store.counts()).map(([state, n]) => ({ labels: { app: appName, state }, value: n }));++ return Object.entries(await sys.store.counts()).map(([state, n]) => ({ labels: { app: appName, state }, value: n }));},});inst.registry.gauge({name: 'tools_jobs_executing', help: 'Jobs executing in this process now, and the concurrency limit', labelNames: ['app', 'kind'],-- collect: () => {++ collect: async () => {const sys = o.jobs();if (!sys) return null;-- const s = sys.stats();++ const s = await sys.stats();return [{ labels: { app: appName, kind: 'executing' }, value: s.executing }, { labels: { app: appName, kind: 'limit' }, value: s.concurrency }];},});@@@ -119,14 +119,14 @@ const checks = {},/** The job runtime: started and not stopped, its store answering (counts are a real query). */-- jobRuntime(name, getSystem, { required = true, description = 'job runtime: worker started, jobs.db answering' } = {}) {++ jobRuntime(name, getSystem, { required = true, description = 'job runtime: worker started, store answering' } = {}) {return {name, required, description,-- check: () => {++ check: async () => {const sys = getSystem();if (!sys) return 'job runtime not set up yet';if (!sys.isRunning()) return 'job runtime is not running';-- const s = sys.stats(); // store.counts() inside: a real query on jobs.db++ const s = await sys.stats(); // store.counts() inside: a real queryreturn { ok: true, detail: { queued: s.queued, running: s.running, executing: s.executing, concurrency: s.concurrency, results: s.results } };},};@@@ -46,7 +46,7 @@ const SVC = principal('svc:network');const log = { log() {}, error() {}, warn: (m) => lines.push(String(m)) };const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-actor-limits-'));const guard = createGuard({-- app: 'test', dataDir, Database: dep('better-sqlite3'), contracts, specs: [], issuer: ISSUER,++ app: 'test', dataDir, contracts, specs: [], issuer: ISSUER,keys: { get: () => publicKey, ensure: async () => publicKey }, env: { TOOLS_GUARD: 'report' }, log, pruneIntervalMs: 0,});const registry = metrics.createRegistry();@@@ -1,181 +1,124 @@'use strict';--// ADR-021 analytics bounds for every satellite (openvibe-shared/analytics, scripts/analytics-prune.js):--// a tracked request stores no IP, user id, city, raw user agent, raw referer or query string anywhere--// in analytics.db; paths are route templates; uniques come from day-scoped hashes deleted after the--// day's rollup; prune removes strictly-older raw rows in bounded batches and never touches rollups;--// scrub rewrites legacy rows without changing a rollup counter; the CLI dry run changes nothing and--// --apply needs --backup or --no-backup.++// ADR-021 analytics bounds for every satellite, on PostgreSQL (plan T8, decision 5): a tracked request++// stores no IP, user id, city, raw user agent, raw referer or query string anywhere in the analytics++// tables; paths are route templates; uniques come from day-scoped hashes deleted after the day's rollup;++// pruneRawEventsPg removes strictly-older raw rows and day hashes in bounded batches and never touches++// rollups. (The SQLite tracker's scrub and file CLI are retired with better-sqlite3.)const assert = require('assert');--const crypto = require('crypto');const fs = require('fs');const os = require('os');const path = require('path');const { dep } = require('./deps');--const { AnalyticsTracker, privacy, retention } = dep('openvibe-shared/analytics');++const { testDb, closeAllTestDbs } = require('./testdb');++const { AnalyticsTrackerPg, privacy, pruneRawEventsPg } = dep('openvibe-shared/analytics/pg');const { sqlTime } = dep('openvibe-shared/analytics/tracker');--const cli = require('../../../scripts/analytics-prune');const express = dep('express');--const Database = dep('better-sqlite3');++const DAY = 86400000;const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-tools-analytics-'));const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';--const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };--const dumpAll = (db) => db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all()-- .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');++const dayOf = (ms) => sqlTime(ms).slice(0, 10);--function seedLegacy(db, nowMs) {-- new AnalyticsTracker(db, 'openvibe-yt', { timers: false }).destroy();-- const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)-- VALUES ('openvibe-yt', 'pageview', ?, 'GET', 200, ?, ?, ?, ?, ?, ?, ?)`);-- const cutoffMs = nowMs - 30 * 86400000;-- for (let i = 0; i < 10; i++) ins.run(`/watch/dQw4w9WgXc${i}?v=${i}`, 7, 'tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/x?y=1', sqlTime(nowMs - (40 + i) * 86400000));-- ins.run('/edge-older', null, null, '198.51.100.50', null, CHROME, '', sqlTime(cutoffMs - 1000));-- ins.run('/edge-exact', null, null, '198.51.100.51', null, CHROME, '', sqlTime(cutoffMs));-- ins.run('/api/info/abc123def?url=https://youtube.com/watch?v=x', 9, 'abcdef0123456789', '198.51.100.52', 'Oslo', FIREFOX, 'https://www.reddit.com/r/a', sqlTime(nowMs - 2 * 86400000));-- const tops = JSON.stringify([{ path: '/watch/dQw4w9WgXcQ', cnt: 3 }, { path: '/watch/9bZkp7q19f0', cnt: 2 }]);-- const refs = JSON.stringify([{ referer: 'https://www.google.com/search?q=x', cnt: 4 }]);-- db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')-- .run('openvibe-yt', sqlTime(nowMs - 60 * 86400000).slice(0, 10), 100, 40, 30, 10, 2, tops, refs);-- db.prepare('INSERT INTO analytics_hourly (service, hour, pageviews, api_calls, unique_visitors, unique_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')-- .run('openvibe-yt', sqlTime(nowMs - 3 * 86400000).slice(0, 13) + ':00:00', 10, 4, 3, 1, tops, refs);-- db.prepare("INSERT INTO analytics_rate_tracking (ip, window_start, hit_count) VALUES ('198.51.100.99', 1, 1)").run();++/** Every analytics row as text, to prove a needle is nowhere. */++async function dumpAll(db) {++ const parts = [];++ for (const t of ['analytics_events', 'analytics_hourly', 'analytics_daily', 'analytics_visitor_days', 'analytics_day_salts']) {++ parts.push(JSON.stringify(await db.prepare(`SELECT * FROM ${t}`).all()));++ }++ return parts.join('\n');}(async () => {-- // ── Templates and reducers ──-- const n = privacy.normalisePath;-- assert.strictEqual(n('/watch/dQw4w9WgXcQ?v=1'), '/watch/:param');-- assert.strictEqual(n('/api/info?url=https://youtube.com/watch?v=abc'), '/api/info');-- assert.strictEqual(n('/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events'), '/api/jobs/:param/events');-- assert.strictEqual(n('/recipe/chicken-tikka'), '/recipe/:param');-- assert.strictEqual(n('/place/@52.37,4.89,14z'), '/place/@:user');-- assert.strictEqual(n('/@alex'), '/@:user');-- assert.strictEqual(n('/api/things/:id'), '/api/things/:id');-- assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=secret'), 'https://www.google.com');-- assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');++ try {++ // ── Templates and reducers ──++ const n = privacy.normalisePath;++ assert.strictEqual(n('/watch/dQw4w9WgXcQ?v=1'), '/watch/:param');++ assert.strictEqual(n('/api/info?url=https://youtube.com/watch?v=abc'), '/api/info');++ assert.strictEqual(n('/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events'), '/api/jobs/:param/events');++ assert.strictEqual(n('/recipe/chicken-tikka'), '/recipe/:param');++ assert.strictEqual(n('/place/@52.37,4.89,14z'), '/place/@:user');++ assert.strictEqual(n('/@alex'), '/@:user');++ assert.strictEqual(n('/api/things/:id'), '/api/things/:id');++ assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=secret'), 'https://www.google.com');++ assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');-- // ── A tracked request through express ──-- {-- const db = newDb('track.db');-- let clock = Date.parse('2026-09-23T10:15:00Z');-- const tracker = new AnalyticsTracker(db, 'openvibe-yt', { timers: false, now: () => clock });-- const app = express();-- app.set('trust proxy', true);-- app.use(tracker.middleware());-- app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT }; next(); });-- app.get('/api/jobs/:id', (req, res) => res.json({ ok: true }));-- app.get('*', (req, res) => res.send('ok'));-- const server = app.listen(0, '127.0.0.1');-- await new Promise((r) => server.once('listening', r));-- const base = `http://127.0.0.1:${server.address().port}`;-- const hdr = (x) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', ...x });-- try {-- await (await fetch(`${base}/api/jobs/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();-- await (await fetch(`${base}/watch/dQw4w9WgXcQ?v=1`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();-- await new Promise((r) => setTimeout(r, 50));-- } finally { server.close(); }-- tracker.flush();-- const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();-- assert.deepStrictEqual(rows.map((r) => r.path), ['/api/jobs/:id', '/watch/:param']);-- assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0]);-- for (const r of rows) {-- assert.ok(r.ip === null && r.user_id === null && r.city === null);-- assert.strictEqual(r.referer, 'https://www.google.com');-- assert.ok(/^[0-9a-f]{16}$/.test(r.session_id));-- }-- const all = dumpAll(db);-- for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'supersecret', 'secret-query', '98765', 'dQw4w9WgXcQ', 'Mozilla/5.0']) {-- assert.ok(!all.includes(needle), `found ${needle}`);++ // ── A tracked request through express ──++ {++ const db = await testDb(path.join(tmp, 'track'));++ let clock = Date.parse('2026-09-23T10:15:00Z');++ const tracker = new AnalyticsTrackerPg(db, 'openvibe-yt', { timers: false, now: () => clock });++ const app = express();++ app.set('trust proxy', true);++ app.use(tracker.middleware());++ app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT }; next(); });++ app.get('/api/jobs/:id', (req, res) => res.json({ ok: true }));++ app.get('*', (req, res) => res.send('ok'));++ const server = app.listen(0, '127.0.0.1');++ await new Promise((r) => server.once('listening', r));++ const base = `http://127.0.0.1:${server.address().port}`;++ const hdr = (x) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', ...x });++ try {++ await (await fetch(`${base}/api/jobs/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();++ await (await fetch(`${base}/watch/dQw4w9WgXcQ?v=1`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();++ await new Promise((r) => setTimeout(r, 50));++ } finally { server.close(); }++ await tracker.flush();++ const rows = await db.prepare('SELECT * FROM analytics_events ORDER BY id').all();++ assert.deepStrictEqual(rows.map((r) => r.path), ['/api/jobs/:id', '/watch/:param']);++ assert.deepStrictEqual(rows.map((r) => Number(r.authenticated)), [1, 0]);++ for (const r of rows) {++ assert.ok(r.ip === null && r.user_id === null && r.city === null);++ assert.strictEqual(r.referer, 'https://www.google.com');++ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id));++ }++ const all = await dumpAll(db);++ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'supersecret', 'secret-query', '98765', 'dQw4w9WgXcQ', 'Mozilla/5.0']) {++ assert.ok(!all.includes(needle), `found ${needle}`);++ }++ await tracker.aggregate();++ const d = await db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();++ assert.strictEqual(Number(d.unique_visitors), 2);++ assert.strictEqual(Number(d.unique_users), 1);++ clock = Date.parse('2026-09-24T00:10:00Z');++ await tracker.aggregate();++ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_visitor_days').get()).n), 0);++ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_day_salts').get()).n), 0);++ assert.strictEqual(Number((await db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").get()).unique_visitors), 2);++ assert.ok((await tracker.getStats({ days: 30 })).summary);++ assert.ok(Array.isArray((await tracker.getBotAnalysis(7)).topBotIPs));++ await tracker.destroy();}-- tracker.aggregate();-- const d = db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();-- assert.strictEqual(d.unique_visitors, 2);-- assert.strictEqual(d.unique_users, 1);-- clock = Date.parse('2026-09-24T00:10:00Z');-- tracker.aggregate();-- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 0);-- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 0);-- assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2);-- assert.ok(tracker.getStats({ days: 30 }).summary);-- assert.ok(Array.isArray(tracker.getBotAnalysis(7).topBotIPs));-- tracker.destroy();-- db.close();-- }---- // ── Prune boundaries, rollups untouched ──-- {-- const db = newDb('prune.db');-- const nowMs = Date.parse('2026-09-23T12:00:00Z');-- seedLegacy(db, nowMs);-- const totals = retention.rollupTotals(db);-- const part = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, maxBatches: 1, now: () => nowMs });-- assert.deepStrictEqual([part.deleted, part.complete], [4, false]);-- const rest = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, now: () => nowMs });-- assert.strictEqual(rest.deleted, 7);-- assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all().map((p) => p.split('?')[0]), ['/edge-exact', '/api/info/abc123def']);-- assert.deepStrictEqual(retention.rollupTotals(db), totals);-- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);---- // Scrub the survivors and the rollup lists.-- await retention.scrubEvents(db, { batchSize: 1 });-- retention.scrubRollups(db);-- assert.deepStrictEqual(retention.rollupTotals(db), totals);-- const rows = db.prepare('SELECT * FROM analytics_events ORDER BY created_at').all();-- assert.deepStrictEqual(rows.map((r) => r.path), ['/edge-exact', '/api/info/:id']);-- assert.deepStrictEqual(rows.map((r) => r.authenticated), [0, 1]);-- assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()), [{ path: '/watch/:param', cnt: 5 }]);-- const all = dumpAll(db);-- for (const needle of ['198.51.100.', 'Lyon', 'Oslo', 'q=x', 'dQw4w9WgXcQ', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);-- db.close();-- }---- // ── CLI ──-- {-- const out = [];-- const log = (l) => out.push(l);-- const make = (name) => {-- const file = path.join(tmp, name, 'data', 'analytics.db');-- fs.mkdirSync(path.dirname(file), { recursive: true });-- const db = new Database(file);-- db.pragma('journal_mode = WAL');-- seedLegacy(db, Date.now() + 120000); // edge rows clear of the CLI's own cutoff: 10 older rows-- db.pragma('wal_checkpoint(TRUNCATE)');-- db.close();-- return file;-- };-- const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');-- const yt = make('yt');-- const food = make('food');-- const hashes = [sha(yt), sha(food)];---- assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--scrub'], log), 0);-- assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'dry run changed a file');-- assert.ok(out.some((l) => /prune\s+10 rows/.test(l)), out.join('\n'));-- assert.ok(out.some((l) => /dry run: nothing changed/.test(l)));-- assert.strictEqual(await cli.main(['--db', yt, '--apply'], log), 2);-- assert.strictEqual(await cli.main(['--db', yt, '--apply', '--no-backup', '--days', '45'], log), 2);-- assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'a refused run changed a file');++ // ── Prune boundaries: strictly-older raw rows and day hashes go; rollups stay ──++ {++ const db = await testDb(path.join(tmp, 'prune'));++ const nowMs = Date.parse('2026-09-23T12:00:00Z');++ const ins = db.prepare("INSERT INTO analytics_events (service, event_type, path, created_at) VALUES ('openvibe-yt', 'pageview', ?, ?)");++ for (let i = 0; i < 5; i++) await ins.run(`/old/${i}`, sqlTime(nowMs - 40 * DAY));++ await ins.run('/recent', sqlTime(nowMs - 20 * DAY));++ const vd = db.prepare('INSERT INTO analytics_visitor_days (service, day, hour, vhash) VALUES (?, ?, ?, ?)');++ await vd.run('openvibe-yt', dayOf(nowMs - 2 * DAY), dayOf(nowMs - 2 * DAY) + 'T00', 'a'.repeat(16));++ await vd.run('openvibe-yt', dayOf(nowMs), dayOf(nowMs) + 'T00', 'b'.repeat(16));++ const ds = db.prepare('INSERT INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)');++ await ds.run('openvibe-yt', dayOf(nowMs - 2 * DAY), 'deadbeef');++ await ds.run('openvibe-yt', dayOf(nowMs), 'cafebabe');++ await db.prepare("INSERT INTO analytics_daily (service, date, pageviews) VALUES ('openvibe-yt', ?, 100)").run(dayOf(nowMs - 40 * DAY));-- const bdir = path.join(tmp, 'backups');-- assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--scrub', '--backup', bdir], log), 0, out.join('\n'));-- for (const [name, file] of [['yt', yt], ['food', food]]) {-- const b = new Database(path.join(bdir, `${name}.analytics.db`), { readonly: true });-- assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13);-- b.close();-- const db = new Database(file, { readonly: true });-- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 3);-- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events WHERE ip IS NOT NULL OR user_id IS NOT NULL OR city IS NOT NULL').pluck().get(), 0);-- assert.strictEqual(db.prepare('SELECT SUM(pageviews) FROM analytics_daily').pluck().get(), 100);-- db.close();++ const r = await pruneRawEventsPg(db, { days: 30, now: () => nowMs });++ assert.strictEqual(r.removed, 5, 'only the strictly-older raw rows');++ const left = (await db.prepare('SELECT path FROM analytics_events ORDER BY path').all()).map((x) => x.path);++ assert.deepStrictEqual(left, ['/recent']);++ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_visitor_days').get()).n), 1, "today's visitor day is kept");++ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_day_salts').get()).n), 1, "today's salt is kept");++ assert.strictEqual(Number((await db.prepare('SELECT pageviews FROM analytics_daily WHERE date = ?').get(dayOf(nowMs - 40 * DAY))).pageviews), 100, 'rollups are never pruned');}-- // Existing backup targets are refused.-- assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--backup', bdir], log), 2);-- }-- fs.rmSync(tmp, { recursive: true, force: true });-- console.log('analytics ADR-021: tracked request, templates, rollups, prune, scrub, CLI ok');++ console.log('analytics ADR-021 (PostgreSQL): tracked request, templates, rollups, prune ok');++ } finally {++ await closeAllTestDbs();++ fs.rmSync(tmp, { recursive: true, force: true });++ }})().catch((e) => { console.error(e); process.exit(1); });@@@ -16,7 +16,6 @@ const { dep } = require('./deps');const express = dep('express');const cookieParser = dep('cookie-parser');const multer = dep('multer');--const Database = dep('better-sqlite3'); // the job store is still SQLite (Phases 2–3 not done)const contracts = dep('openvibe-contracts');const metrics = dep('openvibe-shared/metrics');const guardLib = require('../guard');@@@ -27,6 +26,7 @@ const { createGuardStore } = require('../guard/store');const { ipBucket } = require('../guard/ip');const sniff = require('../guard/sniff');const jobs = require('../jobs');++const { testDb, closeAllTestDbs } = require('./testdb');const ISSUER = 'https://openvibe.network';const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });@@@ -494,10 +494,10 @@ async function serve(app) {// ── The job store's bounds: queued jobs and disk (system.busy) through the guard ──for (const mode of ['enforce', 'report']) {const dataDir = tmp();-- const db = new Database(':memory:');++ const db = await testDb(dataDir);const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, maxQueued: 2, diskBudgetBytes: 1_000_000, log: quiet });system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });-- system.start();++ await system.start();const g = makeGuard({ mode });const app = express();app.set('trust proxy', TRUST_PROXY);@@@ -509,7 +509,7 @@ async function serve(app) {try {assert.strictEqual((await submit()).status, 202);assert.strictEqual((await submit()).status, 202);-- assert.strictEqual(system.busy().reason, 'queue');++ assert.strictEqual((await system.busy()).reason, 'queue');const r = await submit();if (mode === 'enforce') {assert.strictEqual(r.status, 503, 'a third queued job is refused');@@@ -519,15 +519,15 @@ async function serve(app) {assert.strictEqual(r.status, 202, 'report mode accepts it');assert.ok(g.store.abuseRows().some(x => x.reason === 'busy.queue' && x.enforced === 0));}-- } finally { await s.close(); g.close(); system.stop(); db.close(); fs.rmSync(dataDir, { recursive: true, force: true }); }++ } finally { await s.close(); g.close(); system.stop(); fs.rmSync(dataDir, { recursive: true, force: true }); }}// Unfinished browser-session jobs are bounded per address too: a new cookie is no new allowance.for (const mode of ['enforce', 'report']) {const dataDir = tmp();-- const db = new Database(':memory:');++ const db = await testDb(dataDir);const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, maxActivePerOwner: 1, log: quiet });system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });-- system.start();++ await system.start();assert.strictEqual(system.bounds.maxActivePerAddress, 3, '3 × the per-owner bound by default');const g = makeGuard({ mode });const app = express();@@@ -549,24 +549,25 @@ async function serve(app) {for (let i = 0; i < 3; i++) codes.push((await submit('198.51.100.60')).status); // a fresh cookie each timeassert.deepStrictEqual(codes, mode === 'enforce' ? [202, 202, 429] : [202, 202, 202], `${mode}: three per address`);assert.strictEqual((await submit('198.51.100.61')).status, 202, 'another address is fine');-- const row = db.prepare('SELECT ip_key FROM tool_jobs LIMIT 1').get();++ const row = await db.prepare('SELECT ip_key FROM tool_jobs LIMIT 1').get();assert.match(row.ip_key, /^ip:[0-9a-f]{32}$/, 'the job keeps the hashed key, never the address');if (mode === 'report') assert.ok(g.store.abuseRows().some(x => x.reason === 'jobs.address' && x.enforced === 0));-- } finally { await s.close(); g.close(); system.stop(); db.close(); fs.rmSync(dataDir, { recursive: true, force: true }); }++ } finally { await s.close(); g.close(); system.stop(); fs.rmSync(dataDir, { recursive: true, force: true }); }}{const dataDir = tmp();-- const db = new Database(':memory:');++ const db = await testDb(dataDir);const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, diskBudgetBytes: 1000, log: quiet });system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });-- system.start();++ await system.start();await system.refreshDisk();-- assert.strictEqual(system.busy(), null);++ assert.strictEqual(await system.busy(), null);await system.submit({ owner: 'session:x', type: 'test.noop', input: {}, files: [{ buffer: Buffer.alloc(1500), name: 'a.bin', mime: 'application/octet-stream', size: 1500 }] });-- assert.strictEqual(system.busy().reason, 'disk', 'accepted inputs count against the disk budget at once');++ assert.strictEqual((await system.busy()).reason, 'disk', 'accepted inputs count against the disk budget at once');assert.strictEqual((await system.refreshDisk()) >= 1500, true, 'and the walk finds them');-- system.stop(); db.close(); fs.rmSync(dataDir, { recursive: true, force: true });++ system.stop(); fs.rmSync(dataDir, { recursive: true, force: true });}++ await closeAllTestDbs();console.log('guard: callers (/64, one loopback hop, services by token, aud-checked users, sessions, sandbox), bucket math + cost, session sharing, report vs enforce, headers + problem+json, two processes one Valkey quota + outage fallback, sniffing, target throttle, port cap, abuse log + prune + metric, semaphore, job bounds (queue, disk, per address), challenge hook, Origin check (CSRF): all checks passed');})().catch((err) => { console.error(err); process.exit(1); });@@@ -1,16 +1,16 @@'use strict';// tools.job.* → OpenVibe.Events (roadmap Wave 11): every transition writes exactly one event to the--// jobs database's outbox in the transaction that records it; a rolled-back transition writes none;--// sandbox jobs and cancellations write none; payloads carry no input, file name, output data or--// browser session and validate against openvibe-contracts; the relay posts to EVENTS_URL with the--// tools service token; and without EVENTS_URL nothing is set up at all.++// tools database's outbox (openvibe-sdk createPgOutbox) in the transaction that records it; a rolled-back++// transition writes none; sandbox jobs and cancellations write none; payloads carry no input, file name,++// output data or browser session and validate against openvibe-contracts; the relay posts to EVENTS_URL++// with the tools service token; and without EVENTS_URL nothing is set up at all.const assert = require('assert');const fs = require('fs');const os = require('os');const path = require('path');const { dep } = require('./deps');++const { testDb, closeAllTestDbs } = require('./testdb');--const Database = dep('better-sqlite3');const contracts = dep('openvibe-contracts');const sdk = dep('openvibe-sdk');const jobs = require('../jobs');@@@ -100,17 +100,19 @@ function fakeNetwork() {const ENV = { EVENTS_URL: 'http://events.test/', OV_OAUTH_CLIENT_SECRET: 'tools-secret', OV_NETWORK_INTERNAL_URL: 'http://network.test' };--function satellite(dir, { wrap, concurrency = 1, net } = {}) {++async function satellite(dir, { wrap, concurrency = 1, net } = {}) {fs.mkdirSync(dir, { recursive: true });-- const db = new Database(path.join(dir, 'jobs.db'));++ const db = await testDb(dir);const { outbox, reason } = outboxFromEnv({ db, sdk, env: ENV, fetch: net.fetch, log: silent });assert.ok(outbox, `outbox configured (${reason})`);++ await outbox.ready;const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, concurrency, outbox: wrap ? wrap(outbox) : outbox, progressThrottleMs: 0, log: silent });define(system);-- system.start();-- const envelopes = (jobId) => db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all().map(r => JSON.parse(r.envelope))++ await system.start();++ const envelopes = async (jobId) => (await db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all())++ .map(r => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope)).filter(e => !jobId || e.subject.id === jobId);-- return { db, system, outbox, envelopes, close() { system.stop(); outbox.stop(); db.close(); } };++ return { db, system, outbox, envelopes, close: async () => { system.stop(); await outbox.stop(); } };}const types = (list) => list.map(e => e.event_type);@@@ -135,28 +137,28 @@ function noLeaks(list, extra = []) {}}}--const settled = (s, id, state) => until(() => { const r = s.system.get(id); return r && r.state === state && r; }, `${id} ${state}`);++const settled = (s, id, state) => until(async () => { const r = await s.system.get(id); return r && r.state === state && r; }, `${id} ${state}`);(async () => {const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-job-events-'));try {// ── Inert without EVENTS_URL ─────────────────────────────{-- const db = new Database(':memory:');++ const db = await testDb(path.join(root, 'inert-db'));assert.deepStrictEqual(outboxFromEnv({ db, sdk, env: {} }), { outbox: null, reason: 'EVENTS_URL is not set' });assert.strictEqual(outboxFromEnv({ db, sdk, env: { EVENTS_URL: 'http://events.test' } }).outbox, null, 'no secret, no outbox');assert.strictEqual(outboxFromEnv({ db, sdk, env: { ...ENV, EVENTS_PUBLISH: 'off' } }).outbox, null, 'EVENTS_PUBLISH=off');assert.strictEqual(outboxFromEnv({ db, sdk: null, env: ENV }).outbox, null, 'no sdk, no outbox');const dir = path.join(root, 'inert');-- fs.mkdirSync(dir);++ fs.mkdirSync(dir, { recursive: true });const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, log: silent });define(system);-- system.start();++ await system.start();const { job } = await system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'x' } });-- await until(() => system.get(job.id).state === 'succeeded', 'inert job');-- assert.strictEqual(db.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE name = 'event_outbox'").get().n, 0, 'no outbox table without EVENTS_URL');-- assert.deepStrictEqual(system.stats().events, { enabled: false });-- system.stop(); db.close();++ await until(async () => (await system.get(job.id)).state === 'succeeded', 'inert job');++ assert.strictEqual((await db.prepare("SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_name = 'event_outbox'").get()).n, 0, 'no outbox table without EVENTS_URL');++ assert.deepStrictEqual((await system.stats()).events, { enabled: false });++ system.stop();}const net = fakeNetwork();@@@ -164,18 +166,18 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retulet failOn = null;const wrap = (outbox) => ({...outbox,-- enqueue(envelope, opts) {-- const env = outbox.enqueue(envelope, opts);++ async enqueue(t, envelope, opts) {++ const env = await outbox.enqueue(t, envelope, opts);if (failOn === envelope.event_type) throw new Error('disk I/O error (test)');return env;},});-- let s = satellite(path.join(root, 'a'), { net, wrap });++ let s = await satellite(path.join(root, 'a'), { net, wrap });// ── A signed-in person's job: created, started, succeeded — once each ──let { job } = await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: SECRET_INPUT }, files: [{ buffer: Buffer.from('hello'), name: SECRET_FILE, mime: 'text/plain' }] });await settled(s, job.id, 'succeeded');-- let list = s.envelopes(job.id);++ let list = await s.envelopes(job.id);assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);list.forEach(checkContracts);noLeaks(list);@@@ -195,12 +197,20 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retuassert.strictEqual(done.result.files[0].storage, 'local');assert.strictEqual(done.result.files[0].media, null);assert.strictEqual(done.result.files[0].size, 5);-- assert.deepStrictEqual(s.system.stats().events.enabled, true);++ assert.deepStrictEqual((await s.system.stats()).events.enabled, true);++++ // ── A stream that dropped mid-job resumes with Last-Event-ID: the log from there is exactly the++ // tail of the whole sequence, seq for seq — no gap, no repeat (the SSE reconnect path) ──++ const log = await s.system.eventsAfter(job.id, 0);++ assert.deepStrictEqual(log.map(e => e.event), ['job.queued', 'job.running', 'job.succeeded']);++ const resumed = await s.system.eventsAfter(job.id, log[0].seq);++ assert.deepStrictEqual(resumed.map(e => e.event), ['job.running', 'job.succeeded'], 'exactly the events after the last id the client saw');++ assert.deepStrictEqual(resumed.map(e => e.seq), log.slice(1).map(e => e.seq), 'the same per-job seq values, in order');// ── A failure whose message names the input file: failed once, name and path scrubbed ──({ job } = await s.system.submit({ owner: `user:${USER}`, type: 'test.boom', input: { text: SECRET_INPUT }, files: [{ buffer: Buffer.from('x'), name: SECRET_FILE, mime: 'text/plain' }] }));await settled(s, job.id, 'failed');-- list = s.envelopes(job.id);++ list = await s.envelopes(job.id);assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.failed']);list.forEach(checkContracts);noLeaks(list);@@@ -214,7 +224,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu// ── A browser session's job: owner null, the service acts, the session never appears ──({ job } = await s.system.submit({ owner: SESSION, type: 'test.upper', input: { text: SECRET_INPUT } }));await settled(s, job.id, 'succeeded');-- list = s.envelopes(job.id);++ list = await s.envelopes(job.id);assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);list.forEach(checkContracts);noLeaks(list);@@@ -223,7 +233,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu// ── Principals: a service and a production app are subjects ──({ job } = await s.system.submit({ owner: 'svc:live', type: 'test.upper', input: {} }));await settled(s, job.id, 'succeeded');-- list = s.envelopes(job.id);++ list = await s.envelopes(job.id);list.forEach(checkContracts);assert.deepStrictEqual(list[0].actor, { type: 'service', id: 'live' });assert.deepStrictEqual(list[0].payload.owner, { type: 'service', id: 'live' });@@@ -231,90 +241,90 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retuassert.strictEqual(ownerRef(contracts, 'app:nope'), null);// ── Sandbox app jobs are never announced ─────────────────-- const before = s.envelopes().length;++ const before = (await s.envelopes()).length;({ job } = await s.system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: 'sandbox' }, env: 'sandbox' }));await settled(s, job.id, 'succeeded');const sandboxFail = (await s.system.submit({ owner: `app:${APP}`, type: 'test.boom', files: [{ buffer: Buffer.from('x'), name: 'a.txt' }], env: 'sandbox' })).job;await settled(s, sandboxFail.id, 'failed');-- assert.strictEqual(s.envelopes(job.id).length + s.envelopes(sandboxFail.id).length, 0, 'sandbox jobs emit nothing');-- assert.strictEqual(s.envelopes().length, before);++ assert.strictEqual((await s.envelopes(job.id)).length + (await s.envelopes(sandboxFail.id)).length, 0, 'sandbox jobs emit nothing');++ assert.strictEqual((await s.envelopes()).length, before);// ── Retry: a new created (retry_of) + its run; the failed job gets nothing new ──flaky.add('t1');const first = (await s.system.submit({ owner: `user:${USER}`, type: 'test.flaky', input: { tag: 't1' }, files: [{ buffer: Buffer.from('abc'), name: SECRET_FILE }] })).job;await settled(s, first.id, 'failed');-- assert.strictEqual(s.envelopes(first.id).at(-1).payload.retryable, true);++ assert.strictEqual((await s.envelopes(first.id)).at(-1).payload.retryable, true);flaky.delete('t1');-- const retried = s.system.retry(first.id).job;++ const retried = (await s.system.retry(first.id)).job;await settled(s, retried.id, 'succeeded');-- assert.deepStrictEqual(types(s.envelopes(first.id)), ['tools.job.created', 'tools.job.started', 'tools.job.failed'], 'the retried job gets no new event');-- list = s.envelopes(retried.id);++ assert.deepStrictEqual(types(await s.envelopes(first.id)), ['tools.job.created', 'tools.job.started', 'tools.job.failed'], 'the retried job gets no new event');++ list = await s.envelopes(retried.id);assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);list.forEach(checkContracts);noLeaks(list);assert.strictEqual(list[0].payload.retry_of, first.id);-- assert.strictEqual(s.system.retry(first.id).replayed, true);-- assert.strictEqual(s.envelopes(retried.id).length, 3, 'a replayed retry creates no event');++ assert.strictEqual((await s.system.retry(first.id)).replayed, true);++ assert.strictEqual((await s.envelopes(retried.id)).length, 3, 'a replayed retry creates no event');// ── An Idempotency-Key replay creates no job and no event ──const k1 = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'k' }, idempotencyKey: 'key-00000001' })).job;await settled(s, k1.id, 'succeeded');const k2 = await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'k' }, idempotencyKey: 'key-00000001' });assert.strictEqual(k2.replayed, true);-- assert.strictEqual(s.envelopes(k1.id).length, 3);++ assert.strictEqual((await s.envelopes(k1.id)).length, 3);// ── Cancelled jobs are not announced (neither queued nor running) ──const blocker = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { gate: 'g1' } })).job;await settled(s, blocker.id, 'running');const queued = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'q' } })).job;-- assert.strictEqual(s.system.cancel(queued.id).changed, true);-- assert.deepStrictEqual(types(s.envelopes(queued.id)), ['tools.job.created']);-- s.system.cancel(blocker.id);++ assert.strictEqual((await s.system.cancel(queued.id)).changed, true);++ assert.deepStrictEqual(types(await s.envelopes(queued.id)), ['tools.job.created']);++ await s.system.cancel(blocker.id);await settled(s, blocker.id, 'cancelled');-- assert.deepStrictEqual(types(s.envelopes(blocker.id)), ['tools.job.created', 'tools.job.started']);++ assert.deepStrictEqual(types(await s.envelopes(blocker.id)), ['tools.job.created', 'tools.job.started']);// ── Rollbacks: a failed outbox write undoes the transition, and no event is left ──-- const count = () => s.envelopes().length;-- let n = count();++ const count = async () => (await s.envelopes()).length;++ let n = await count();failOn = 'tools.job.created';await assert.rejects(s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'rollback' } }), /disk I\/O error/);failOn = null;-- assert.strictEqual(count(), n, 'rolled-back submit: no event');-- assert.strictEqual(s.db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE input_json LIKE '%rollback%'").get().n, 0, 'rolled-back submit: no job');++ assert.strictEqual(await count(), n, 'rolled-back submit: no event');++ assert.strictEqual((await s.db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE input_json LIKE '%rollback%'").get()).n, 0, 'rolled-back submit: no job');// claim(): the job stays queued and nothing says it started.failOn = 'tools.job.started';const stuck = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'claim' } })).job;await sleep(50);-- assert.strictEqual(s.system.get(stuck.id).state, 'queued', 'rolled-back claim: still queued');-- assert.strictEqual(s.system.get(stuck.id).attempts, 0);-- assert.deepStrictEqual(types(s.envelopes(stuck.id)), ['tools.job.created'], 'rolled-back claim: no started event');++ assert.strictEqual((await s.system.get(stuck.id)).state, 'queued', 'rolled-back claim: still queued');++ assert.strictEqual((await s.system.get(stuck.id)).attempts, 0);++ assert.deepStrictEqual(types(await s.envelopes(stuck.id)), ['tools.job.created'], 'rolled-back claim: no started event');failOn = null;// The next kick (any submit) starts it: exactly one started.const kicker = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'kick' } })).job;await settled(s, stuck.id, 'succeeded');await settled(s, kicker.id, 'succeeded');-- assert.deepStrictEqual(types(s.envelopes(stuck.id)), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);++ assert.deepStrictEqual(types(await s.envelopes(stuck.id)), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);// finish(): the job stays running (recovered on the next start) and nothing says it ended.failOn = 'tools.job.succeeded';const unfinished = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'finish' } })).job;-- await until(() => s.envelopes(unfinished.id).length === 2 && s.system.stats().executing === 0, 'finish attempt');++ await until(async () => (await s.envelopes(unfinished.id)).length === 2 && (await s.system.stats()).executing === 0, 'finish attempt');await sleep(30);-- assert.strictEqual(s.system.get(unfinished.id).state, 'running', 'rolled-back finish: still running');-- assert.deepStrictEqual(types(s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started']);++ assert.strictEqual((await s.system.get(unfinished.id)).state, 'running', 'rolled-back finish: still running');++ assert.deepStrictEqual(types(await s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started']);failOn = null;// ── Restart: the stuck job is re-queued (not announced) and started again (announced again);// a running onRestart:'fail' job fails once with tools.job.interrupted ──const fragile = (await s.system.submit({ owner: `user:${USER}`, type: 'test.fragile', input: { gate: 'never' } })).job;await settled(s, fragile.id, 'running');-- s.close();-- s = satellite(path.join(root, 'a'), { net });++ await s.close();++ s = await satellite(path.join(root, 'a'), { net });await settled(s, unfinished.id, 'succeeded');-- assert.deepStrictEqual(types(s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started', 'tools.job.started', 'tools.job.succeeded']);-- assert.strictEqual(s.envelopes(unfinished.id)[2].payload.attempts, 2);-- const fr = s.envelopes(fragile.id);++ assert.deepStrictEqual(types(await s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started', 'tools.job.started', 'tools.job.succeeded']);++ assert.strictEqual((await s.envelopes(unfinished.id))[2].payload.attempts, 2);++ const fr = await s.envelopes(fragile.id);assert.deepStrictEqual(types(fr), ['tools.job.created', 'tools.job.started', 'tools.job.failed']);fr.forEach(checkContracts);assert.strictEqual(fr[2].payload.error.code, 'tools.job.interrupted');@@@ -322,21 +332,21 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retuassert.ok(fr[2].payload.started_at);// ── Every event so far: valid, and never a secret ──-- const all = s.envelopes();++ const all = await s.envelopes();all.forEach(checkContracts);noLeaks(all);assert.strictEqual(new Set(all.map(e => e.event_id)).size, all.length, 'unique event ids');// ── The relay: POST EVENTS_URL/api/v1/events with the tools token for openvibe.events ──-- assert.strictEqual(s.outbox.pending(), all.length);++ assert.strictEqual(await s.outbox.pending(), all.length);const r = await s.outbox.flush();assert.strictEqual(r.sent, all.length);-- assert.strictEqual(s.outbox.pending(), 0);++ assert.strictEqual(await s.outbox.pending(), 0);const published = net.calls.publish.flatMap(c => c.events);assert.deepStrictEqual(published.map(e => e.event_id), all.map(e => e.event_id), 'published in order, once');assert.ok(net.calls.publish.every(c => c.authorization === 'Bearer events-token'));assert.deepStrictEqual(net.calls.token[0], { grant_type: 'client_credentials', client_id: 'tools', client_secret: 'tools-secret', audience: 'openvibe.events', scope: 'events.event.publish' });-- s.close();++ await s.close();// ── Payload builder: a result in Media is { media_id, role } only ──const row = {@@@ -353,6 +363,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retuconsole.log('job events (tools.job.* outbox): all checks passed');} finally {++ await closeAllTestDbs();fs.rmSync(root, { recursive: true, force: true });}})().catch((err) => { console.error(err); process.exit(1); });@@@ -4,14 +4,14 @@// failures by code with their job id and submit trace id; nobody else's jobs and no cancelled job// count; each closed hour is written to the outbox once as tools.usage.recorded (valid against// openvibe-contracts, no owner, input or file name); a job ending in an hour already sent re-sends it--// as revision 2; without an outbox nothing is counted.++// as revision 2; without an outbox nothing is counted. On PostgreSQL (plan T8): openvibe-sdk/db.const assert = require('assert');const fs = require('fs');const os = require('os');const path = require('path');const { dep } = require('./deps');++const { testDb, closeAllTestDbs } = require('./testdb');--const Database = dep('better-sqlite3');const contracts = dep('openvibe-contracts');const sdk = dep('openvibe-sdk');const jobs = require('../jobs');@@@ -60,29 +60,30 @@ function define(system) {try {// ── Without an outbox: nothing counted, no table ──{-- const db = new Database(':memory:');++ const db = await testDb(path.join(root, 'inert-db'));const dir = path.join(root, 'inert');-- fs.mkdirSync(dir);++ fs.mkdirSync(dir, { recursive: true });const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, log: silent });define(system);-- system.start();++ await system.start();const { job } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: 'x' }, project: PRJ });-- await until(() => system.get(job.id).state === 'succeeded', 'inert job');-- assert.strictEqual(db.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE name = 'tool_job_usage'").get().n, 0);-- assert.deepStrictEqual(system.stats().usage, { enabled: false });-- system.stop(); db.close();++ await until(async () => (await system.get(job.id)).state === 'succeeded', 'inert job');++ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM tool_job_usage').get()).n), 0, 'inert: nothing counted');++ assert.deepStrictEqual((await system.stats()).usage, { enabled: false });++ system.stop();}const dir = path.join(root, 'a');-- fs.mkdirSync(dir);-- const db = new Database(path.join(dir, 'jobs.db'));++ fs.mkdirSync(dir, { recursive: true });++ const db = await testDb(dir);const { outbox } = outboxFromEnv({ db, sdk, env: ENV, fetch: fakeFetch, log: silent });++ await outbox.ready;const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, concurrency: 1, outbox, progressThrottleMs: 0, log: silent });define(system);-- system.start();++ await system.start();const run = async (o, state) => {const { job } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: SECRET_INPUT }, ...o });-- await until(() => { const r = system.get(job.id); return r && r.state === state; }, `${job.id} ${state}`);++ await until(async () => { const r = await system.get(job.id); return r && r.state === state; }, `${job.id} ${state}`);return job.id;};@@@ -93,13 +94,13 @@ function define(system) {await run({ project: PRJ, env: 'sandbox' }, 'succeeded');await run({ project: PRJ, tool: 'image-resize' }, 'succeeded');await run({ project: null, owner: `user:${USER}` }, 'succeeded'); // a person's job: no project, not counted-- assert.strictEqual(system.get(failedId).trace_id, TRACE, 'the submit request\'s trace id is kept with the job');++ assert.strictEqual((await system.get(failedId)).trace_id, TRACE, 'the submit request\'s trace id is kept with the job');const { job: held } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { hold: true }, project: PRJ });-- await until(() => system.get(held.id).state === 'running', 'held job running');-- system.cancel(held.id);-- await until(() => system.get(held.id).state === 'cancelled', 'held job cancelled');++ await until(async () => (await system.get(held.id)).state === 'running', 'held job running');++ await system.cancel(held.id);++ await until(async () => (await system.get(held.id)).state === 'cancelled', 'held job cancelled');-- const rows = db.prepare('SELECT * FROM tool_job_usage ORDER BY capability, dimension, env').all();++ const rows = await db.prepare('SELECT * FROM tool_job_usage ORDER BY capability, dimension, env').all();const view = rows.map(r => [r.capability, r.dimension, r.env, r.quantity, r.errors]);assert.deepStrictEqual(view, [['tools.job.create', 'test.boom', 'production', 1, 1],@@@ -111,18 +112,18 @@ function define(system) {assert.deepStrictEqual(JSON.parse(boom.error_codes), { 'tools.test.unreadable': 1 });const [sample] = JSON.parse(boom.samples);assert.deepStrictEqual([sample.code, sample.status, sample.trace_id, sample.ref], ['tools.test.unreadable', 422, TRACE, failedId]);-- assert.strictEqual(keyOf({ ...system.get(failedId), state: 'cancelled' }), null);++ assert.strictEqual(keyOf({ ...(await system.get(failedId)), state: 'cancelled' }), null);// ── Nothing leaves before the hour closes ──-- const before = db.prepare("SELECT COUNT(*) AS n FROM event_outbox WHERE envelope LIKE '%tools.usage.recorded%'").get().n;-- assert.strictEqual(before, 0);-- assert.strictEqual(system.flushUsage().queued, 0);++ const before = (await db.prepare("SELECT COUNT(*) AS n FROM event_outbox WHERE envelope->>'event_type' = 'tools.usage.recorded'").get()).n;++ assert.strictEqual(Number(before), 0);++ assert.strictEqual((await system.flushUsage()).queued, 0);// ── After the hour: one tools.usage.recorded per rollup, once ──-- const later = rows[0].window_start + HOUR_MS + 2 * 60 * 1000;-- assert.strictEqual(system.flushUsage(later).queued, 4);-- assert.strictEqual(system.flushUsage(later).queued, 0, 'sent once');-- const sent = db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all().map(r => JSON.parse(r.envelope)).filter(e => e.event_type === 'tools.usage.recorded');++ const later = Number(rows[0].window_start) + HOUR_MS + 2 * 60 * 1000;++ assert.strictEqual((await system.flushUsage(later)).queued, 4);++ assert.strictEqual((await system.flushUsage(later)).queued, 0, 'sent once');++ const sent = (await db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all()).map(r => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope)).filter(e => e.event_type === 'tools.usage.recorded');assert.strictEqual(sent.length, 4);for (const env of sent) {const v = contracts.validate('tools.usage.recorded@1', env.payload);@@@ -138,16 +139,17 @@ function define(system) {assert.deepStrictEqual([failedRollup.quantity, failedRollup.errors, failedRollup.samples[0].ref], [1, 1, failedId]);// ── A job that ends in an hour already sent: re-sent as revision 2 with the new totals ──-- db.transaction(() => system.usage.finished({ ...system.get(failedId), state: 'succeeded' }))();-- assert.strictEqual(system.flushUsage(later).queued, 1);-- const resent = db.prepare('SELECT envelope FROM event_outbox ORDER BY id DESC LIMIT 1').get();-- const p = JSON.parse(resent.envelope).payload;++ await db.tx(async () => system.usage.finished({ ...(await system.get(failedId)), state: 'succeeded' }));++ assert.strictEqual((await system.flushUsage(later)).queued, 1);++ const resent = await db.prepare('SELECT envelope FROM event_outbox ORDER BY id DESC LIMIT 1').get();++ const p = (typeof resent.envelope === 'string' ? JSON.parse(resent.envelope) : resent.envelope).payload;assert.deepStrictEqual([p.dimension, p.quantity, p.errors, p.revision], ['test.boom', 2, 1, 2]);-- assert.deepStrictEqual(system.stats().usage, { pending: 0, invalid: 0, last_invalid: null });++ assert.deepStrictEqual((await system.stats()).usage, { pending: 0, invalid: 0, last_invalid: null });-- system.stop(); outbox.stop(); db.close();++ system.stop(); await outbox.stop();console.log('job usage: all checks passed');} finally {++ await closeAllTestDbs();fs.rmSync(root, { recursive: true, force: true });}})().catch((e) => { console.error(e); process.exit(1); });@@@ -14,9 +14,9 @@ const { dep } = require('./deps');const express = dep('express');const cookieParser = dep('cookie-parser');--const Database = dep('better-sqlite3');const contracts = dep('openvibe-contracts');const jobs = require('../jobs');++const { testDb: dbFor, closeAllTestDbs: closeDbs, quiet } = require('./testdb');const ISSUER = 'https://openvibe.network';const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });@@@ -114,11 +114,11 @@ function fakeMedia() {}const fakeTokens = { authHeaders: async () => ({ Authorization: 'Bearer media-token' }), invalidate() {} };--// ── A satellite: express + the job routes over one SQLite file ──--function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10 } = {}) {++// ── A satellite: express + the job routes over the shared `tools` database ──++async function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10 } = {}) {fs.mkdirSync(dir, { recursive: true });-- const db = new Database(path.join(dir, 'jobs.db'));-- const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir: dir, concurrency, media, maxActivePerOwner, progressThrottleMs: 0, log: { log() {}, warn() {}, error() {} } });++ const db = await dbFor(dir);++ const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir: dir, concurrency, media, maxActivePerOwner, progressThrottleMs: 0, log: quiet });define(system);const app = express();app.use(cookieParser());@@@ -128,11 +128,11 @@ function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10const upload = multer({ dest: path.join(dir, 'tmp') });const resolveOwner = jobs.createOwnerResolver({ contracts, getPublicKey: () => publicKey, issuer: ISSUER, secureCookie: false });jobs.mountJobRoutes(app, { system, contracts, resolveOwner, receive: upload.any() });-- system.start();++ await system.start();return new Promise(resolve => {const server = app.listen(0, '127.0.0.1', () => resolve({system, db, base: `http://127.0.0.1:${server.address().port}`,-- close: () => new Promise(r => { system.stop(); server.closeAllConnections(); server.close(() => { db.close(); r(); }); }),++ close: () => new Promise(r => { system.stop(); server.closeAllConnections(); server.close(() => r()); }),}));});}@@@ -246,7 +246,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}assert.strictEqual((await guest1(`/api/v1/jobs/${guestJob}`)).status, 200, 'the session that made it can reattach');assert.strictEqual((await guest2(`/api/v1/jobs/${guestJob}`)).status, 404, 'another browser cannot');assert.strictEqual((await alice(`/api/v1/jobs/${guestJob}`)).status, 404, 'nor can a signed-in stranger');-- assert.ok(s.db.prepare('SELECT owner FROM tool_jobs').all().every(x => /^(user:usr_[0-9A-Z]{26}|session:[0-9a-f]{40})$/.test(x.owner)), 'owners are canonical subjects or hashed sessions, never raw cookies');++ assert.ok((await s.db.prepare('SELECT owner FROM tool_jobs').all()).every(x => /^(user:usr_[0-9A-Z]{26}|session:[0-9a-f]{40})$/.test(x.owner)), 'owners are canonical subjects or hashed sessions, never raw cookies');// ── SSE: events carry ids; Last-Event-ID resumes after them ──const all = await readSse(`${s.base}/api/v1/jobs/${first}/events`, { headers: { 'x-test-user': USER_A } });@@@ -267,6 +267,19 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}const live = await streaming;assert.strictEqual(live.events[live.events.length - 1].event, 'job.succeeded');assert.strictEqual(live.events[live.events.length - 1].data.result.files.length, 1, 'the terminal event carries the result');++ // A forced disconnect mid-stream: the reader drops after the first frame it sees, then reconnects++ // with the last id it got. The two reads together are the exact log — no gap, no repeat.++ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'cut', gate: 'cut' } }));++ const cutId = r.body.id;++ await until(async () => (await alice(`/api/v1/jobs/${cutId}`)).body.state === 'running', 'disconnect job running');++ const head = await readSse(`${s.base}/api/v1/jobs/${cutId}/events`, { headers: { 'x-test-user': USER_A }, stop: (ev) => ev.length >= 1 });++ assert.ok(head.events.length >= 1, 'the stream was cut mid-job');++ gate('cut').open();++ const tail = await readSse(`${s.base}/api/v1/jobs/${cutId}/events`, { headers: { 'x-test-user': USER_A, 'last-event-id': String(head.events[head.events.length - 1].id) } });++ const joined = [...head.events, ...tail.events];++ const whole = await readSse(`${s.base}/api/v1/jobs/${cutId}/events`, { headers: { 'x-test-user': USER_A } });++ assert.deepStrictEqual(joined.map(e => e.event), whole.events.map(e => e.event), 'the exact sequence across the disconnect');++ assert.deepStrictEqual(joined.map(e => e.id), whole.events.map(e => e.id), 'the job\'s log, in order, once each');// ── Cancel ──────────────────────────────────────────────await s.close();@@@ -294,14 +307,14 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}const dave = client(s.base, { user: USER_B });const ids = [];for (let i = 0; i < 4; i++) ids.push((await dave('/api/v1/jobs', json({ type: 'test.upper', input: { gate: 'pool' } }))).body.id);-- await until(async () => s.system.stats().running === 2, 'two running');++ await until(async () => (await s.system.stats()).running === 2, 'two running');await sleep(50);-- assert.strictEqual(s.system.stats().running, 2, 'never more than the concurrency limit');-- assert.strictEqual(s.system.stats().queued, 2);++ assert.strictEqual((await s.system.stats()).running, 2, 'never more than the concurrency limit');++ assert.strictEqual((await s.system.stats()).queued, 2);r = await dave('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'one too many' } }));assert.strictEqual(r.status, 429); assert.strictEqual(r.body.code, 'tools.job.too_many_active');gate('pool').open();-- await until(async () => s.system.stats().succeeded === 4, 'pool drained');++ await until(async () => (await s.system.stats()).succeeded === 4, 'pool drained');// ── Restart: an accepted job is not lost ────────────────await s.close();@@@ -339,7 +352,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}assert.ok(pruned >= 3);assert.strictEqual((await erin(`/api/v1/jobs/${accepted}`)).status, 404, 'expired jobs are gone');assert.ok(!fs.existsSync(path.join(dir, 'jobs', accepted)), 'with their files');-- assert.strictEqual(s.db.prepare('SELECT COUNT(*) AS n FROM tool_job_events WHERE job_id = ?').get(accepted).n, 0, 'and their events');++ assert.strictEqual((await s.db.prepare('SELECT COUNT(*) AS n FROM tool_job_events WHERE job_id = ?').get(accepted)).n, 0, 'and their events');await s.close();// ── Results as Media objects ────────────────────────────@@@ -488,7 +501,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}assert.strictEqual(r.headers.get('set-cookie'), null, 'principals get no session cookie');const devJob = r.body.id;await until(async () => (await dev(`/api/v1/jobs/${devJob}`)).body.state === 'succeeded', 'app job');-- assert.strictEqual(s.system.get(devJob).owner, 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC');++ assert.strictEqual((await s.system.get(devJob)).owner, 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC');r = await dev(`/api/v1/jobs/${devJob}`, { method: 'DELETE' });assert.strictEqual(r.status, 403, 'cancel needs tools.job.cancel');r = await client(s.base, { bearer: token({ cap: ['tools.job.read'] }) })('/api/v1/jobs', json({ type: 'test.upper', input: {} }));@@@ -504,8 +517,8 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}assert.strictEqual(r.status, 202, 'a sandbox app token can submit');const sbxJob = r.body.id;await until(async () => (await sbx(`/api/v1/jobs/${sbxJob}`)).body.state === 'succeeded', 'sandbox job');-- assert.strictEqual(s.system.get(sbxJob).env, 'sandbox', 'the job remembers its environment');-- assert.ok(s.system.get(sbxJob).ttl_ms <= 30 * 60 * 1000, 'sandbox jobs are kept briefly');++ assert.strictEqual((await s.system.get(sbxJob)).env, 'sandbox', 'the job remembers its environment');++ assert.ok((await s.system.get(sbxJob)).ttl_ms <= 30 * 60 * 1000, 'sandbox jobs are kept briefly');r = await client(s.base, { bearer: token({ sub: 'app:app_01JDDDDDDDDDDDDDDDDDDDDDDD', env: 'sandbox' }) })(`/api/v1/jobs/${sbxJob}`);assert.strictEqual(r.status, 404, 'another sandbox app cannot read it');r = await client(s.base, { bearer: contracts.serviceAuth.signServiceToken({@@@ -524,7 +537,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}const done = async (c, id) => until(async () => { const g = await c(`/api/v1/jobs/${id}`); return g.body.state === 'succeeded' && g.body; }, 'project job');const prod = client(s.base, { bearer: token({}) });let j = await done(prod, (await prod('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'prod' } }))).body.id);-- assert.strictEqual(s.system.get(j.id).project_id, PRJ, 'the job remembers its project');++ assert.strictEqual((await s.system.get(j.id)).project_id, PRJ, 'the job remembers its project');assert.strictEqual(j.result.files[0].storage, 'media');assert.strictEqual(fm2.objects.get(j.result.files[0].media.media_id).meta.namespace, `tools.app.${PRJ}`, 'production: tools.app.<project>');assert.strictEqual(j.result.files[0].media.namespace, `tools.app.${PRJ}`);@@@ -535,13 +548,14 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}const person = client(s.base, { user: USER_B });j = await done(person, (await person('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'person' } }))).body.id);assert.strictEqual(fm2.objects.get(j.result.files[0].media.media_id).meta.namespace, undefined, "a person's job stays in the tools root");-- assert.strictEqual(s.system.get(j.id).project_id, null);++ assert.strictEqual((await s.system.get(j.id)).project_id, null);await s.close();ms2.close();}console.log('jobs (shared runtime): all checks passed');} finally {++ await closeDbs();fs.rmSync(root, { recursive: true, force: true });}})().catch((err) => { console.error(err); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,42 @@++'use strict';++// The SQLite era is gone (plan T8): every Tools app serves from PostgreSQL (openvibe-sdk/db) and the++// job store lives in the one `tools` database. Nothing under apps/ may require better-sqlite3, and no++// apps/*/package.json may list it. This fails loudly if either comes back.++const assert = require('assert');++const fs = require('fs');++const path = require('path');++++const APPS = path.resolve(__dirname, '..', '..');++const SKIP = new Set(['node_modules', 'data', 'dist', 'build', 'coverage', '.git']);++const REQUIRE_RE = /\b(?:require\s*\(\s*['"]better-sqlite3['"]|dep\s*\(\s*['"]better-sqlite3['"]|from\s+['"]better-sqlite3['"])/;++++/** Every file under dir (source and manifests), skipping node_modules and build output. */++function walk(dir, out = []) {++ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {++ if (SKIP.has(e.name)) continue;++ const p = path.join(dir, e.name);++ if (e.isDirectory()) walk(p, out);++ else if (/\.(?:js|mjs|cjs|ts|json)$/.test(e.name)) out.push(p);++ }++ return out;++}++++const files = walk(APPS).filter((f) => !/\.test\.js$/.test(f) || f !== __filename);++const offenders = files++ .filter((f) => path.basename(f) !== 'no-sqlite.test.js')++ .filter((f) => REQUIRE_RE.test(fs.readFileSync(f, 'utf8')))++ .map((f) => path.relative(APPS, f));++++assert.deepStrictEqual(offenders, [], `these files still require better-sqlite3:\n ${offenders.join('\n ')}`);++++// The dependency must be gone from every app manifest (and the root, if it ever had one).++const manifests = [path.join(APPS, '..', 'package.json'), ...fs.readdirSync(APPS, { withFileTypes: true })++ .filter((e) => e.isDirectory() && !SKIP.has(e.name))++ .map((e) => path.join(APPS, e.name, 'package.json'))++ .filter((p) => fs.existsSync(p))];++for (const m of manifests) {++ const pkg = JSON.parse(fs.readFileSync(m, 'utf8'));++ assert.ok(!(pkg.dependencies && pkg.dependencies['better-sqlite3']), `${path.relative(APPS, m)} still depends on better-sqlite3`);++}++++console.log(`no sqlite (nothing under apps/ requires better-sqlite3): all checks passed (${files.length} files scanned)`);@@@ -84,7 +84,7 @@ function assertShape(body, service) {let r = await json(`${img.base}/api/ready`);assertShape(r.body, 'tools-img');assert.strictEqual(r.status, 200, JSON.stringify(r.body));-- for (const n of ['jobs_db', 'job_runtime', 'data_dir', 'uploads_dir', 'output_dir']) {++ for (const n of ['tools_db', 'job_runtime', 'data_dir', 'uploads_dir', 'output_dir']) {assert.strictEqual(r.body.checks[n].required, true, `${n} is required`);assert.strictEqual(r.body.checks[n].status, 'ok', `${n} ok`);}@@@ -24,6 +24,13 @@ async function startApp(app, env = {}, port, { readyPath = '/api/health' } = {})port = port || await freePort();const dir = path.join(__dirname, '..', '..', app);let out = '';++ // Without a DATABASE_URL the app serves from an embedded PGlite database. Point it at a directory++ // under the test's own DATA_DIR, so it survives the kill a restart test does (production uses a real++ // PostgreSQL, which outlives the process the same way).++ const url = env.DATABASE_URL !== undefined ? env.DATABASE_URL : process.env.DATABASE_URL;++ const pglite = (!String(url || '').trim() && env.DATA_DIR && env.TOOLS_PGLITE_DIR === undefined)++ ? { TOOLS_PGLITE_DIR: path.join(env.DATA_DIR, 'pglite') }++ : {};const child = spawn(process.execPath, ['server/index.js'], {cwd: dir,env: {@@@ -35,6 +42,7 @@ async function startApp(app, env = {}, port, { readyPath = '/api/health' } = {})OV_NETWORK_URL: 'http://127.0.0.1:9',OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',TOOLS_JOB_RESULTS: 'local',++ ...pglite,...env,},stdio: ['ignore', 'pipe', 'pipe'],nnew file mode 100644@@@ -0,0 +1,29 @@++'use strict';++// The one `tools` database for a test run (plan T8): one PGlite database per test directory, migrated++// with the repo's migrations. It is kept across a satellite restart (the data must survive it), so a++// test that restarts a satellite calls testDb() with the same directory again; closeAllTestDbs() at the++// end of the run. Nothing here needs better-sqlite3: PGlite is real PostgreSQL in-process.++const fs = require('fs');++const path = require('path');++const { dep } = require('./deps');++++const MIGRATIONS = path.resolve(__dirname, '..', '..', '..', 'migrations');++const quiet = { log() {}, warn() {}, error() {} };++const dbs = new Map();++++async function testDb(dir) {++ if (!dbs.has(dir)) {++ fs.mkdirSync(path.join(dir, 'pglite'), { recursive: true });++ const db = dep('openvibe-sdk/db').createDb({ pglite: path.join(dir, 'pglite'), service: 'tools-test', log: quiet });++ await db.migrate({ dir: MIGRATIONS, log: quiet });++ dbs.set(dir, db);++ }++ return dbs.get(dir);++}++++async function closeAllTestDbs() {++ for (const db of dbs.values()) await db.close().catch(() => {});++ dbs.clear();++}++++module.exports = { testDb, closeAllTestDbs, MIGRATIONS, quiet };@@@ -327,14 +327,14 @@ function createRunApi(o) {const notFound = () => new Refusal(404, 'tools.run.file_not_found', `${ref.media_id || `${ref.job_id} file ${ref.index}`} is not a file you can read here: a result of one of your own Tools jobs that has not expired.`);let row = null, index = ref.index;if (ref.job_id) {-- row = system.get(ref.job_id);++ row = await system.get(ref.job_id);if (!row && o.jobs.fetchSibling) {const got = await o.jobs.fetchSibling(ref, req, dir);if (got) return got;throw notFound();}} else {-- const hit = system.findResultMedia(owner, ref.media_id);++ const hit = await system.findResultMedia(owner, ref.media_id);if (hit) { row = hit.row; index = hit.index; }}if (!row || row.owner !== owner || row.state !== 'succeeded') throw notFound();@@@ -352,23 +352,22 @@ function createRunApi(o) {throw notFound();}-- function waitFor(system, id, ms, res) {-- return new Promise((resolve) => {-- const row = system.get(id);-- if (!row || TERMINAL.has(row.state) || ms <= 0) return resolve();++ async function waitFor(system, id, ms, res) {++ const row = await system.get(id);++ if (!row || TERMINAL.has(row.state) || ms <= 0) return;++ await new Promise((resolve) => {let done = false;const finish = () => { if (done) return; done = true; clearTimeout(timer); off(); res.removeListener('close', finish); resolve(); };const off = system.subscribe(id, (e) => { if (TERMINAL.has(e.data.state)) finish(); });const timer = setTimeout(finish, ms);res.on('close', finish);// It may have finished between the first read and the subscription.-- const again = system.get(id);-- if (!again || TERMINAL.has(again.state)) finish();++ system.get(id).then((again) => { if (!again || TERMINAL.has(again.state)) finish(); }).catch(() => {});});}-- function runView(system, d, row) {-- const job = system.view(row);++ async function runView(system, d, row) {++ const job = await system.view(row);const took = row.finished_at ? Math.max(0, row.finished_at - row.created_at) : 0;if (row.state === 'succeeded') return { state: 'succeeded', tool: d.id, result: { data: (job.result && job.result.data) || {}, files: (job.result && job.result.files) || [] }, took_ms: took, job };if (row.state === 'failed' || row.state === 'cancelled') return { state: row.state, tool: d.id, error: job.error || contracts.http.problem(500, 'tools.job.failed', { detail: 'The job failed' }), took_ms: took, job };@@@ -400,9 +399,9 @@ function createRunApi(o) {// The bytes against files.accept (hard; a misleading name or type is corrected).if (spec && !guard.checkFiles(req, res, files, spec.accept, d.id)) return undefined;// Sessions from one address share a bound on unfinished jobs; the store's queue and disk bounds.-- const full = system.addressFull ? system.addressFull(owner, who.ipKey) : null;++ const full = system.addressFull ? await system.addressFull(owner, who.ipKey) : null;if (full && guard.refuse(req, res, { status: 429, code: 'tools.job.too_many_active', reason: 'jobs.address', tool: d.id, retryAfter: 30, detail: `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish.`, extra: { scope: 'address' } })) return undefined;-- const busy = system.busy ? system.busy() : null;++ const busy = system.busy ? await system.busy() : null;if (busy && !guard.jobsBusy(req, res, busy, d.id)) return undefined;let row, replayed;try {@@@ -421,8 +420,8 @@ function createRunApi(o) {}files.length = 0; // the job owns them nowif (wait > 0) await waitFor(system, row.id, wait, res);-- const fresh = system.get(row.id) || row;-- const body = runView(system, d, fresh);++ const fresh = (await system.get(row.id)) || row;++ const body = await runView(system, d, fresh);const headers = replayed ? { 'Idempotent-Replayed': 'true' } : {};if (body.location) headers.Location = body.location;return send(res, replayed || !body.location ? 200 : 202, body, headers);@@@ -12,7 +12,6 @@"openvibeToolsShared": "^1.2.0","dependencies": {"@electric-sql/pglite": "^0.5.8",-- "better-sqlite3": "^11.0.0","cookie-parser": "^1.4.7","iovalkey": "^0.4.0","pg": "^8.23.0",@@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');const sdk = require('openvibe-sdk');// ── Analytics ────────────────────────────────────────────────--const Database = require('better-sqlite3');const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recordedconst { requireInternalAccess } = require('../../_shared/internal-token');@@@ -72,9 +71,8 @@ const obs = observe({service: 'tools-audio', release: release.release,checks: [ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),-- ready.sqlite('jobs_db', () => jobs.db, { sql: 'SELECT COUNT(*) AS n FROM tool_jobs', description: 'job store (jobs.db)' }),ready.jobRuntime('job_runtime', () => jobs),-- ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'jobs.db and job inputs/results' }),++ ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'job inputs/results' }),ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),@@@ -160,9 +158,9 @@ app.use((req, _res, next) => {// ── API Routes ───────────────────────────────────────────────// Health check--app.get('/api/health', (_req, res) => {++app.get('/api/health', async (_req, res) => {const stats = retention.getStats();-- res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats, jobs: jobs.stats() });++ res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats, jobs: await jobs.stats() });});// Domain context (frontend calls this on load to get branding). It also starts this browser's session@@@ -204,11 +202,11 @@ app.post('/api/probe', guard.originCheck(), burstLimiter, processLimiter, guard.});// ── Jobs (/api/v1/jobs) ──────────────────────────────────────--// The audio operations, asynchronous and durable: accepted into data/jobs.db, followed over SSE++// The audio operations, asynchronous and durable: accepted into the shared `tools` database, followed over SSE// (ffmpeg's own progress), cancellable (ffmpeg is killed), reattachable by id after a reload or a// restart (apps/_shared/jobs).const jobs = jobsRuntime.setupJobs({-- app, service: 'audio', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts, sdk,++ app, service: 'audio', dataDir: path.resolve(__dirname, '..', config.dataDir), db: toolsDb.db, contracts, sdk,getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,define: defineJobs,receive: uploadAny,@@@ -341,7 +339,7 @@ require('../../_shared/graceful').gracefulStop({close: [() => require('../../_shared/usage').stopRecorder(800),() => analytics.destroy(),-- () => jobs.close(), // running jobs stay 'running' in data/jobs.db; the next boot re-queues them++ () => jobs.close(), // running jobs stay 'running' in the tools database; the next boot re-queues them() => guard.close(),() => toolsDb.db.close(),() => toolsValkey && toolsValkey.close(),@@@ -12,7 +12,6 @@"openvibeToolsShared": "^1.2.0","dependencies": {"@electric-sql/pglite": "^0.5.8",-- "better-sqlite3": "^11.0.0","cookie-parser": "^1.4.7","iovalkey": "^0.4.0","pg": "^8.23.0",@@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');const sdk = require('openvibe-sdk');// ── Analytics ────────────────────────────────────────────────--const Database = require('better-sqlite3');const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recordedconst { requireInternalAccess } = require('../../_shared/internal-token');@@@ -89,9 +88,8 @@ const obs = observe({service: 'tools-docs', release: release.release,checks: [ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),-- ready.sqlite('jobs_db', () => jobs.db, { sql: 'SELECT COUNT(*) AS n FROM tool_jobs', description: 'job store (jobs.db)' }),ready.jobRuntime('job_runtime', () => jobs),-- ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'jobs.db and job inputs/results' }),++ ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'job inputs/results' }),ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),@@@ -189,9 +187,9 @@ function sendToolError(req, res, err, label) {}// Health check--app.get('/api/health', (_req, res) => {++app.get('/api/health', async (_req, res) => {const stats = retention.getStats();-- res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats, jobs: jobs.stats(), workers: pool.stats() });++ res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats, jobs: await jobs.stats(), workers: pool.stats() });});// Domain context (frontend calls this on load to get branding). It also starts this browser's session@@@ -232,10 +230,10 @@ app.post('/api/info', guard.originCheck(), burstLimiter, processLimiter, guard.t});// ── Jobs (/api/v1/jobs) ──────────────────────────────────────--// The document operations, asynchronous and durable: accepted into data/jobs.db, followed over SSE,++// The document operations, asynchronous and durable: accepted into the shared `tools` database, followed over SSE,// reattachable by id after a reload or a restart.const jobs = jobsRuntime.setupJobs({-- app, service: 'docs', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts, sdk,++ app, service: 'docs', dataDir: path.resolve(__dirname, '..', config.dataDir), db: toolsDb.db, contracts, sdk,getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,define: (system) => defineJobs(system, pool),receive: uploadAny,@@@ -341,7 +339,7 @@ require('../../_shared/graceful').gracefulStop({close: [() => require('../../_shared/usage').stopRecorder(800),() => analytics.destroy(),-- () => jobs.close(), // running jobs stay 'running' in data/jobs.db; the next boot re-queues them++ () => jobs.close(), // running jobs stay 'running' in the tools database; the next boot re-queues them() => pool.close(),() => guard.close(),() => toolsDb.db.close(),@@@ -15,7 +15,6 @@"dotenv": "^16.3.1","express": "^4.18.2","@electric-sql/pglite": "^0.5.8",-- "better-sqlite3": "^11.0.0","iovalkey": "^0.4.0","pg": "^8.23.0","express-rate-limit": "^7.1.5",@@@ -12,7 +12,6 @@"openvibeToolsShared": "^1.2.0","dependencies": {"@electric-sql/pglite": "^0.5.8",-- "better-sqlite3": "^11.10.0","cookie-parser": "^1.4.7","iovalkey": "^0.4.0","pg": "^8.23.0",@@@ -12,7 +12,6 @@"openvibeToolsShared": "^1.2.0","dependencies": {"@electric-sql/pglite": "^0.5.8",-- "better-sqlite3": "^11.0.0","cookie-parser": "^1.4.7","iovalkey": "^0.4.0","pg": "^8.23.0",@@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');const sdk = require('openvibe-sdk');// ── Analytics ────────────────────────────────────────────────--const Database = require('better-sqlite3');const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recordedconst { requireInternalAccess } = require('../../_shared/internal-token');@@@ -78,9 +77,8 @@ const obs = observe({service: 'tools-img', release: release.release,checks: [ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),-- ready.sqlite('jobs_db', () => jobs.db, { sql: 'SELECT COUNT(*) AS n FROM tool_jobs', description: 'job store (jobs.db)' }),ready.jobRuntime('job_runtime', () => jobs),-- ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'jobs.db and job inputs/results' }),++ ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'job inputs/results' }),ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),@@@ -166,9 +164,9 @@ app.use((req, _res, next) => {// ── API Routes ───────────────────────────────────────────────// Health check--app.get('/api/health', (_req, res) => {++app.get('/api/health', async (_req, res) => {const stats = retention.getStats();-- res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats, jobs: jobs.stats(), workers: pool.stats() });++ res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats, jobs: await jobs.stats(), workers: pool.stats() });});// Domain context (frontend calls this on load to get branding). It also starts this browser's session@@@ -199,10 +197,10 @@ app.get('/api/tools', (_req, res) => {});// ── Jobs (/api/v1/jobs) ──────────────────────────────────────--// The image operations, asynchronous and durable: accepted into data/jobs.db, followed over SSE,--// reattachable by id after a reload or a restart (apps/_shared/jobs).++// The image operations, asynchronous and durable: accepted into the shared `tools` database, followed++// over SSE, reattachable by id after a reload or a restart (apps/_shared/jobs).const jobs = jobsRuntime.setupJobs({-- app, service: 'img', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts, sdk,++ app, service: 'img', dataDir: path.resolve(__dirname, '..', config.dataDir), db: toolsDb.db, contracts, sdk,getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,define: (system) => defineJobs(system, pool),receive: uploadSingle,@@@ -306,7 +304,7 @@ require('../../_shared/graceful').gracefulStop({close: [() => require('../../_shared/usage').stopRecorder(800),() => analytics.destroy(),-- () => jobs.close(), // running jobs stay 'running' in data/jobs.db; the next boot re-queues them++ () => jobs.close(), // running jobs stay 'running' in the tools database; the next boot re-queues them() => pool.close(),() => guard.close(),() => toolsDb.db.close(),@@@ -11,7 +11,6 @@"openvibeToolsShared": "^1.2.0","dependencies": {"axios": "^1.7.0",-- "better-sqlite3": "^11.0.0","iovalkey": "^0.4.0","pg": "^8.23.0","@electric-sql/pglite": "^0.5.8",@@@ -14,7 +14,6 @@"helmet": "^7.1.0","cors": "^2.8.5","cookie-parser": "^1.4.6",-- "better-sqlite3": "^11.0.0","@electric-sql/pglite": "^0.5.8","iovalkey": "^0.4.0","pg": "^8.23.0",@@@ -17,7 +17,6 @@"express": "^4.21.2","express-rate-limit": "^7.5.0","helmet": "^8.1.0",-- "better-sqlite3": "^11.0.0","iovalkey": "^0.4.0","pg": "^8.23.0","@electric-sql/pglite": "^0.5.8",
@@ -3,9 +3,9 @@
@@ -3,9 +3,9 @@
// The Tools guard (apps/_shared/guard): anti-abuse for the gateway and every satellite, driven by
// The Tools guard (apps/_shared/guard): anti-abuse for the gateway and every satellite, driven by
// the tools' descriptors (tools.tool@1: quotaClass, cost, auth, files.accept, limits) and the host-wide
// the tools' descriptors (tools.tool@1: quotaClass, cost, auth, files.accept, limits) and the host-wide
// numbers in ./limits.js. Like the rest of apps/_shared it has no dependencies: the app passes its
// numbers in ./limits.js. Like the rest of apps/_shared it has no dependencies: the app passes its
-// better-sqlite3 class, openvibe-contracts (when it has it) and its descriptor specs.
+// openvibe-sdk/db handle, its Valkey and openvibe-contracts (when it has it) and its descriptor specs.
//
//
-// const guard = createGuard({ app: 'img', dataDir, Database, contracts, specs, networkUrl, … });
+// const guard = createGuard({ app: 'img', dataDir, db, valkey, contracts, specs, networkUrl, … });
// app.set('trust proxy', TRUST_PROXY); // ./ip.js: one loopback hop
// app.set('trust proxy', TRUST_PROXY); // ./ip.js: one loopback hop
// app.use(guard.identify); // req.user (Network sign-in, aud checked)
// app.use(guard.identify); // req.user (Network sign-in, aud checked)
// app.use('/api/', legacyLimiter, guard.apiQuota); // every /api/ request (tools-api)
// app.use('/api/', legacyLimiter, guard.apiQuota); // every /api/ request (tools-api)
@@ -82,8 +82,9 @@ function discardUploads(req) {
@@ -82,8 +82,9 @@ function discardUploads(req) {
/**
/**
* @param {object} o
* @param {object} o
* @param {string} o.app 'gateway' | 'img' | … (logs, metrics)
* @param {string} o.app 'gateway' | 'img' | … (logs, metrics)
- * @param {string} [o.dataDir] guard.db goes here (with o.Database)
- * @param {Function} [o.Database] require('better-sqlite3'); without it everything stays in memory
+ * @param {string} [o.dataDir] where the app keeps its files (job inputs/results)
+ * @param {object} [o.db] openvibe-sdk/db handle (guard_abuse); without it everything stays in memory
+ * @param {object} [o.valkey] the shared Valkey (day counters, salt, buckets)
* @param {object} [o.contracts] openvibe-contracts, when the app has it
* @param {object} [o.contracts] openvibe-contracts, when the app has it
* @param {object[]} [o.specs] the app's descriptor specs (server/descriptors.js SPECS)
* @param {object[]} [o.specs] the app's descriptor specs (server/descriptors.js SPECS)
* @param {object} [o.keys] { get(), ensure() } — or networkUrl/networkInternalUrl/publicKeyFiles
* @param {object} [o.keys] { get(), ensure() } — or networkUrl/networkInternalUrl/publicKeyFiles
@@ -7,12 +7,11 @@
@@ -7,12 +7,11 @@
// succeeded → tools.job.succeeded priority important actor: service:tools
// succeeded → tools.job.succeeded priority important actor: service:tools
// failed (tool, timeout, boot) → tools.job.failed priority important actor: service:tools
// failed (tool, timeout, boot) → tools.job.failed priority important actor: service:tools
//
//
-// Each event is written to the satellite's own `event_outbox` table (openvibe-sdk createOutbox) in
-// the SQLite transaction that records the transition, so it exists if and only if the transition
-// committed; the relay posts it to OpenVibe.Events afterwards (at least once; Events dedupes on
-// event_id). A cancelled job, a progress tick and a requeue after a restart are not announced (a
-// requeued job is announced again when it starts). Sandbox jobs (developer-app sandbox tokens) are
-// never announced.
+// Each event is written to the one tools database's `event_outbox` table (openvibe-sdk createPgOutbox) in
+// the SAME transaction that records the transition, so it exists if and only if the transition committed;
+// the relay posts it to OpenVibe.Events afterwards (at least once; Events dedupes on event_id). A cancelled
+// job, a progress tick and a requeue after a restart are not announced (a requeued job is announced again
+// when it starts). Sandbox jobs (developer-app sandbox tokens) are never announced.
//
//
// Payloads (openvibe-contracts tools.job.*@1, validated before they are enqueued) carry ids, type,
// Payloads (openvibe-contracts tools.job.*@1, validated before they are enqueued) carry ids, type,
// owner, state, times, where result files are and the error; never the job's input, file names,
// owner, state, times, where result files are and the error; never the job's input, file names,
@@ -29,7 +28,7 @@ const SERVICE_ACTOR = Object.freeze({ type: 'service', id: SOURCE });
@@ -29,7 +28,7 @@ const SERVICE_ACTOR = Object.freeze({ type: 'service', id: SOURCE });
const OWNER_TYPES = { user: 'user', svc: 'service', app: 'app', mod: 'mod' };
const OWNER_TYPES = { user: 'user', svc: 'service', app: 'app', mod: 'mod' };
const PRIORITY = { 'tools.job.created': 'low', 'tools.job.started': 'low', 'tools.job.succeeded': 'important', 'tools.job.failed': 'important' };
const PRIORITY = { 'tools.job.created': 'low', 'tools.job.started': 'low', 'tools.job.succeeded': 'important', 'tools.job.failed': 'important' };
-const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());
+const iso = (ms) => (ms == null ? null : new Date(Number(ms)).toISOString());
const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };
const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };
/** 'user:usr_…' | 'svc:<slug>' | 'app:app_…' | 'mod:mod_…' → SubjectRef; anything else (a session) → null. */
/** 'user:usr_…' | 'svc:<slug>' | 'app:app_…' | 'mod:mod_…' → SubjectRef; anything else (a session) → null. */
@@ -60,11 +59,11 @@ function payloadFor(eventType, row, { contracts, service, referenced = false })
@@ -60,11 +59,11 @@ function payloadFor(eventType, row, { contracts, service, referenced = false })
job_id: row.id,
job_id: row.id,
service,
service,
type: row.type,
type: row.type,
- type_version: row.type_version,
+ type_version: Number(row.type_version),
state: row.state,
state: row.state,
owner: ownerRef(contracts, row.owner),
owner: ownerRef(contracts, row.owner),
- attempts: row.attempts,
- max_attempts: row.max_attempts,
+ attempts: Number(row.attempts),
+ max_attempts: Number(row.max_attempts),
created_at: iso(row.created_at),
created_at: iso(row.created_at),
retry_of: row.retry_of || null,
retry_of: row.retry_of || null,
};
};
@@ -103,23 +102,24 @@ function payloadFor(eventType, row, { contracts, service, referenced = false })
@@ -103,23 +102,24 @@ function payloadFor(eventType, row, { contracts, service, referenced = false })
* @param {object} o
* @param {object} o
* @param {object} o.contracts require('openvibe-contracts') (v0.30.0+ knows tools.job.*@1)
* @param {object} o.contracts require('openvibe-contracts') (v0.30.0+ knows tools.job.*@1)
* @param {string} o.service the satellite ('img' | 'audio' | 'docs' | …) → payload.service
* @param {string} o.service the satellite ('img' | 'audio' | 'docs' | …) → payload.service
- * @param {object|null} o.outbox openvibe-sdk createOutbox(db, …) on the jobs database, or null
- * @param {(id) => number} [o.referenceCount]
+ * @param {object|null} o.outbox openvibe-sdk createPgOutbox(db, …) on the one tools database, or null
+ * @param {(id) => Promise<number>} [o.referenceCount]
* @param {object} [o.log]
* @param {object} [o.log]
*/
*/
-function createJobEvents({ contracts, service, outbox = null, referenceCount = () => 0, log = console }) {
+function createJobEvents({ contracts, service, outbox = null, referenceCount = async () => 0, log = console }) {
const stats = { queued: 0, invalid: 0, lastInvalid: null };
const stats = { queued: 0, invalid: 0, lastInvalid: null };
/**
/**
- * Enqueue `eventType` for this row (just read back inside the caller's transaction). A payload that
- * fails its contract is logged and skipped, never thrown: a bug here must not stop jobs. An outbox
- * write error does throw, so the caller's transaction rolls back with it.
+ * Enqueue `eventType` for this row inside the caller's transaction `t` (the row was just read back).
+ * A payload that fails its contract is logged and skipped, never thrown: a bug here must not stop
+ * jobs. An outbox write error does throw, so the caller's transaction rolls back with it.
*/
*/
- function announce(eventType, row) {
+ async function announce(t, eventType, row) {
if (!outbox || !row || row.env === 'sandbox') return null;
if (!outbox || !row || row.env === 'sandbox') return null;
let payload;
let payload;
try {
try {
- payload = payloadFor(eventType, row, { contracts, service, referenced: eventType !== 'tools.job.created' && eventType !== 'tools.job.started' && referenceCount(row.id) > 0 });
+ const referenced = eventType !== 'tools.job.created' && eventType !== 'tools.job.started' && (await referenceCount(row.id)) > 0;
+ payload = payloadFor(eventType, row, { contracts, service, referenced });
const v = contracts.validate(`${eventType}@1`, payload);
const v = contracts.validate(`${eventType}@1`, payload);
if (!v.valid) throw new Error(v.errors.map(e => `${e.path} ${e.message}`).join('; '));
if (!v.valid) throw new Error(v.errors.map(e => `${e.path} ${e.message}`).join('; '));
} catch (err) {
} catch (err) {
@@ -128,7 +128,7 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (
@@ -128,7 +128,7 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (
log.error(`[Jobs] ${eventType} for ${row.id} not announced (payload does not match the contract): ${err.message}`);
log.error(`[Jobs] ${eventType} for ${row.id} not announced (payload does not match the contract): ${err.message}`);
return null;
return null;
}
}
- const env = outbox.enqueue({
+ const env = await outbox.enqueue(t, {
event_type: eventType,
event_type: eventType,
actor: eventType === 'tools.job.created' && payload.owner ? payload.owner : SERVICE_ACTOR,
actor: eventType === 'tools.job.created' && payload.owner ? payload.owner : SERVICE_ACTOR,
subject: { type: 'job', id: row.id },
subject: { type: 'job', id: row.id },
@@ -143,14 +143,14 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (
@@ -143,14 +143,14 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (
return {
return {
enabled: !!outbox,
enabled: !!outbox,
- created: (row) => announce('tools.job.created', row),
- started: (row) => announce('tools.job.started', row),
+ created: (t, row) => announce(t, 'tools.job.created', row),
+ started: (t, row) => announce(t, 'tools.job.started', row),
/** succeeded → tools.job.succeeded, failed → tools.job.failed; cancelled is not announced. */
/** succeeded → tools.job.succeeded, failed → tools.job.failed; cancelled is not announced. */
- finished: (row) => (row && row.state === 'succeeded' ? announce('tools.job.succeeded', row)
- : row && row.state === 'failed' ? announce('tools.job.failed', row) : null),
- status() {
+ finished: (t, row) => (row && row.state === 'succeeded' ? announce(t, 'tools.job.succeeded', row)
+ : row && row.state === 'failed' ? announce(t, 'tools.job.failed', row) : null),
+ async status() {
if (!outbox) return { enabled: false };
if (!outbox) return { enabled: false };
- return { enabled: true, pending: outbox.pending(), rejected: outbox.rejected(), queued_since_boot: stats.queued, invalid: stats.invalid, last_invalid: stats.lastInvalid };
+ return { enabled: true, pending: await outbox.pending(), rejected: await outbox.rejected(), queued_since_boot: stats.queued, invalid: stats.invalid, last_invalid: stats.lastInvalid };
},
},
};
};
}
}
@@ -165,8 +165,11 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (
@@ -165,8 +165,11 @@ function createJobEvents({ contracts, service, outbox = null, referenceCount = (
* OV_NETWORK_INTERNAL_URL token endpoint host (default http://127.0.0.1:4000)
* OV_NETWORK_INTERNAL_URL token endpoint host (default http://127.0.0.1:4000)
* EVENTS_RELAY_INTERVAL_MS relay poll (default 2000)
* EVENTS_RELAY_INTERVAL_MS relay poll (default 2000)
*
*
+ * The outbox table is created at boot (createPgOutbox.ensureSchema); outbox.ready is that promise, for a
+ * test (or a caller) that must know the table exists.
+ *
* @param {object} o
* @param {object} o
- * @param {object} o.db the jobs database (better-sqlite3) — the outbox table lives beside tool_jobs
+ * @param {object} o.db openvibe-sdk/db handle — the outbox table lives beside tool_jobs
* @param {object} [o.sdk] require('openvibe-sdk')
* @param {object} [o.sdk] require('openvibe-sdk')
* @param {object} [o.env]
* @param {object} [o.env]
* @param {Function} [o.fetch]
* @param {Function} [o.fetch]
@@ -190,7 +193,7 @@ function outboxFromEnv({ db, sdk, env = process.env, fetch: fetchImpl, log = con
@@ -190,7 +193,7 @@ function outboxFromEnv({ db, sdk, env = process.env, fetch: fetchImpl, log = con
const client = sdk.createClient({ baseUrls: { events: url }, tokenProvider: tokens, retries: 0, ...(fetchImpl ? { fetch: fetchImpl } : {}) });
const client = sdk.createClient({ baseUrls: { events: url }, tokenProvider: tokens, retries: 0, ...(fetchImpl ? { fetch: fetchImpl } : {}) });
let lastError = null;
let lastError = null;
const interval = parseInt(env.EVENTS_RELAY_INTERVAL_MS, 10);
const interval = parseInt(env.EVENTS_RELAY_INTERVAL_MS, 10);
- const outbox = sdk.events.createOutbox(db, {
+ const outbox = sdk.events.createPgOutbox(db, {
events: sdk.events.createEventsClient(client, { source: SOURCE }),
events: sdk.events.createEventsClient(client, { source: SOURCE }),
intervalMs: Number.isFinite(interval) && interval >= 100 ? interval : 2000,
intervalMs: Number.isFinite(interval) && interval >= 100 ? interval : 2000,
onError: (err) => {
onError: (err) => {
@@ -199,8 +202,8 @@ function outboxFromEnv({ db, sdk, env = process.env, fetch: fetchImpl, log = con
@@ -199,8 +202,8 @@ function outboxFromEnv({ db, sdk, env = process.env, fetch: fetchImpl, log = con
lastError = msg;
lastError = msg;
},
},
});
});
- outbox.ensureSchema();
outbox.url = url;
outbox.url = url;
+ outbox.ready = outbox.ensureSchema();
return { outbox, reason: null };
return { outbox, reason: null };
}
}
@@ -95,11 +95,11 @@ function mountJobRoutes(app, o) {
@@ -95,11 +95,11 @@ function mountJobRoutes(app, o) {
return w || { owner: null };
return w || { owner: null };
}
}
/** The job, if it exists AND belongs to the caller; otherwise the same 404 either way. */
/** The job, if it exists AND belongs to the caller; otherwise the same 404 either way. */
- function load(req, res, action) {
+ async function load(req, res, action) {
const w = who(req, res, { action });
const w = who(req, res, { action });
if (!w) return null;
if (!w) return null;
const id = String(req.params.id || '');
const id = String(req.params.id || '');
- const row = ID_RE.test(id) ? system.get(id) : null;
+ const row = ID_RE.test(id) ? await system.get(id) : null;
if (!row || !w.owner || row.owner !== w.owner) { send(res, 404, 'tools.job.not_found', 'No such job'); return null; }
if (!row || !w.owner || row.owner !== w.owner) { send(res, 404, 'tools.job.not_found', 'No such job'); return null; }
return row;
return row;
}
}
@@ -109,11 +109,11 @@ function mountJobRoutes(app, o) {
@@ -109,11 +109,11 @@ function mountJobRoutes(app, o) {
// Which satellite holds a job? Only for a caller on this host with no proxy in between (the
// Which satellite holds a job? Only for a caller on this host with no proxy in between (the
// gateway's facade); through nginx or the gateway's own proxy it is not found. Says nothing but
// gateway's facade); through nginx or the gateway's own proxy it is not found. Says nothing but
// whether the id exists here.
// whether the id exists here.
- app.get('/api/internal/jobs/:id', (req, res) => {
+ app.get('/api/internal/jobs/:id', async (req, res) => {
res.set('Cache-Control', 'no-store');
res.set('Cache-Control', 'no-store');
const direct = isLoopback(req.socket && req.socket.remoteAddress) && !req.headers['x-forwarded-for'] && !req.headers['x-real-ip'];
const direct = isLoopback(req.socket && req.socket.remoteAddress) && !req.headers['x-forwarded-for'] && !req.headers['x-real-ip'];
const id = String(req.params.id || '');
const id = String(req.params.id || '');
- if (!direct || !ID_RE.test(id) || !system.get(id)) return send(res, 404, 'tools.job.not_found', 'No such job');
+ if (!direct || !ID_RE.test(id) || !(await system.get(id))) return send(res, 404, 'tools.job.not_found', 'No such job');
return res.json({ id, service: system.service || null });
return res.json({ id, service: system.service || null });
});
});
@@ -136,8 +136,8 @@ function mountJobRoutes(app, o) {
@@ -136,8 +136,8 @@ function mountJobRoutes(app, o) {
};
};
// Over the store's bounds (queued jobs, disk): answer before an upload is accepted.
// Over the store's bounds (queued jobs, disk): answer before an upload is accepted.
- const notBusy = (req, res, next) => {
- const busy = system.busy ? system.busy() : null;
+ const notBusy = async (req, res, next) => {
+ const busy = system.busy ? await system.busy() : null;
if (!busy) return next();
if (!busy) return next();
if (o.onBusy) return o.onBusy(req, res, busy) ? next() : undefined;
if (o.onBusy) return o.onBusy(req, res, busy) ? next() : undefined;
res.set('Retry-After', String(busy.retryAfter || 30));
res.set('Retry-After', String(busy.retryAfter || 30));
@@ -160,7 +160,7 @@ function mountJobRoutes(app, o) {
@@ -160,7 +160,7 @@ function mountJobRoutes(app, o) {
const files = list.map(f => ({ path: f.path, buffer: f.path ? undefined : f.buffer, name: f.originalname, mime: f.mimetype, size: f.size }));
const files = list.map(f => ({ path: f.path, buffer: f.path ? undefined : f.buffer, name: f.originalname, mime: f.mimetype, size: f.size }));
const key = req.headers['idempotency-key'] != null ? String(req.headers['idempotency-key']) : (b.idempotency_key != null ? String(b.idempotency_key) : null);
const key = req.headers['idempotency-key'] != null ? String(req.headers['idempotency-key']) : (b.idempotency_key != null ? String(b.idempotency_key) : null);
// Sessions from one address share a bound on unfinished jobs: a new cookie is no new allowance.
// Sessions from one address share a bound on unfinished jobs: a new cookie is no new allowance.
- const full = w.caller && system.addressFull ? system.addressFull(w.owner, w.caller.ipKey) : null;
+ const full = w.caller && system.addressFull ? await system.addressFull(w.owner, w.caller.ipKey) : null;
if (full) {
if (full) {
const goOn = o.onAddressFull ? o.onAddressFull(req, res, full) : (send(res, 429, 'tools.job.too_many_active', `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish`), false);
const goOn = o.onAddressFull ? o.onAddressFull(req, res, full) : (send(res, 429, 'tools.job.too_many_active', `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish`), false);
if (!goOn) { for (const f of files) if (f.path) fs.unlink(f.path, () => {}); return; }
if (!goOn) { for (const f of files) if (f.path) fs.unlink(f.path, () => {}); return; }
@@ -177,7 +177,7 @@ function mountJobRoutes(app, o) {
@@ -177,7 +177,7 @@ function mountJobRoutes(app, o) {
const { job, replayed } = await system.submit({ owner: w.owner, type: String(b.type || ''), input, files, idempotencyKey: key, ttlMs: ttlMs(req, w), env: w.env || 'production', ipKey: w.caller ? w.caller.ipKey : null, project: w.kind === 'principal' && w.claims && w.claims.actor_type === 'app' ? w.claims.project_id : null, traceId: req.ov && req.ov.traceId });
const { job, replayed } = await system.submit({ owner: w.owner, type: String(b.type || ''), input, files, idempotencyKey: key, ttlMs: ttlMs(req, w), env: w.env || 'production', ipKey: w.caller ? w.caller.ipKey : null, project: w.kind === 'principal' && w.claims && w.claims.actor_type === 'app' ? w.claims.project_id : null, traceId: req.ov && req.ov.traceId });
res.set('Location', `/api/v1/jobs/${job.id}`);
res.set('Location', `/api/v1/jobs/${job.id}`);
if (replayed) res.set('Idempotent-Replayed', 'true');
if (replayed) res.set('Idempotent-Replayed', 'true');
- return res.status(replayed ? 200 : 202).json(system.view(job));
+ return res.status(replayed ? 200 : 202).json(await system.view(job));
} catch (err) {
} catch (err) {
if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);
if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);
console.error('[Jobs] submit failed:', err.message);
console.error('[Jobs] submit failed:', err.message);
@@ -185,31 +185,31 @@ function mountJobRoutes(app, o) {
@@ -185,31 +185,31 @@ function mountJobRoutes(app, o) {
}
}
});
});
- app.get('/api/v1/jobs/:id', (req, res) => {
+ app.get('/api/v1/jobs/:id', async (req, res) => {
noStore(res);
noStore(res);
- const row = load(req, res, 'read');
- if (row) res.json(system.view(row));
+ const row = await load(req, res, 'read');
+ if (row) res.json(await system.view(row));
});
});
- app.delete('/api/v1/jobs/:id', origin, (req, res) => {
+ app.delete('/api/v1/jobs/:id', origin, async (req, res) => {
noStore(res);
noStore(res);
- const row = load(req, res, 'cancel');
+ const row = await load(req, res, 'cancel');
if (!row) return;
if (!row) return;
if (row.state === 'succeeded' || row.state === 'failed') return send(res, 409, 'tools.job.already_finished', `The job already ${row.state}`, { state: row.state });
if (row.state === 'succeeded' || row.state === 'failed') return send(res, 409, 'tools.job.already_finished', `The job already ${row.state}`, { state: row.state });
- const { job } = system.cancel(row.id);
- return res.status(job.state === 'cancelled' ? 200 : 202).json(system.view(job));
+ const { job } = await system.cancel(row.id);
+ return res.status(job.state === 'cancelled' ? 200 : 202).json(await system.view(job));
});
});
// Retry and references are writes: a principal needs tools.job.create, like a submit.
// Retry and references are writes: a principal needs tools.job.create, like a submit.
- app.post('/api/v1/jobs/:id/retry', origin, ...(o.limiters || []), notBusy, (req, res) => {
+ app.post('/api/v1/jobs/:id/retry', origin, ...(o.limiters || []), notBusy, async (req, res) => {
noStore(res);
noStore(res);
- const row = load(req, res, 'create');
+ const row = await load(req, res, 'create');
if (!row) return;
if (!row) return;
try {
try {
- const { job, replayed } = system.retry(row.id);
+ const { job, replayed } = await system.retry(row.id);
res.set('Location', `/api/v1/jobs/${job.id}`);
res.set('Location', `/api/v1/jobs/${job.id}`);
if (replayed) res.set('Idempotent-Replayed', 'true');
if (replayed) res.set('Idempotent-Replayed', 'true');
- return res.status(replayed ? 200 : 202).json(system.view(job));
+ return res.status(replayed ? 200 : 202).json(await system.view(job));
} catch (err) {
} catch (err) {
if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);
if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);
console.error('[Jobs] retry failed:', err.message);
console.error('[Jobs] retry failed:', err.message);
@@ -218,13 +218,13 @@ function mountJobRoutes(app, o) {
@@ -218,13 +218,13 @@ function mountJobRoutes(app, o) {
});
});
function referenceRoute(fn) {
function referenceRoute(fn) {
- return (req, res) => {
+ return async (req, res) => {
noStore(res);
noStore(res);
- const row = load(req, res, 'create');
+ const row = await load(req, res, 'create');
if (!row) return;
if (!row) return;
try {
try {
- const r = fn(row.id, String(req.params.ref || ''));
- return res.status(r.created ? 201 : 200).json(system.view(r.job));
+ const r = await fn(row.id, String(req.params.ref || ''));
+ return res.status(r.created ? 201 : 200).json(await system.view(r.job));
} catch (err) {
} catch (err) {
if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);
if (err instanceof system.JobError) return send(res, err.status, err.code, err.detail, err.extra);
console.error('[Jobs] reference failed:', err.message);
console.error('[Jobs] reference failed:', err.message);
@@ -235,12 +235,12 @@ function mountJobRoutes(app, o) {
@@ -235,12 +235,12 @@ function mountJobRoutes(app, o) {
app.put('/api/v1/jobs/:id/references/:ref', origin, referenceRoute((id, ref) => system.reference(id, ref)));
app.put('/api/v1/jobs/:id/references/:ref', origin, referenceRoute((id, ref) => system.reference(id, ref)));
app.delete('/api/v1/jobs/:id/references/:ref', origin, referenceRoute((id, ref) => system.unreference(id, ref)));
app.delete('/api/v1/jobs/:id/references/:ref', origin, referenceRoute((id, ref) => system.unreference(id, ref)));
- app.get('/api/v1/jobs/:id/events', (req, res) => {
- const row = load(req, res, 'read');
+ app.get('/api/v1/jobs/:id/events', async (req, res) => {
+ const row = await load(req, res, 'read');
if (!row) return;
if (!row) return;
const raw = req.headers['last-event-id'] != null ? req.headers['last-event-id'] : req.query.last_event_id;
const raw = req.headers['last-event-id'] != null ? req.headers['last-event-id'] : req.query.last_event_id;
const after = Math.max(0, parseInt(raw, 10) || 0);
const after = Math.max(0, parseInt(raw, 10) || 0);
- const pending = system.eventsAfter(row.id, after);
+ const pending = await system.eventsAfter(row.id, after);
// Finished and nothing new since the client's last event: 204 tells EventSource to stop reconnecting.
// Finished and nothing new since the client's last event: 204 tells EventSource to stop reconnecting.
if (TERMINAL.has(row.state) && !pending.length) return res.status(204).set('Cache-Control', 'no-store').end();
if (TERMINAL.has(row.state) && !pending.length) return res.status(204).set('Cache-Control', 'no-store').end();
@@ -264,7 +264,7 @@ function mountJobRoutes(app, o) {
@@ -264,7 +264,7 @@ function mountJobRoutes(app, o) {
};
};
// Subscribe before replaying so nothing emitted in between is lost; the seq check drops duplicates.
// Subscribe before replaying so nothing emitted in between is lost; the seq check drops duplicates.
const unsubscribe = system.subscribe(row.id, (e) => (replaying ? queue.push(e) : write(e)));
const unsubscribe = system.subscribe(row.id, (e) => (replaying ? queue.push(e) : write(e)));
- for (const e of system.eventsAfter(row.id, after)) write(e);
+ for (const e of await system.eventsAfter(row.id, after)) write(e);
replaying = false;
replaying = false;
queue.forEach(write);
queue.forEach(write);
const keepAlive = setInterval(() => { if (!closed) res.write(': keep-alive\n\n'); }, 15000);
const keepAlive = setInterval(() => { if (!closed) res.write(': keep-alive\n\n'); }, 15000);
@@ -273,7 +273,7 @@ function mountJobRoutes(app, o) {
@@ -273,7 +273,7 @@ function mountJobRoutes(app, o) {
});
});
app.get('/api/v1/jobs/:id/files/:n', async (req, res) => {
app.get('/api/v1/jobs/:id/files/:n', async (req, res) => {
- const row = load(req, res, 'read');
+ const row = await load(req, res, 'read');
if (!row) return;
if (!row) return;
if (row.state !== 'succeeded') return send(res, 409, 'tools.job.not_ready', `The job is ${row.state}`);
if (row.state !== 'succeeded') return send(res, 409, 'tools.job.not_ready', `The job is ${row.state}`);
const f = system.resultFile(row, parseInt(req.params.n, 10));
const f = system.resultFile(row, parseInt(req.params.n, 10));
@@ -1,10 +1,12 @@
@@ -1,10 +1,12 @@
'use strict';
'use strict';
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
// Tools job runtime (roadmap Wave 11). Shared by the satellites the way apps/_shared always is:
// Tools job runtime (roadmap Wave 11). Shared by the satellites the way apps/_shared always is:
-// required by relative path, no dependencies of its own — each app passes its better-sqlite3,
-// openvibe-contracts and openvibe-sdk. See system.js (lifecycle), http.js (routes), media.js (results
-// in Media), events.js (tools.job.* to OpenVibe.Events), usage.js (tools.usage.recorded, a developer
-// project's jobs per hour), client.js (browser helper).
+// required by relative path, no dependencies of its own — each app passes the one openvibe-sdk/db
+// handle (the shared `tools` database, plan T8), openvibe-contracts and openvibe-sdk. See system.js
+// (lifecycle), http.js (routes), media.js (results in Media), events.js (tools.job.* to OpenVibe.Events),
+// usage.js (tools.usage.recorded, a developer project's jobs per hour), client.js (browser helper).
+//
+// The job store is part of every app's one `tools` database: no per-satellite jobs.db.
//
//
// Environment (one /etc/openvibe/tools.env for every unit):
// Environment (one /etc/openvibe/tools.env for every unit):
// TOOLS_JOBS_CONCURRENCY jobs running at once per satellite (default 2)
// TOOLS_JOBS_CONCURRENCY jobs running at once per satellite (default 2)
@@ -65,14 +67,14 @@ function mediaOrigin(env = process.env) {
@@ -65,14 +67,14 @@ function mediaOrigin(env = process.env) {
}
}
/**
/**
- * One call per satellite: open <dataDir>/jobs.db, build the system, let the app define its job
- * types, mount the routes and start (boot recovery happens here).
+ * One call per satellite: build the system on the app's shared `tools` database, let the app define its
+ * job types, mount the routes and start (boot recovery happens here).
*
*
* @param {object} o
* @param {object} o
* @param {object} o.app Express app
* @param {object} o.app Express app
* @param {string} o.service 'img' | 'audio' | 'docs'
* @param {string} o.service 'img' | 'audio' | 'docs'
* @param {string} o.dataDir
* @param {string} o.dataDir
- * @param {Function} o.Database require('better-sqlite3')
+ * @param {object} o.db the app's shared openvibe-sdk/db handle (one `tools` database)
* @param {object} o.contracts require('openvibe-contracts')
* @param {object} o.contracts require('openvibe-contracts')
* @param {object} [o.sdk] require('openvibe-sdk') — the outbox relay to OpenVibe.Events
* @param {object} [o.sdk] require('openvibe-sdk') — the outbox relay to OpenVibe.Events
* @param {() => string|null} o.getPublicKey
* @param {() => string|null} o.getPublicKey
@@ -89,12 +91,14 @@ function mediaOrigin(env = process.env) {
@@ -89,12 +91,14 @@ function mediaOrigin(env = process.env) {
*/
*/
function setupJobs(o) {
function setupJobs(o) {
fs.mkdirSync(o.dataDir, { recursive: true });
fs.mkdirSync(o.dataDir, { recursive: true });
- const db = new o.Database(path.join(o.dataDir, 'jobs.db'));
+ const db = o.db;
+ const log = o.log || console;
const results = mediaFromEnv(o.contracts);
const results = mediaFromEnv(o.contracts);
if (results.reason && String(process.env.TOOLS_JOB_RESULTS || '').toLowerCase() === 'media') console.warn(`[Jobs] ${o.service}: results stay local — ${results.reason}`);
if (results.reason && String(process.env.TOOLS_JOB_RESULTS || '').toLowerCase() === 'media') console.warn(`[Jobs] ${o.service}: results stay local — ${results.reason}`);
- // tools.job.* → OpenVibe.Events through an outbox table in this same jobs.db (inert without EVENTS_URL).
- const events = outboxFromEnv({ db, sdk: o.sdk });
+ // tools.job.* → OpenVibe.Events through the `event_outbox` table in this same database (inert without EVENTS_URL).
+ const events = outboxFromEnv({ db, sdk: o.sdk, log });
if (events.reason && process.env.EVENTS_URL) console.warn(`[Jobs] ${o.service}: job events are not published — ${events.reason}`);
if (events.reason && process.env.EVENTS_URL) console.warn(`[Jobs] ${o.service}: job events are not published — ${events.reason}`);
+ if (events.outbox) events.outbox.ready.catch((err) => log.error(`[Jobs] ${o.service}: could not create the event outbox:`, err.message));
const system = createJobSystem({
const system = createJobSystem({
db, contracts: o.contracts, service: o.service, dataDir: o.dataDir,
db, contracts: o.contracts, service: o.service, dataDir: o.dataDir,
concurrency: envInt(`TOOLS_JOBS_CONCURRENCY_${o.service.toUpperCase()}`, envInt('TOOLS_JOBS_CONCURRENCY', 2)),
concurrency: envInt(`TOOLS_JOBS_CONCURRENCY_${o.service.toUpperCase()}`, envInt('TOOLS_JOBS_CONCURRENCY', 2)),
@@ -104,6 +108,7 @@ function setupJobs(o) {
@@ -104,6 +108,7 @@ function setupJobs(o) {
diskBudgetBytes: envInt('TOOLS_DISK_BUDGET_MB', 8192) * 1024 * 1024,
diskBudgetBytes: envInt('TOOLS_DISK_BUDGET_MB', 8192) * 1024 * 1024,
media: results.media,
media: results.media,
outbox: events.outbox,
outbox: events.outbox,
+ log,
});
});
o.define(system);
o.define(system);
const g = o.guard || null;
const g = o.guard || null;
@@ -117,19 +122,19 @@ function setupJobs(o) {
@@ -117,19 +122,19 @@ function setupJobs(o) {
onAddressFull: (req, res, full) => !g.refuse(req, res, { status: 429, code: 'tools.job.too_many_active', reason: 'jobs.address', tool: o.jobTool ? o.jobTool(req, null, {}) : null, retryAfter: 30, detail: `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish.`, extra: { scope: 'address' } }),
onAddressFull: (req, res, full) => !g.refuse(req, res, { status: 429, code: 'tools.job.too_many_active', reason: 'jobs.address', tool: o.jobTool ? o.jobTool(req, null, {}) : null, retryAfter: 30, detail: `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish.`, extra: { scope: 'address' } }),
}),
}),
});
});
- system.start();
+ system.start().catch((err) => log.error(`[Jobs] ${o.service}: boot recovery failed:`, err.message));
let pruneOutbox = null;
let pruneOutbox = null;
if (events.outbox) {
if (events.outbox) {
events.outbox.start();
events.outbox.start();
- pruneOutbox = setInterval(() => { try { events.outbox.prune(); } catch { /* next time */ } }, 6 * 60 * 60 * 1000);
+ pruneOutbox = setInterval(() => { events.outbox.prune().catch(() => { /* next time */ }); }, 6 * 60 * 60 * 1000);
if (pruneOutbox.unref) pruneOutbox.unref();
if (pruneOutbox.unref) pruneOutbox.unref();
- console.log(`[Jobs] ${o.service}: job events → ${events.outbox.url} (${events.outbox.pending()} pending)`);
+ console.log(`[Jobs] ${o.service}: job events → ${events.outbox.url}`);
}
}
system.db = db;
system.db = db;
+ // The `tools` database is shared with the rest of the app and closed there; this only stops the worker.
system.close = () => {
system.close = () => {
system.stop();
system.stop();
- if (events.outbox) { events.outbox.stop(); clearInterval(pruneOutbox); }
- try { db.close(); } catch { /* already closed */ }
+ if (events.outbox) { events.outbox.stop().catch(() => {}); clearInterval(pruneOutbox); }
};
};
return system;
return system;
}
}
@@ -1,173 +1,118 @@
@@ -1,173 +1,118 @@
'use strict';
'use strict';
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
-// Job store — the satellite's own SQLite (better-sqlite3 handle passed in).
+// Job store — the one `tools` PostgreSQL database (plan T8, decision 3), through openvibe-sdk/db.
//
//
-// Three tables, created on first use (idempotent):
+// Three tables, all created by migrations/0001_tools.sql (not here):
// tool_jobs one row per job; the durable truth a restart recovers from
// tool_jobs one row per job; the durable truth a restart recovers from
-// tool_job_events the ordered event log per job; its AUTOINCREMENT seq is the SSE event id,
+// tool_job_events the ordered event log per job; its per-job identity seq is the SSE event id,
// so a reconnect with Last-Event-ID replays exactly what was missed
// so a reconnect with Last-Event-ID replays exactly what was missed
// tool_job_references who still points at a job's result (a paste, a project, …); a referenced
// tool_job_references who still points at a job's result (a paste, a project, …); a referenced
// job never expires, so its files and Media objects are not pruned
// job never expires, so its files and Media objects are not pruned
+//
+// Eight apps share the one database, so every row carries `app` and every read that could see another
+// app's rows filters on it. All statements are async (openvibe-sdk/db) and join an ambient transaction
+// when one is open, so a store call inside db.tx(fn) commits with it.
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
const TERMINAL = ['succeeded', 'failed', 'cancelled'];
const TERMINAL = ['succeeded', 'failed', 'cancelled'];
const STATES = ['queued', 'running', ...TERMINAL];
const STATES = ['queued', 'running', ...TERMINAL];
-const SCHEMA = `
-CREATE TABLE IF NOT EXISTS tool_jobs (
- id TEXT PRIMARY KEY,
- type TEXT NOT NULL,
- type_version INTEGER NOT NULL DEFAULT 1,
- owner TEXT NOT NULL,
- state TEXT NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled')),
- progress REAL,
- progress_message TEXT,
- input_json TEXT NOT NULL DEFAULT '{}',
- files_json TEXT NOT NULL DEFAULT '[]',
- result_json TEXT,
- error_json TEXT,
- retryable INTEGER NOT NULL DEFAULT 0,
- idempotency_key TEXT,
- request_hash TEXT,
- attempts INTEGER NOT NULL DEFAULT 0,
- max_attempts INTEGER NOT NULL DEFAULT 1,
- cancel_requested INTEGER NOT NULL DEFAULT 0,
- ttl_ms INTEGER NOT NULL DEFAULT 3600000,
- created_at INTEGER NOT NULL,
- updated_at INTEGER NOT NULL,
- started_at INTEGER,
- finished_at INTEGER,
- expires_at INTEGER,
- env TEXT NOT NULL DEFAULT 'production' CHECK (env IN ('production','sandbox'))
-);
-CREATE UNIQUE INDEX IF NOT EXISTS tool_jobs_idem ON tool_jobs(owner, idempotency_key) WHERE idempotency_key IS NOT NULL;
-CREATE INDEX IF NOT EXISTS tool_jobs_state ON tool_jobs(state, id);
-CREATE INDEX IF NOT EXISTS tool_jobs_expiry ON tool_jobs(expires_at) WHERE expires_at IS NOT NULL;
-CREATE INDEX IF NOT EXISTS tool_jobs_owner ON tool_jobs(owner, state);
-CREATE TABLE IF NOT EXISTS tool_job_events (
- seq INTEGER PRIMARY KEY AUTOINCREMENT,
- job_id TEXT NOT NULL,
- event TEXT NOT NULL,
- data TEXT NOT NULL,
- at INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS tool_job_events_job ON tool_job_events(job_id, seq);
-CREATE TABLE IF NOT EXISTS tool_job_references (
- job_id TEXT NOT NULL,
- ref TEXT NOT NULL,
- created_at INTEGER NOT NULL,
- PRIMARY KEY (job_id, ref)
-);
-`;
-
-// Columns added after the first release, each added when missing (idempotent).
-const ADDED_COLUMNS = [
- ['env', "TEXT NOT NULL DEFAULT 'production' CHECK (env IN ('production','sandbox'))"], // developer-app sandboxes
- ['retry_of', 'TEXT'], // the failed job this one retries
- ['retried_by', 'TEXT'], // set on a failed job once it has been retried: the retry's id
- ['ip_key', 'TEXT'], // a browser session's job: the guard's hashed address key (never the address), so
- // unfinished jobs are also bounded per address (a new cookie is no new allowance)
- ['tool', 'TEXT'], // the tool (tools.tool@1 id) whose run created the job (POST /api/v1/tools/:id/run)
- ['project_id', 'TEXT'], // a developer app's job: its project (prj_…), so its results go to Media under the
- // project's namespace, metered per project (WS-L task 5)
- ['trace_id', 'TEXT'], // the trace id of the request that submitted the job, sampled with a developer
- // project's failures in its usage rollups (./usage.js, WS-N task 4)
-];
-
const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };
const parse = (s, fallback) => { if (s == null) return fallback; try { return JSON.parse(s); } catch { return fallback; } };
-function createStore(db) {
- db.pragma('journal_mode = WAL');
- db.pragma('busy_timeout = 5000');
- db.exec(SCHEMA);
- const have = new Set(db.prepare('PRAGMA table_info(tool_jobs)').all().map((c) => c.name));
- for (const [name, decl] of ADDED_COLUMNS) if (!have.has(name)) db.exec(`ALTER TABLE tool_jobs ADD COLUMN ${name} ${decl}`);
- db.exec("CREATE INDEX IF NOT EXISTS tool_jobs_ip ON tool_jobs(ip_key, state) WHERE ip_key IS NOT NULL");
-
+/**
+ * @param {object} db openvibe-sdk/db handle (or the gated one apps/_shared/db.js returns)
+ * @param {object} o
+ * @param {string} o.app the owning app ('img' | 'audio' | 'docs' | …): every row this store writes or reads
+ */
+function createStore(db, { app } = {}) {
+ if (!db || typeof db.prepare !== 'function') throw new TypeError('createStore needs an openvibe-sdk/db handle');
+ if (!app) throw new TypeError('createStore needs app');
const q = {
const q = {
- insert: db.prepare(`INSERT INTO tool_jobs (id, type, type_version, owner, state, input_json, files_json, idempotency_key, request_hash, max_attempts, ttl_ms, created_at, updated_at, env, retry_of, ip_key, tool, project_id, trace_id)
- VALUES (@id, @type, @type_version, @owner, 'queued', @input_json, @files_json, @idempotency_key, @request_hash, @max_attempts, @ttl_ms, @now, @now, @env, @retry_of, @ip_key, @tool, @project_id, @trace_id)`),
+ insert: db.prepare(`INSERT INTO tool_jobs (id, app, type, type_version, owner, state, input_json, files_json, idempotency_key, request_hash, max_attempts, ttl_ms, created_at, updated_at, env, retry_of, ip_key, tool, project_id, trace_id)
+ VALUES (@id, @app, @type, @type_version, @owner, 'queued', @input_json, @files_json, @idempotency_key, @request_hash, @max_attempts, @ttl_ms, @now, @now, @env, @retry_of, @ip_key, @tool, @project_id, @trace_id)`),
markRetried: db.prepare("UPDATE tool_jobs SET retried_by = @next, updated_at = @now WHERE id = @id AND state = 'failed' AND retried_by IS NULL"),
markRetried: db.prepare("UPDATE tool_jobs SET retried_by = @next, updated_at = @now WHERE id = @id AND state = 'failed' AND retried_by IS NULL"),
- get: db.prepare('SELECT * FROM tool_jobs WHERE id = ?'),
- byIdem: db.prepare('SELECT * FROM tool_jobs WHERE owner = ? AND idempotency_key = ?'),
+ get: db.prepare('SELECT * FROM tool_jobs WHERE id = ? AND app = ?'),
+ byIdem: db.prepare('SELECT * FROM tool_jobs WHERE app = @app AND owner = @owner AND idempotency_key = @idempotency_key'),
// A result file stored in Media, by its media_id (a run's { media_id } reference); the owner's only.
// A result file stored in Media, by its media_id (a run's { media_id } reference); the owner's only.
- byResultMedia: db.prepare("SELECT * FROM tool_jobs WHERE owner = ? AND state = 'succeeded' AND result_json LIKE ? ORDER BY id DESC LIMIT 5"),
- nextQueued: db.prepare("SELECT * FROM tool_jobs WHERE state = 'queued' ORDER BY id LIMIT ?"),
+ byResultMedia: db.prepare("SELECT * FROM tool_jobs WHERE app = @app AND owner = @owner AND state = 'succeeded' AND result_json ILIKE @like ORDER BY id DESC LIMIT 5"),
+ nextQueued: db.prepare("SELECT * FROM tool_jobs WHERE app = @app AND state = 'queued' ORDER BY id LIMIT @limit"),
claim: db.prepare("UPDATE tool_jobs SET state = 'running', attempts = attempts + 1, started_at = @now, updated_at = @now, progress = NULL, progress_message = NULL WHERE id = @id AND state = 'queued'"),
claim: db.prepare("UPDATE tool_jobs SET state = 'running', attempts = attempts + 1, started_at = @now, updated_at = @now, progress = NULL, progress_message = NULL WHERE id = @id AND state = 'queued'"),
- running: db.prepare("SELECT * FROM tool_jobs WHERE state = 'running'"),
- activeForOwner: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE owner = ? AND state IN ('queued','running')"),
- activeForIp: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE ip_key = ? AND state IN ('queued','running')"),
+ running: db.prepare("SELECT * FROM tool_jobs WHERE app = @app AND state = 'running'"),
+ activeForOwner: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE app = @app AND owner = @owner AND state IN ('queued','running')"),
+ activeForIp: db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE app = @app AND ip_key = @ip_key AND state IN ('queued','running')"),
progress: db.prepare('UPDATE tool_jobs SET progress = @progress, progress_message = @message, updated_at = @now WHERE id = @id AND state = \'running\''),
progress: db.prepare('UPDATE tool_jobs SET progress = @progress, progress_message = @message, updated_at = @now WHERE id = @id AND state = \'running\''),
requestCancel: db.prepare("UPDATE tool_jobs SET cancel_requested = 1, updated_at = @now WHERE id = @id AND state IN ('queued','running')"),
requestCancel: db.prepare("UPDATE tool_jobs SET cancel_requested = 1, updated_at = @now WHERE id = @id AND state IN ('queued','running')"),
requeue: db.prepare("UPDATE tool_jobs SET state = 'queued', started_at = NULL, progress = NULL, progress_message = NULL, updated_at = @now WHERE id = @id AND state = 'running'"),
requeue: db.prepare("UPDATE tool_jobs SET state = 'queued', started_at = NULL, progress = NULL, progress_message = NULL, updated_at = @now WHERE id = @id AND state = 'running'"),
finish: db.prepare(`UPDATE tool_jobs SET state = @state, result_json = @result_json, error_json = @error_json, retryable = @retryable,
finish: db.prepare(`UPDATE tool_jobs SET state = @state, result_json = @result_json, error_json = @error_json, retryable = @retryable,
progress = CASE WHEN @state = 'succeeded' THEN 100 ELSE progress END, progress_message = NULL,
progress = CASE WHEN @state = 'succeeded' THEN 100 ELSE progress END, progress_message = NULL,
- finished_at = @now, updated_at = @now, expires_at = @now + ttl_ms WHERE id = @id AND state IN ('queued','running')`),
+ finished_at = @now, updated_at = @now, expires_at = @now + ttl_ms::bigint WHERE id = @id AND state IN ('queued','running')`),
setFiles: db.prepare('UPDATE tool_jobs SET files_json = @files_json, updated_at = @now WHERE id = @id'),
setFiles: db.prepare('UPDATE tool_jobs SET files_json = @files_json, updated_at = @now WHERE id = @id'),
- event: db.prepare('INSERT INTO tool_job_events (job_id, event, data, at) VALUES (?, ?, ?, ?)'),
+ event: db.prepare('INSERT INTO tool_job_events (job_id, event, data, at) VALUES (@job_id, @event, @data, @at) RETURNING seq'),
eventsAfter: db.prepare('SELECT seq, event, data, at FROM tool_job_events WHERE job_id = ? AND seq > ? ORDER BY seq'),
eventsAfter: db.prepare('SELECT seq, event, data, at FROM tool_job_events WHERE job_id = ? AND seq > ? ORDER BY seq'),
lastSeq: db.prepare('SELECT MAX(seq) AS seq FROM tool_job_events WHERE job_id = ?'),
lastSeq: db.prepare('SELECT MAX(seq) AS seq FROM tool_job_events WHERE job_id = ?'),
// A job something still references is never expired (see tool_job_references).
// A job something still references is never expired (see tool_job_references).
- expired: db.prepare(`SELECT * FROM tool_jobs j WHERE expires_at IS NOT NULL AND expires_at <= ?
+ expired: db.prepare(`SELECT * FROM tool_jobs j WHERE app = @app AND expires_at IS NOT NULL AND expires_at <= @now
AND NOT EXISTS (SELECT 1 FROM tool_job_references r WHERE r.job_id = j.id) LIMIT 500`),
AND NOT EXISTS (SELECT 1 FROM tool_job_references r WHERE r.job_id = j.id) LIMIT 500`),
deferExpiry: db.prepare('UPDATE tool_jobs SET expires_at = @until, updated_at = @now WHERE id = @id'),
deferExpiry: db.prepare('UPDATE tool_jobs SET expires_at = @until, updated_at = @now WHERE id = @id'),
del: db.prepare('DELETE FROM tool_jobs WHERE id = ?'),
del: db.prepare('DELETE FROM tool_jobs WHERE id = ?'),
delEvents: db.prepare('DELETE FROM tool_job_events WHERE job_id = ?'),
delEvents: db.prepare('DELETE FROM tool_job_events WHERE job_id = ?'),
delRefs: db.prepare('DELETE FROM tool_job_references WHERE job_id = ?'),
delRefs: db.prepare('DELETE FROM tool_job_references WHERE job_id = ?'),
- addRef: db.prepare('INSERT OR IGNORE INTO tool_job_references (job_id, ref, created_at) VALUES (?, ?, ?)'),
+ addRef: db.prepare('INSERT INTO tool_job_references (job_id, ref, created_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING'),
dropRef: db.prepare('DELETE FROM tool_job_references WHERE job_id = ? AND ref = ?'),
dropRef: db.prepare('DELETE FROM tool_job_references WHERE job_id = ? AND ref = ?'),
refs: db.prepare('SELECT ref, created_at FROM tool_job_references WHERE job_id = ? ORDER BY created_at, ref'),
refs: db.prepare('SELECT ref, created_at FROM tool_job_references WHERE job_id = ? ORDER BY created_at, ref'),
refCount: db.prepare('SELECT COUNT(*) AS n FROM tool_job_references WHERE job_id = ?'),
refCount: db.prepare('SELECT COUNT(*) AS n FROM tool_job_references WHERE job_id = ?'),
// After the last reference goes, the result is kept at least one more ttl.
// After the last reference goes, the result is kept at least one more ttl.
- releaseExpiry: db.prepare('UPDATE tool_jobs SET expires_at = MAX(COALESCE(expires_at, 0), @now + ttl_ms), updated_at = @now WHERE id = @id AND finished_at IS NOT NULL'),
- counts: db.prepare('SELECT state, COUNT(*) AS n FROM tool_jobs GROUP BY state'),
+ releaseExpiry: db.prepare('UPDATE tool_jobs SET expires_at = GREATEST(COALESCE(expires_at, 0), @now + ttl_ms::bigint), updated_at = @now WHERE id = @id AND finished_at IS NOT NULL'),
+ counts: db.prepare('SELECT state, COUNT(*) AS n FROM tool_jobs WHERE app = @app GROUP BY state'),
};
};
/** Append one event; returns its seq (the SSE id). */
/** Append one event; returns its seq (the SSE id). */
- function appendEvent(jobId, event, data, now = Date.now()) {
- return Number(q.event.run(jobId, event, JSON.stringify(data), now).lastInsertRowid);
+ async function appendEvent(jobId, event, data, now = Date.now()) {
+ const r = await q.event.run({ job_id: jobId, event, data: JSON.stringify(data), at: now });
+ return Number(r.lastInsertRowid);
}
}
return {
return {
db,
db,
+ app,
TERMINAL, STATES,
TERMINAL, STATES,
- insert(row) { q.insert.run({ retry_of: null, ip_key: null, tool: null, project_id: null, trace_id: null, ...row }); },
- byResultMedia: (owner, mediaId) => q.byResultMedia.all(owner, `%"media_id":"${String(mediaId).replace(/[%"\\]/g, '')}"%`),
- markRetried: (id, next, now = Date.now()) => q.markRetried.run({ id, next, now }).changes === 1,
- get: (id) => q.get.get(id) || null,
- byIdempotencyKey: (owner, key) => q.byIdem.get(owner, key) || null,
- nextQueued: (limit) => q.nextQueued.all(limit),
- claim: (id, now = Date.now()) => q.claim.run({ id, now }).changes === 1,
- running: () => q.running.all(),
- activeForOwner: (owner) => q.activeForOwner.get(owner).n,
- activeForIp: (ipKey) => (ipKey ? q.activeForIp.get(ipKey).n : 0),
- setProgress: (id, progress, message, now = Date.now()) => q.progress.run({ id, progress, message, now }).changes === 1,
- requestCancel: (id, now = Date.now()) => q.requestCancel.run({ id, now }).changes === 1,
- requeue: (id, now = Date.now()) => q.requeue.run({ id, now }).changes === 1,
- finish: (id, { state, result = null, error = null, retryable = false }, now = Date.now()) => q.finish.run({
+ async insert(row) { await q.insert.run({ app, retry_of: null, ip_key: null, tool: null, project_id: null, trace_id: null, ...row }); },
+ byResultMedia: async (owner, mediaId) => q.byResultMedia.all({ app, owner, like: `%"media_id":"${String(mediaId).replace(/[%"\\]/g, '')}"%` }),
+ markRetried: async (id, next, now = Date.now()) => (await q.markRetried.run({ id, next, now })).changes === 1,
+ get: async (id) => (await q.get.get(id, app)) || null,
+ byIdempotencyKey: async (owner, key) => (await q.byIdem.get({ app, owner, idempotency_key: key })) || null,
+ nextQueued: (limit) => q.nextQueued.all({ app, limit }),
+ claim: async (id, now = Date.now()) => (await q.claim.run({ id, now })).changes === 1,
+ running: () => q.running.all({ app }),
+ activeForOwner: async (owner) => (await q.activeForOwner.get({ app, owner })).n,
+ activeForIp: async (ipKey) => (ipKey ? (await q.activeForIp.get({ app, ip_key: ipKey })).n : 0),
+ setProgress: async (id, progress, message, now = Date.now()) => (await q.progress.run({ id, progress, message, now })).changes === 1,
+ requestCancel: async (id, now = Date.now()) => (await q.requestCancel.run({ id, now })).changes === 1,
+ requeue: async (id, now = Date.now()) => (await q.requeue.run({ id, now })).changes === 1,
+ finish: async (id, { state, result = null, error = null, retryable = false }, now = Date.now()) => (await q.finish.run({
id, state, now, retryable: retryable ? 1 : 0,
id, state, now, retryable: retryable ? 1 : 0,
result_json: result == null ? null : JSON.stringify(result),
result_json: result == null ? null : JSON.stringify(result),
error_json: error == null ? null : JSON.stringify(error),
error_json: error == null ? null : JSON.stringify(error),
- }).changes === 1,
+ })).changes === 1,
setFiles: (id, files, now = Date.now()) => q.setFiles.run({ id, files_json: JSON.stringify(files), now }),
setFiles: (id, files, now = Date.now()) => q.setFiles.run({ id, files_json: JSON.stringify(files), now }),
appendEvent,
appendEvent,
- eventsAfter: (jobId, seq) => q.eventsAfter.all(jobId, seq).map(e => ({ seq: e.seq, event: e.event, data: parse(e.data, {}), at: e.at })),
- lastSeq: (jobId) => q.lastSeq.get(jobId).seq || 0,
- expired: (now = Date.now()) => q.expired.all(now),
+ eventsAfter: async (jobId, seq) => (await q.eventsAfter.all(jobId, seq)).map(e => ({ seq: Number(e.seq), event: e.event, data: parse(e.data, {}), at: Number(e.at) })),
+ lastSeq: async (jobId) => Number((await q.lastSeq.get(jobId)).seq) || 0,
+ expired: (now = Date.now()) => q.expired.all({ app, now }),
deferExpiry: (id, until, now = Date.now()) => q.deferExpiry.run({ id, until, now }),
deferExpiry: (id, until, now = Date.now()) => q.deferExpiry.run({ id, until, now }),
- remove: db.transaction((id) => { q.delEvents.run(id); q.delRefs.run(id); q.del.run(id); }),
+ remove: (id) => db.tx(async () => { await q.delEvents.run(id); await q.delRefs.run(id); await q.del.run(id); }),
/** → true when the reference is new. */
/** → true when the reference is new. */
- addReference: (id, ref, now = Date.now()) => q.addRef.run(id, ref, now).changes === 1,
+ addReference: async (id, ref, now = Date.now()) => (await q.addRef.run(id, ref, now)).changes === 1,
/** → true when it existed; dropping the last one restarts the job's expiry clock. */
/** → true when it existed; dropping the last one restarts the job's expiry clock. */
- dropReference: db.transaction((id, ref, now = Date.now()) => {
- const gone = q.dropRef.run(id, ref).changes === 1;
- if (gone && q.refCount.get(id).n === 0) q.releaseExpiry.run({ id, now });
+ dropReference: (id, ref, now = Date.now()) => db.tx(async () => {
+ const gone = (await q.dropRef.run(id, ref)).changes === 1;
+ if (gone && (await q.refCount.get(id)).n === 0) await q.releaseExpiry.run({ id, now });
return gone;
return gone;
}),
}),
references: (id) => q.refs.all(id),
references: (id) => q.refs.all(id),
- referenceCount: (id) => q.refCount.get(id).n,
- counts() { const out = Object.fromEntries(STATES.map(s => [s, 0])); for (const r of q.counts.all()) out[r.state] = r.n; return out; },
- transaction: (fn) => db.transaction(fn),
+ referenceCount: async (id) => (await q.refCount.get(id)).n,
+ counts: async () => { const out = Object.fromEntries(STATES.map(s => [s, 0])); for (const r of await q.counts.all({ app })) out[r.state] = Number(r.n); return out; },
+ tx: (fn) => db.tx(fn),
parse,
parse,
};
};
}
}
@@ -4,34 +4,34 @@
@@ -4,34 +4,34 @@
//
//
// accepted (202) → queued → running → succeeded | failed | cancelled
// accepted (202) → queued → running → succeeded | failed | cancelled
//
//
-// Durable: a job is written to the satellite's SQLite, and its input files moved under
-// <dataDir>/jobs/<id>/in/, BEFORE the 202 goes out. On boot, rows left 'running' by a restart
-// are re-queued (onRestart: 'requeue', while attempts remain) or failed with retryable = true
-// (onRestart: 'fail'), per job type. Queued rows simply start again.
+// Durable: a job is written to the one `tools` PostgreSQL database (plan T8), and its input files moved
+// under <dataDir>/jobs/<id>/in/, BEFORE the 202 goes out. On boot, rows left 'running' by a restart are
+// re-queued (onRestart: 'requeue', while attempts remain) or failed with retryable = true (onRestart:
+// 'fail'), per job type. Queued rows simply start again.
//
//
-// Bounded: at most `concurrency` jobs run at once in this process, and each owner may have at
-// most `maxActivePerOwner` queued + running jobs. busy() says when the whole store is over its
-// bounds — `maxQueued` jobs waiting (all owners), or `diskBudgetBytes` under <dataDir>/jobs — so the
-// HTTP layer can answer 503 tools.busy with Retry-After before it accepts an upload (through the
-// guard: refused in enforce mode, recorded in report mode). Finished jobs expire (ttl chosen at submit),
-// and the pruner deletes the row, its events, its files and any Media objects it made — except
-// while something references the result (reference(), e.g. a paste or a project that points at the
-// Media object), and never while Media keeps an object (a retention hold, or Media unreachable):
-// then the job is kept and looked at again later, so no referenced result loses its record.
+// Bounded: at most `concurrency` jobs run at once in this process, and each owner may have at most
+// `maxActivePerOwner` queued + running jobs. busy() says when the whole store is over its bounds —
+// `maxQueued` jobs waiting (all owners), or `diskBudgetBytes` under <dataDir>/jobs — so the HTTP layer can
+// answer 503 tools.busy with Retry-After before it accepts an upload (through the guard: refused in
+// enforce mode, recorded in report mode). Finished jobs expire (ttl chosen at submit), and the pruner
+// deletes the row, its events, its files and any Media objects it made — except while something
+// references the result (reference(), e.g. a paste or a project that points at the Media object), and
+// never while Media keeps an object (a retention hold, or Media unreachable): then the job is kept and
+// looked at again later, so no referenced result loses its record.
//
//
// Retry: a failed job keeps its input files until it expires; retry(id) moves them to a new job
// Retry: a failed job keeps its input files until it expires; retry(id) moves them to a new job
-// (retry_of → the failed one, which records retried_by). Retrying the same failed job again
-// returns that same new job, so the call is idempotent.
+// (retry_of → the failed one, which records retried_by). Retrying the same failed job again returns
+// that same new job, so the call is idempotent.
//
//
-// Events: with an outbox (./events, openvibe-sdk createOutbox on this same database), created,
-// started, succeeded and failed are also announced to OpenVibe.Events as tools.job.*, each written
-// in the transaction that records the transition. Without one, nothing is announced.
+// Events: with an outbox (./events, openvibe-sdk createPgOutbox on this same database), created,
+// started, succeeded and failed are also announced to OpenVibe.Events as tools.job.*, each written in
+// the transaction that records the transition. Without one, nothing is announced.
//
//
// Usage (WS-N task 4): with an outbox, a developer project's job that ends is also counted in the
// Usage (WS-N task 4): with an outbox, a developer project's job that ends is also counted in the
// transaction that records its end (./usage.js), and each closed hour goes out as tools.usage.recorded
// transaction that records its end (./usage.js), and each closed hour goes out as tools.usage.recorded
// from the pruner's timer.
// from the pruner's timer.
//
//
-// No dependencies of its own: the app passes its better-sqlite3 handle and openvibe-contracts.
+// No dependencies of its own: the app passes the shared openvibe-sdk/db handle and openvibe-contracts.
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
const fs = require('fs');
const fs = require('fs');
@@ -86,9 +86,9 @@ const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]')
@@ -86,9 +86,9 @@ const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]')
/**
/**
* @param {object} o
* @param {object} o
- * @param {object} o.db better-sqlite3 Database (the satellite's own jobs database)
+ * @param {object} o.db openvibe-sdk/db handle (the one tools database)
+ * @param {string} o.service 'img' | 'audio' | 'docs' | … (also the store's `app`)
* @param {object} o.contracts require('openvibe-contracts')
* @param {object} o.contracts require('openvibe-contracts')
- * @param {string} o.service 'img' | 'audio' | 'docs' | …
* @param {string} o.dataDir where job files live (<dataDir>/jobs/<id>/…)
* @param {string} o.dataDir where job files live (<dataDir>/jobs/<id>/…)
* @param {number} [o.concurrency=2]
* @param {number} [o.concurrency=2]
* @param {number} [o.maxActivePerOwner=10]
* @param {number} [o.maxActivePerOwner=10]
@@ -104,7 +104,7 @@ const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]')
@@ -104,7 +104,7 @@ const scrub = (msg) => String(msg || '').replace(/(?:\/[\w.-]+){2,}/g, '[file]')
function createJobSystem(o) {
function createJobSystem(o) {
if (!o || !o.db || !o.contracts || !o.dataDir) throw new TypeError('createJobSystem needs db, contracts and dataDir');
if (!o || !o.db || !o.contracts || !o.dataDir) throw new TypeError('createJobSystem needs db, contracts and dataDir');
const { contracts } = o;
const { contracts } = o;
- const store = createStore(o.db);
+ const store = createStore(o.db, { app: o.service });
const log = o.log || console;
const log = o.log || console;
const concurrency = Math.max(0, Number.isFinite(o.concurrency) ? o.concurrency : 2);
const concurrency = Math.max(0, Number.isFinite(o.concurrency) ? o.concurrency : 2);
const maxActivePerOwner = Math.max(1, o.maxActivePerOwner || 10);
const maxActivePerOwner = Math.max(1, o.maxActivePerOwner || 10);
@@ -117,17 +117,18 @@ function createJobSystem(o) {
@@ -117,17 +117,18 @@ function createJobSystem(o) {
const diskBudget = o.diskBudgetBytes > 0 ? o.diskBudgetBytes : 0;
const diskBudget = o.diskBudgetBytes > 0 ? o.diskBudgetBytes : 0;
const disk = { bytes: 0, at: 0, scanning: null };
const disk = { bytes: 0, at: 0, scanning: null };
const announce = createJobEvents({ contracts, service: o.service, outbox: o.outbox || null, referenceCount: (id) => store.referenceCount(id), log });
const announce = createJobEvents({ contracts, service: o.service, outbox: o.outbox || null, referenceCount: (id) => store.referenceCount(id), log });
- const usage = createJobUsage({ db: o.db, contracts, outbox: o.outbox || null, log });
- /** Inside the transaction that recorded the end: tools.job.succeeded|failed and the project's usage. */
- function ended(id) {
- const row = store.get(id);
- announce.finished(row);
- usage.finished(row);
+ const usage = createJobUsage({ db: o.db, app: o.service, contracts, outbox: o.outbox || null, log });
+ /** Inside the transaction `t` that recorded the end: tools.job.succeeded|failed and the project's usage. */
+ async function ended(t, id) {
+ const row = await store.get(id);
+ await announce.finished(t, row);
+ await usage.finished(row);
}
}
const types = new Map();
const types = new Map();
const active = new Map(); // id → { ctrl, reason, last: { pct, msg, at } }
const active = new Map(); // id → { ctrl, reason, last: { pct, msg, at } }
const listeners = new Map(); // id → Set<fn>
const listeners = new Map(); // id → Set<fn>
+ const emitChains = new Map(); // id → the tail of this job's event appends (strict per-job order)
const stopHooks = new Set(); // e.g. open SSE streams, ended on stop()
const stopHooks = new Set(); // e.g. open SSE streams, ended on stop()
let started = false, stopped = false, pruneTimer = null, kicking = false;
let started = false, stopped = false, pruneTimer = null, kicking = false;
@@ -151,7 +152,7 @@ function createJobSystem(o) {
@@ -151,7 +152,7 @@ function createJobSystem(o) {
}
}
// ── Public representation ─────────────────────────────────
// ── Public representation ─────────────────────────────────
- const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());
+ const iso = (ms) => (ms == null ? null : new Date(Number(ms)).toISOString());
function publicResult(result, id) {
function publicResult(result, id) {
if (!result) return null;
if (!result) return null;
return {
return {
@@ -164,21 +165,21 @@ function createJobSystem(o) {
@@ -164,21 +165,21 @@ function createJobSystem(o) {
data: result.data || {},
data: result.data || {},
};
};
}
}
- function view(row) {
+ async function view(row) {
if (!row) return null;
if (!row) return null;
const live = row.state === 'queued' || row.state === 'running';
const live = row.state === 'queued' || row.state === 'running';
- const references = store.references(row.id).map(r => ({ ref: r.ref, created_at: iso(r.created_at) }));
+ const references = (await store.references(row.id)).map(r => ({ ref: r.ref, created_at: iso(r.created_at) }));
return {
return {
id: row.id,
id: row.id,
object: 'tools.job',
object: 'tools.job',
service: o.service,
service: o.service,
...(row.tool && { tool: row.tool }),
...(row.tool && { tool: row.tool }),
type: row.type,
type: row.type,
- type_version: row.type_version,
+ type_version: Number(row.type_version),
state: row.state,
state: row.state,
- progress: { percent: row.progress == null ? null : Math.round(row.progress * 10) / 10, message: row.progress_message || null },
- attempts: row.attempts,
- max_attempts: row.max_attempts,
+ progress: { percent: row.progress == null ? null : Math.round(Number(row.progress) * 10) / 10, message: row.progress_message || null },
+ attempts: Number(row.attempts),
+ max_attempts: Number(row.max_attempts),
cancel_requested: !!row.cancel_requested,
cancel_requested: !!row.cancel_requested,
created_at: iso(row.created_at),
created_at: iso(row.created_at),
started_at: iso(row.started_at),
started_at: iso(row.started_at),
@@ -202,13 +203,20 @@ function createJobSystem(o) {
@@ -202,13 +203,20 @@ function createJobSystem(o) {
}
}
// ── Events ─────────────────────────────────────────────────
// ── Events ─────────────────────────────────────────────────
+ /** Append this job's next SSE event and tell its listeners; appends are serialized per job. */
function emit(id, event) {
function emit(id, event) {
- if (stopped) return;
- const data = view(store.get(id));
- if (!data) return;
- const seq = store.appendEvent(id, event, data);
- const set = listeners.get(id);
- if (set) for (const fn of [...set]) { try { fn({ seq, event, data }); } catch (err) { log.error('[Jobs] listener error:', err.message); } }
+ const prev = emitChains.get(id) || Promise.resolve();
+ const next = prev.then(async () => {
+ if (stopped) return;
+ const data = await view(await store.get(id));
+ if (!data) return;
+ const seq = await store.appendEvent(id, event, data);
+ const set = listeners.get(id);
+ if (set) for (const fn of [...set]) { try { fn({ seq, event, data }); } catch (err) { log.error('[Jobs] listener error:', err.message); } }
+ }).catch(err => log.error('[Jobs] could not record the event:', err.message))
+ .finally(() => { if (emitChains.get(id) === next) emitChains.delete(id); });
+ emitChains.set(id, next);
+ return next;
}
}
function subscribe(id, fn) {
function subscribe(id, fn) {
if (!listeners.has(id)) listeners.set(id, new Set());
if (!listeners.has(id)) listeners.set(id, new Set());
@@ -247,7 +255,7 @@ function createJobSystem(o) {
@@ -247,7 +255,7 @@ function createJobSystem(o) {
const requestHash = crypto.createHash('sha256').update(canonical({ type, input, files: fileHashes })).digest('hex');
const requestHash = crypto.createHash('sha256').update(canonical({ type, input, files: fileHashes })).digest('hex');
if (idempotencyKey != null) {
if (idempotencyKey != null) {
- const prior = store.byIdempotencyKey(owner, String(idempotencyKey));
+ const prior = await store.byIdempotencyKey(owner, String(idempotencyKey));
if (prior) {
if (prior) {
await discard();
await discard();
if (prior.request_hash !== requestHash) throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request', { job_id: prior.id });
if (prior.request_hash !== requestHash) throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request', { job_id: prior.id });
@@ -255,7 +263,7 @@ function createJobSystem(o) {
@@ -255,7 +263,7 @@ function createJobSystem(o) {
}
}
}
}
const limit = env === 'sandbox' ? Math.min(SANDBOX_MAX_ACTIVE, maxActivePerOwner) : maxActivePerOwner;
const limit = env === 'sandbox' ? Math.min(SANDBOX_MAX_ACTIVE, maxActivePerOwner) : maxActivePerOwner;
- if (store.activeForOwner(owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);
+ if (await store.activeForOwner(owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);
const id = `job_${contracts.ids.ulid()}`;
const id = `job_${contracts.ids.ulid()}`;
const inDir = path.join(jobDir(id), 'in');
const inDir = path.join(jobDir(id), 'in');
@@ -265,11 +273,11 @@ function createJobSystem(o) {
@@ -265,11 +273,11 @@ function createJobSystem(o) {
const f = files[i];
const f = files[i];
const dest = path.join(inDir, `${i}-${SAFE_NAME(f.name)}`);
const dest = path.join(inDir, `${i}-${SAFE_NAME(f.name)}`);
if (f.buffer) await fsp.writeFile(dest, f.buffer); else await moveFile(f.path, dest);
if (f.buffer) await fsp.writeFile(dest, f.buffer); else await moveFile(f.path, dest);
- stored.push({ name: String(f.name || `file-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size: f.size != null ? f.size : fs.statSync(dest).size, sha256: fileHashes[i], _path: dest });
+ stored.push({ name: String(f.name || `file-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size: f.size != null ? f.size : (await fsp.stat(dest)).size, sha256: fileHashes[i], _path: dest });
}
}
try {
try {
- store.transaction(() => {
- store.insert({
+ await store.tx(async (t) => {
+ await store.insert({
id, type, type_version: def.version, owner, input_json: JSON.stringify(input), files_json: JSON.stringify(stored),
id, type, type_version: def.version, owner, input_json: JSON.stringify(input), files_json: JSON.stringify(stored),
idempotency_key: idempotencyKey == null ? null : String(idempotencyKey), request_hash: requestHash,
idempotency_key: idempotencyKey == null ? null : String(idempotencyKey), request_hash: requestHash,
max_attempts: def.maxAttempts, ttl_ms: ttlMs, now: Date.now(), env,
max_attempts: def.maxAttempts, ttl_ms: ttlMs, now: Date.now(), env,
@@ -278,22 +286,22 @@ function createJobSystem(o) {
@@ -278,22 +286,22 @@ function createJobSystem(o) {
project_id: projectOf(project),
project_id: projectOf(project),
trace_id: /^[0-9a-f]{32}$/.test(String(traceId || '')) ? String(traceId) : null,
trace_id: /^[0-9a-f]{32}$/.test(String(traceId || '')) ? String(traceId) : null,
});
});
- announce.created(store.get(id));
- })();
+ await announce.created(t, await store.get(id));
+ });
} catch (err) {
} catch (err) {
await fsp.rm(jobDir(id), { recursive: true, force: true });
await fsp.rm(jobDir(id), { recursive: true, force: true });
// Two submits with one key raced: the other one won, so this is a replay of it.
// Two submits with one key raced: the other one won, so this is a replay of it.
- if (/UNIQUE/.test(err.message) && idempotencyKey != null) {
- const prior = store.byIdempotencyKey(owner, String(idempotencyKey));
+ if (/UNIQUE|duplicate key/i.test(err.message) && idempotencyKey != null) {
+ const prior = await store.byIdempotencyKey(owner, String(idempotencyKey));
if (prior && prior.request_hash === requestHash) return { job: prior, replayed: true };
if (prior && prior.request_hash === requestHash) return { job: prior, replayed: true };
throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request');
throw new JobError(409, 'tools.job.idempotency_conflict', 'This Idempotency-Key was used for a different request');
}
}
throw err;
throw err;
}
}
disk.bytes += stored.reduce((n, f) => n + (f.size || 0), 0);
disk.bytes += stored.reduce((n, f) => n + (f.size || 0), 0);
- emit(id, 'job.queued');
+ await emit(id, 'job.queued');
kick();
kick();
- return { job: store.get(id), replayed: false };
+ return { job: await store.get(id), replayed: false };
} catch (err) {
} catch (err) {
await discard();
await discard();
throw err;
throw err;
@@ -301,21 +309,21 @@ function createJobSystem(o) {
@@ -301,21 +309,21 @@ function createJobSystem(o) {
}
}
// ── Scheduling ─────────────────────────────────────────────
// ── Scheduling ─────────────────────────────────────────────
- function kick() {
+ async function kick() {
if (!started || stopped || kicking) return;
if (!started || stopped || kicking) return;
kicking = true;
kicking = true;
try {
try {
while (active.size < concurrency) {
while (active.size < concurrency) {
- const [row] = store.nextQueued(1);
+ const [row] = await store.nextQueued(1);
if (!row) break;
if (!row) break;
// The claim and its tools.job.started event commit together (or neither does).
// The claim and its tools.job.started event commit together (or neither does).
let claimed;
let claimed;
try {
try {
- claimed = store.transaction(() => {
- if (!store.claim(row.id)) return false;
- announce.started(store.get(row.id));
+ claimed = await store.tx(async (t) => {
+ if (!(await store.claim(row.id))) return false;
+ await announce.started(t, await store.get(row.id));
return true;
return true;
- })();
+ });
} catch (err) {
} catch (err) {
log.error(`[Jobs] could not start ${row.id}:`, err.message);
log.error(`[Jobs] could not start ${row.id}:`, err.message);
break; // it stays queued; the next kick tries again
break; // it stays queued; the next kick tries again
@@ -323,7 +331,7 @@ function createJobSystem(o) {
@@ -323,7 +331,7 @@ function createJobSystem(o) {
if (!claimed) continue;
if (!claimed) continue;
const entry = { ctrl: new AbortController(), reason: null, last: { pct: -1, msg: null, at: 0 } };
const entry = { ctrl: new AbortController(), reason: null, last: { pct: -1, msg: null, at: 0 } };
active.set(row.id, entry);
active.set(row.id, entry);
- emit(row.id, 'job.running');
+ await emit(row.id, 'job.running');
setImmediate(() => execute(row.id, entry));
setImmediate(() => execute(row.id, entry));
}
}
} finally { kicking = false; }
} finally { kicking = false; }
@@ -337,12 +345,12 @@ function createJobSystem(o) {
@@ -337,12 +345,12 @@ function createJobSystem(o) {
const now = Date.now();
const now = Date.now();
if (msg === entry.last.msg && (Math.abs(pct - entry.last.pct) < 1 || now - entry.last.at < throttleMs)) return;
if (msg === entry.last.msg && (Math.abs(pct - entry.last.pct) < 1 || now - entry.last.at < throttleMs)) return;
entry.last = { pct, msg, at: now };
entry.last = { pct, msg, at: now };
- if (store.setProgress(id, pct, msg, now)) emit(id, 'job.progress');
+ store.setProgress(id, pct, msg, now).then((changed) => { if (changed) return emit(id, 'job.progress'); }).catch(() => {});
};
};
}
}
async function execute(id, entry) {
async function execute(id, entry) {
- const row = stopped ? null : store.get(id);
+ const row = stopped ? null : await store.get(id);
if (!row) { active.delete(id); return; }
if (!row) { active.delete(id); return; }
const def = types.get(row.type);
const def = types.get(row.type);
const outDir = path.join(jobDir(row.id), 'out');
const outDir = path.join(jobDir(row.id), 'out');
@@ -351,9 +359,12 @@ function createJobSystem(o) {
@@ -351,9 +359,12 @@ function createJobSystem(o) {
if (timer.unref) timer.unref();
if (timer.unref) timer.unref();
try {
try {
await fsp.mkdir(outDir, { recursive: true });
await fsp.mkdir(outDir, { recursive: true });
+ // A cancel (or a timeout) that arrived while we were preparing must end the job now: the run
+ // would not see an abort that fired before it registered its listener.
+ if (entry.reason) throw new Error(entry.reason);
const files = store.parse(row.files_json, []).map(f => ({ name: f.name, mime: f.mime, size: f.size, sha256: f.sha256, path: f._path }));
const files = store.parse(row.files_json, []).map(f => ({ name: f.name, mime: f.mime, size: f.size, sha256: f.sha256, path: f._path }));
const out = await def.run({
const out = await def.run({
- job: { id: row.id, type: row.type, attempt: row.attempts, owner: row.owner },
+ job: { id: row.id, type: row.type, attempt: Number(row.attempts), owner: row.owner },
input: store.parse(row.input_json, {}), files, outDir,
input: store.parse(row.input_json, {}), files, outDir,
signal: entry.ctrl.signal,
signal: entry.ctrl.signal,
progress: progressFn(row.id, entry),
progress: progressFn(row.id, entry),
@@ -366,7 +377,7 @@ function createJobSystem(o) {
@@ -366,7 +377,7 @@ function createJobSystem(o) {
outcome = { state: 'succeeded', result };
outcome = { state: 'succeeded', result };
} catch (err) {
} catch (err) {
if (stopped) return;
if (stopped) return;
- const fresh = store.get(row.id) || row;
+ const fresh = (await store.get(row.id)) || row;
if (entry.reason === 'cancel' || fresh.cancel_requested) outcome = { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') };
if (entry.reason === 'cancel' || fresh.cancel_requested) outcome = { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') };
else if (entry.reason === 'timeout') outcome = { state: 'failed', error: problem(504, 'tools.job.timeout', `The job ran longer than ${Math.round(def.timeoutMs / 1000)} s`), retryable: true };
else if (entry.reason === 'timeout') outcome = { state: 'failed', error: problem(504, 'tools.job.timeout', `The job ran longer than ${Math.round(def.timeoutMs / 1000)} s`), retryable: true };
else {
else {
@@ -381,9 +392,9 @@ function createJobSystem(o) {
@@ -381,9 +392,9 @@ function createJobSystem(o) {
let recorded = false;
let recorded = false;
try {
try {
// The end state and its tools.job.succeeded|failed event commit together (or neither does).
// The end state and its tools.job.succeeded|failed event commit together (or neither does).
- store.transaction(() => { if (store.finish(row.id, outcome)) ended(row.id); })();
+ await store.tx(async (t) => { if (await store.finish(row.id, outcome)) await ended(t, row.id); });
recorded = true;
recorded = true;
- emit(row.id, EVENT_FOR[outcome.state]);
+ await emit(row.id, EVENT_FOR[outcome.state]);
} catch (err) {
} catch (err) {
// Nothing committed: the row is still 'running' with its inputs, which the next start() recovers.
// Nothing committed: the row is still 'running' with its inputs, which the next start() recovers.
log.error(`[Jobs] could not record the end of ${row.id}:`, err.message);
log.error(`[Jobs] could not record the end of ${row.id}:`, err.message);
@@ -409,7 +420,7 @@ function createJobSystem(o) {
@@ -409,7 +420,7 @@ function createJobSystem(o) {
await moveFile(local, dest);
await moveFile(local, dest);
local = dest;
local = dest;
}
}
- const size = fs.statSync(local).size;
+ const size = (await fsp.stat(local)).size;
const sha256 = await sha256File(local);
const sha256 = await sha256File(local);
const rec = { name: String(f.name || `result-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size, sha256, storage: 'local', media: null, _path: local };
const rec = { name: String(f.name || `result-${i}`).slice(0, 200), mime: f.mime || 'application/octet-stream', size, sha256, storage: 'local', media: null, _path: local };
// A developer app's results (sandbox too) go under its project's namespace (WS-L task 5); other
// A developer app's results (sandbox too) go under its project's namespace (WS-L task 5); other
@@ -442,23 +453,29 @@ function createJobSystem(o) {
@@ -442,23 +453,29 @@ function createJobSystem(o) {
// ── Cancel ─────────────────────────────────────────────────
// ── Cancel ─────────────────────────────────────────────────
/** → { job, changed } — queued jobs end at once; running ones are signalled and end as cancelled. */
/** → { job, changed } — queued jobs end at once; running ones are signalled and end as cancelled. */
- function cancel(id) {
- const row = store.get(id);
+ async function cancel(id) {
+ const row = await store.get(id);
if (!row) return null;
if (!row) return null;
if (TERMINAL.includes(row.state)) return { job: row, changed: false };
if (TERMINAL.includes(row.state)) return { job: row, changed: false };
if (row.state === 'queued') {
if (row.state === 'queued') {
- const done = store.transaction(() => store.requestCancel(id) && store.finish(id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled before it started') }))();
+ const done = await store.tx(async () => {
+ if (!(await store.requestCancel(id))) return false;
+ return store.finish(id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled before it started') });
+ });
if (done) {
if (done) {
- emit(id, 'job.cancelled');
+ await emit(id, 'job.cancelled');
fsp.rm(jobDir(id), { recursive: true, force: true }).catch(() => {});
fsp.rm(jobDir(id), { recursive: true, force: true }).catch(() => {});
}
}
- return { job: store.get(id), changed: done };
+ return { job: await store.get(id), changed: done };
}
}
- store.requestCancel(id);
+ await store.requestCancel(id);
+ // Read the row (still 'running') before signalling the worker: the HTTP layer decides 200 vs 202
+ // from it, and the abort must not race that answer.
+ const job = await store.get(id);
const entry = active.get(id);
const entry = active.get(id);
if (entry && !entry.reason) { entry.reason = 'cancel'; entry.ctrl.abort(new Error('cancelled')); }
if (entry && !entry.reason) { entry.reason = 'cancel'; entry.ctrl.abort(new Error('cancelled')); }
- emit(id, 'job.cancel_requested');
- return { job: store.get(id), changed: true };
+ await emit(id, 'job.cancel_requested');
+ return { job, changed: true };
}
}
// ── Retry ──────────────────────────────────────────────────
// ── Retry ──────────────────────────────────────────────────
@@ -466,13 +483,12 @@ function createJobSystem(o) {
@@ -466,13 +483,12 @@ function createJobSystem(o) {
* Retry a failed job as a new job with the same type, input, files and lifetime.
* Retry a failed job as a new job with the same type, input, files and lifetime.
* → { job, replayed } | null (no such job). Idempotent: a failed job is retried once, and asking
* → { job, replayed } | null (no such job). Idempotent: a failed job is retried once, and asking
* again returns that retry, whatever state it is in now. Only failed jobs can be retried.
* again returns that retry, whatever state it is in now. Only failed jobs can be retried.
- * Everything up to the insert is synchronous, so two concurrent calls cannot both retry it.
*/
*/
- function retry(id) {
- const row = store.get(id);
+ async function retry(id) {
+ const row = await store.get(id);
if (!row) return null;
if (!row) return null;
if (row.retried_by) {
if (row.retried_by) {
- const next = store.get(row.retried_by);
+ const next = await store.get(row.retried_by);
if (next) return { job: next, replayed: true };
if (next) return { job: next, replayed: true };
throw new JobError(410, 'tools.job.retry_gone', 'This job was retried and the retry has since expired', { retried_by: row.retried_by });
throw new JobError(410, 'tools.job.retry_gone', 'This job was retried and the retry has since expired', { retried_by: row.retried_by });
}
}
@@ -481,7 +497,7 @@ function createJobSystem(o) {
@@ -481,7 +497,7 @@ function createJobSystem(o) {
const def = types.get(row.type);
const def = types.get(row.type);
if (!def) throw new JobError(409, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`);
if (!def) throw new JobError(409, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`);
const limit = row.env === 'sandbox' ? Math.min(SANDBOX_MAX_ACTIVE, maxActivePerOwner) : maxActivePerOwner;
const limit = row.env === 'sandbox' ? Math.min(SANDBOX_MAX_ACTIVE, maxActivePerOwner) : maxActivePerOwner;
- if (store.activeForOwner(row.owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);
+ if (await store.activeForOwner(row.owner) >= limit) throw new JobError(429, 'tools.job.too_many_active', `At most ${limit} unfinished jobs at a time; wait for one to finish or cancel one`);
const inputs = store.parse(row.files_json, []);
const inputs = store.parse(row.files_json, []);
if (inputs.some(f => !f._path || !fs.existsSync(f._path))) throw new JobError(410, 'tools.job.inputs_gone', 'The input files of this job are no longer kept; submit it again');
if (inputs.some(f => !f._path || !fs.existsSync(f._path))) throw new JobError(410, 'tools.job.inputs_gone', 'The input files of this job are no longer kept; submit it again');
@@ -496,23 +512,23 @@ function createJobSystem(o) {
@@ -496,23 +512,23 @@ function createJobSystem(o) {
}
}
const stored = inputs.map(f => ({ ...f, _path: path.join(newIn, path.basename(f._path)) }));
const stored = inputs.map(f => ({ ...f, _path: path.join(newIn, path.basename(f._path)) }));
try {
try {
- store.transaction(() => {
- if (!store.markRetried(row.id, next)) throw new JobError(409, 'tools.job.not_failed', 'This job cannot be retried any more');
- store.insert({
+ await store.tx(async (t) => {
+ if (!(await store.markRetried(row.id, next))) throw new JobError(409, 'tools.job.not_failed', 'This job cannot be retried any more');
+ await store.insert({
id: next, type: row.type, type_version: def.version, owner: row.owner, input_json: row.input_json, files_json: JSON.stringify(stored),
id: next, type: row.type, type_version: def.version, owner: row.owner, input_json: row.input_json, files_json: JSON.stringify(stored),
idempotency_key: null, request_hash: row.request_hash, max_attempts: def.maxAttempts, ttl_ms: row.ttl_ms, now: Date.now(),
idempotency_key: null, request_hash: row.request_hash, max_attempts: def.maxAttempts, ttl_ms: row.ttl_ms, now: Date.now(),
env: row.env, retry_of: row.id, ip_key: row.ip_key || null, tool: row.tool || null, project_id: row.project_id || null,
env: row.env, retry_of: row.id, ip_key: row.ip_key || null, tool: row.tool || null, project_id: row.project_id || null,
});
});
- announce.created(store.get(next));
- })();
+ await announce.created(t, await store.get(next));
+ });
} catch (err) {
} catch (err) {
if (moved) { try { fs.renameSync(newIn, oldIn); } catch { /* best effort */ } }
if (moved) { try { fs.renameSync(newIn, oldIn); } catch { /* best effort */ } }
fs.rmSync(jobDir(next), { recursive: true, force: true });
fs.rmSync(jobDir(next), { recursive: true, force: true });
throw err;
throw err;
}
}
- emit(next, 'job.queued');
+ await emit(next, 'job.queued');
kick();
kick();
- return { job: store.get(next), replayed: false };
+ return { job: await store.get(next), replayed: false };
}
}
// ── References (results something still points at) ─────────
// ── References (results something still points at) ─────────
@@ -525,58 +541,58 @@ function createJobSystem(o) {
@@ -525,58 +541,58 @@ function createJobSystem(o) {
/**
/**
* Keep a succeeded job's result while `ref` points at it. → { job, created }. Idempotent.
* Keep a succeeded job's result while `ref` points at it. → { job, created }. Idempotent.
*/
*/
- function reference(id, ref) {
+ async function reference(id, ref) {
ref = checkRef(ref);
ref = checkRef(ref);
- const row = store.get(id);
+ const row = await store.get(id);
if (!row) return null;
if (!row) return null;
if (row.state !== 'succeeded') throw new JobError(409, 'tools.job.not_succeeded', `Only a succeeded job's result can be referenced; this one is ${row.state}`, { state: row.state });
if (row.state !== 'succeeded') throw new JobError(409, 'tools.job.not_succeeded', `Only a succeeded job's result can be referenced; this one is ${row.state}`, { state: row.state });
if (row.env === 'sandbox') throw new JobError(409, 'tools.job.sandbox', 'Sandbox results are kept briefly and cannot be referenced');
if (row.env === 'sandbox') throw new JobError(409, 'tools.job.sandbox', 'Sandbox results are kept briefly and cannot be referenced');
- const has = store.references(id).some(r => r.ref === ref);
- if (!has && store.referenceCount(id) >= MAX_REFS) throw new JobError(409, 'tools.job.too_many_references', `At most ${MAX_REFS} references per job`);
- const created = store.addReference(id, ref);
- return { job: store.get(id), created };
+ const has = (await store.references(id)).some(r => r.ref === ref);
+ if (!has && await store.referenceCount(id) >= MAX_REFS) throw new JobError(409, 'tools.job.too_many_references', `At most ${MAX_REFS} references per job`);
+ const created = await store.addReference(id, ref);
+ return { job: await store.get(id), created };
}
}
/** Stop keeping the result for `ref`. → { job, removed }. Idempotent. */
/** Stop keeping the result for `ref`. → { job, removed }. Idempotent. */
- function unreference(id, ref) {
+ async function unreference(id, ref) {
ref = checkRef(ref);
ref = checkRef(ref);
- if (!store.get(id)) return null;
- const removed = store.dropReference(id, ref);
- return { job: store.get(id), removed };
+ if (!(await store.get(id))) return null;
+ const removed = await store.dropReference(id, ref);
+ return { job: await store.get(id), removed };
}
}
// ── Boot recovery, pruning ─────────────────────────────────
// ── Boot recovery, pruning ─────────────────────────────────
/** A job the restart ended as failed: the row and its tools.job.failed event in one transaction. */
/** A job the restart ended as failed: the row and its tools.job.failed event in one transaction. */
- function failRecovered(id, outcome) {
- store.transaction(() => { if (store.finish(id, outcome)) ended(id); })();
+ async function failRecovered(id, outcome) {
+ await store.tx(async (t) => { if (await store.finish(id, outcome)) await ended(t, id); });
}
}
- function recover() {
+ async function recover() {
let requeued = 0, failed = 0;
let requeued = 0, failed = 0;
- for (const row of store.running()) {
+ for (const row of await store.running()) {
const def = types.get(row.type);
const def = types.get(row.type);
if (row.cancel_requested) {
if (row.cancel_requested) {
- store.finish(row.id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') });
- emit(row.id, 'job.cancelled');
+ await store.finish(row.id, { state: 'cancelled', error: problem(409, 'tools.job.cancelled', 'The job was cancelled') });
+ await emit(row.id, 'job.cancelled');
} else if (!def) {
} else if (!def) {
- failRecovered(row.id, { state: 'failed', error: problem(500, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`) });
- emit(row.id, 'job.failed'); failed++;
- } else if (def.onRestart === 'requeue' && row.attempts < row.max_attempts) {
- store.requeue(row.id);
- emit(row.id, 'job.queued'); requeued++;
+ await failRecovered(row.id, { state: 'failed', error: problem(500, 'tools.job.unknown_type', `This service no longer runs "${row.type}" jobs`) });
+ await emit(row.id, 'job.failed'); failed++;
+ } else if (def.onRestart === 'requeue' && Number(row.attempts) < Number(row.max_attempts)) {
+ await store.requeue(row.id);
+ await emit(row.id, 'job.queued'); requeued++;
} else {
} else {
const retryable = def.onRestart === 'fail';
const retryable = def.onRestart === 'fail';
- failRecovered(row.id, {
+ await failRecovered(row.id, {
state: 'failed', retryable,
state: 'failed', retryable,
error: problem(503, 'tools.job.interrupted', retryable ? 'The service restarted while this job was running; submit it again' : `The service restarted during each of ${row.attempts} attempts`),
error: problem(503, 'tools.job.interrupted', retryable ? 'The service restarted while this job was running; submit it again' : `The service restarted during each of ${row.attempts} attempts`),
});
});
- emit(row.id, 'job.failed'); failed++;
+ await emit(row.id, 'job.failed'); failed++;
}
}
fs.rmSync(path.join(jobDir(row.id), 'out'), { recursive: true, force: true });
fs.rmSync(path.join(jobDir(row.id), 'out'), { recursive: true, force: true });
}
}
// Directories without a row: a crash between moving the upload and recording the job.
// Directories without a row: a crash between moving the upload and recording the job.
// That job was never accepted (no 202 went out), so its files have no owner.
// That job was never accepted (no 202 went out), so its files have no owner.
for (const name of fs.readdirSync(root)) {
for (const name of fs.readdirSync(root)) {
- if (/^job_[0-9A-Z]{26}$/.test(name) && !store.get(name)) fs.rmSync(path.join(root, name), { recursive: true, force: true });
+ if (/^job_[0-9A-Z]{26}$/.test(name) && !(await store.get(name))) fs.rmSync(path.join(root, name), { recursive: true, force: true });
}
}
return { requeued, failed };
return { requeued, failed };
}
}
@@ -590,7 +606,7 @@ function createJobSystem(o) {
@@ -590,7 +606,7 @@ function createJobSystem(o) {
*/
*/
async function prune(now = Date.now()) {
async function prune(now = Date.now()) {
let n = 0;
let n = 0;
- for (const row of store.expired(now)) {
+ for (const row of await store.expired(now)) {
const result = store.parse(row.result_json, null);
const result = store.parse(row.result_json, null);
let kept = null;
let kept = null;
for (const f of (result && result.files) || []) {
for (const f of (result && result.files) || []) {
@@ -602,11 +618,11 @@ function createJobSystem(o) {
@@ -602,11 +618,11 @@ function createJobSystem(o) {
}
}
if (kept) {
if (kept) {
log.warn(`[Jobs] kept expired job ${row.id} (${kept}); checking again later`);
log.warn(`[Jobs] kept expired job ${row.id} (${kept}); checking again later`);
- store.deferExpiry(row.id, now + PRUNE_RECHECK_MS);
+ await store.deferExpiry(row.id, now + PRUNE_RECHECK_MS);
continue;
continue;
}
}
await fsp.rm(jobDir(row.id), { recursive: true, force: true }).catch(() => {});
await fsp.rm(jobDir(row.id), { recursive: true, force: true }).catch(() => {});
- store.remove(row.id);
+ await store.remove(row.id);
listeners.delete(row.id);
listeners.delete(row.id);
n++;
n++;
}
}
@@ -640,9 +656,9 @@ function createJobSystem(o) {
@@ -640,9 +656,9 @@ function createJobSystem(o) {
* Is the store over its bounds? → null, or { reason: 'queue'|'disk', detail, retryAfter } for a
* Is the store over its bounds? → null, or { reason: 'queue'|'disk', detail, retryAfter } for a
* 503 tools.busy. Cheap: one COUNT and the last disk walk (a stale walk is refreshed in the background).
* 503 tools.busy. Cheap: one COUNT and the last disk walk (a stale walk is refreshed in the background).
*/
*/
- function busy() {
+ async function busy() {
if (maxQueued) {
if (maxQueued) {
- const queued = store.counts().queued || 0;
+ const queued = (await store.counts()).queued || 0;
if (queued >= maxQueued) return { reason: 'queue', detail: `${queued} jobs are already waiting on this server; try again in a minute.`, retryAfter: 60, queued, limit: maxQueued };
if (queued >= maxQueued) return { reason: 'queue', detail: `${queued} jobs are already waiting on this server; try again in a minute.`, retryAfter: 60, queued, limit: maxQueued };
}
}
if (diskBudget) {
if (diskBudget) {
@@ -657,26 +673,26 @@ function createJobSystem(o) {
@@ -657,26 +673,26 @@ function createJobSystem(o) {
* { active, limit } when a new session job from `ipKey` would pass it (the HTTP layer answers 429
* { active, limit } when a new session job from `ipKey` would pass it (the HTTP layer answers 429
* tools.job.too_many_active through the guard). Principals and people are bounded per owner only.
* tools.job.too_many_active through the guard). Principals and people are bounded per owner only.
*/
*/
- function addressFull(owner, ipKey) {
+ async function addressFull(owner, ipKey) {
if (!ipKey || !String(owner || '').startsWith('session:')) return null;
if (!ipKey || !String(owner || '').startsWith('session:')) return null;
- const active = store.activeForIp(ipKey);
+ const active = await store.activeForIp(ipKey);
return active >= maxActivePerAddress ? { active, limit: maxActivePerAddress } : null;
return active >= maxActivePerAddress ? { active, limit: maxActivePerAddress } : null;
}
}
- function start() {
+ async function start() {
if (started) return api;
if (started) return api;
started = true;
started = true;
if (diskBudget) refreshDisk();
if (diskBudget) refreshDisk();
- const r = recover();
+ const r = await recover();
if (r.requeued || r.failed) log.log(`[Jobs] ${o.service}: ${r.requeued} job(s) re-queued, ${r.failed} failed after restart`);
if (r.requeued || r.failed) log.log(`[Jobs] ${o.service}: ${r.requeued} job(s) re-queued, ${r.failed} failed after restart`);
- prune().catch(() => {});
- flushUsage();
+ await prune().catch(() => {});
+ await flushUsage();
pruneTimer = setInterval(() => {
pruneTimer = setInterval(() => {
prune().catch(err => log.error('[Jobs] prune:', err.message));
prune().catch(err => log.error('[Jobs] prune:', err.message));
flushUsage();
flushUsage();
}, o.pruneIntervalMs || 5 * 60 * 1000);
}, o.pruneIntervalMs || 5 * 60 * 1000);
if (pruneTimer.unref) pruneTimer.unref();
if (pruneTimer.unref) pruneTimer.unref();
- kick();
+ await kick();
return api;
return api;
}
}
@@ -685,8 +701,8 @@ function createJobSystem(o) {
@@ -685,8 +701,8 @@ function createJobSystem(o) {
* crash or a restart leaves behind; the next start() recovers them per type.
* crash or a restart leaves behind; the next start() recovers them per type.
*/
*/
/** Closed hours of project usage → the outbox (./usage.js); a failure is logged and retried next tick. */
/** Closed hours of project usage → the outbox (./usage.js); a failure is logged and retried next tick. */
- function flushUsage(at) {
- try { return usage.flush(at); } catch (err) { log.error('[Jobs] usage flush:', err.message); return { queued: 0, invalid: 0 }; }
+ async function flushUsage(at) {
+ try { return await usage.flush(at); } catch (err) { log.error('[Jobs] usage flush:', err.message); return { queued: 0, invalid: 0 }; }
}
}
function stop() {
function stop() {
@@ -701,8 +717,8 @@ function createJobSystem(o) {
@@ -701,8 +717,8 @@ function createJobSystem(o) {
function onStop(fn) { stopHooks.add(fn); return () => stopHooks.delete(fn); }
function onStop(fn) { stopHooks.add(fn); return () => stopHooks.delete(fn); }
/** The owner's succeeded job with a result file stored as Media object `mediaId` → { row, index } or null. */
/** The owner's succeeded job with a result file stored as Media object `mediaId` → { row, index } or null. */
- function findResultMedia(owner, mediaId) {
- for (const row of store.byResultMedia(owner, mediaId)) {
+ async function findResultMedia(owner, mediaId) {
+ for (const row of await store.byResultMedia(owner, mediaId)) {
const files = (store.parse(row.result_json, null) || {}).files || [];
const files = (store.parse(row.result_json, null) || {}).files || [];
const index = files.findIndex(f => f.media && f.media.media_id === mediaId);
const index = files.findIndex(f => f.media && f.media.media_id === mediaId);
if (index >= 0) return { row, index };
if (index >= 0) return { row, index };
@@ -728,7 +744,7 @@ function createJobSystem(o) {
@@ -728,7 +744,7 @@ function createJobSystem(o) {
types: () => [...types.keys()],
types: () => [...types.keys()],
media,
media,
// running (after the spread) is the store's count of rows in 'running'; executing is this process's.
// running (after the spread) is the store's count of rows in 'running'; executing is this process's.
- stats: () => ({ running: active.size, concurrency, ...store.counts(), executing: active.size, results: media ? 'media' : 'local', events: announce.status(), usage: usage.enabled ? usage.status() : { enabled: false } }),
+ stats: async () => ({ running: active.size, concurrency, ...(await store.counts()), executing: active.size, results: media ? 'media' : 'local', events: await announce.status(), usage: usage.enabled ? await usage.status() : { enabled: false } }),
usage, flushUsage,
usage, flushUsage,
outbox: o.outbox || null,
outbox: o.outbox || null,
/** True between start() and stop(): the worker picks up queued jobs. */
/** True between start() and stop(): the worker picks up queued jobs. */
@@ -19,7 +19,8 @@
@@ -19,7 +19,8 @@
// outbox relay publishes it. A job that ends in an hour already sent (a clock step back) reopens it as
// outbox relay publishes it. A job that ends in an hour already sent (a clock step back) reopens it as
// revision + 1. Only counts leave: no owner, session, address, input, file name or output.
// revision + 1. Only counts leave: no owner, session, address, input, file name or output.
//
//
-// Inert without an outbox (EVENTS_URL unset, like ./events.js): nothing is counted.
+// On PostgreSQL (plan T8): openvibe-sdk/db, ambient transactions, createPgOutbox. Inert without an
+// outbox (EVENTS_URL unset, like ./events.js): nothing is counted.
// ═══════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════
const HOUR_MS = 60 * 60 * 1000;
const HOUR_MS = 60 * 60 * 1000;
@@ -32,25 +33,6 @@ const CODE_RE = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+)+$/;
@@ -32,25 +33,6 @@ const CODE_RE = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+)+$/;
const TRACE_RE = /^[0-9a-f]{32}$/;
const TRACE_RE = /^[0-9a-f]{32}$/;
const DIMENSION_RE = /^[a-z0-9][a-z0-9_.:-]{0,79}$/;
const DIMENSION_RE = /^[a-z0-9][a-z0-9_.:-]{0,79}$/;
-const SCHEMA = `
-CREATE TABLE IF NOT EXISTS tool_job_usage (
- project_id TEXT NOT NULL,
- env TEXT NOT NULL,
- capability TEXT NOT NULL,
- dimension TEXT NOT NULL,
- window_start INTEGER NOT NULL,
- quantity INTEGER NOT NULL DEFAULT 0,
- errors INTEGER NOT NULL DEFAULT 0,
- error_codes TEXT NOT NULL DEFAULT '{}',
- samples TEXT NOT NULL DEFAULT '[]',
- revision INTEGER NOT NULL DEFAULT 1,
- event_id TEXT,
- emitted_at INTEGER,
- PRIMARY KEY (project_id, env, capability, dimension, window_start)
-);
-CREATE INDEX IF NOT EXISTS tool_job_usage_open ON tool_job_usage(emitted_at, window_start);
-`;
-
const hourOf = (ms) => Math.floor(ms / HOUR_MS) * HOUR_MS;
const hourOf = (ms) => Math.floor(ms / HOUR_MS) * HOUR_MS;
const parse = (s, d) => { try { return JSON.parse(s); } catch { return d; } };
const parse = (s, d) => { try { return JSON.parse(s); } catch { return d; } };
@@ -69,60 +51,60 @@ function keyOf(row) {
@@ -69,60 +51,60 @@ function keyOf(row) {
/**
/**
* @param {object} o
* @param {object} o
- * @param {object} o.db the jobs database (better-sqlite3), where the outbox lives too
+ * @param {object} o.db openvibe-sdk/db handle (the one tools database), where the outbox lives too
+ * @param {string} o.app the owning app ('img' | …)
* @param {object} o.contracts require('openvibe-contracts') (0.63.0+ knows tools.usage.recorded@1)
* @param {object} o.contracts require('openvibe-contracts') (0.63.0+ knows tools.usage.recorded@1)
- * @param {object|null} o.outbox openvibe-sdk outbox on `db` (./events outboxFromEnv); null = inert
+ * @param {object|null} o.outbox openvibe-sdk createPgOutbox on `db` (./events outboxFromEnv); null = inert
* @param {() => number} [o.now]
* @param {() => number} [o.now]
* @param {object} [o.log]
* @param {object} [o.log]
*/
*/
-function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(), log = console }) {
- if (!outbox) return { enabled: false, finished() {}, flush: () => ({ queued: 0, invalid: 0 }), pending: () => 0 };
- db.exec(SCHEMA);
+function createJobUsage({ db, app, contracts, outbox = null, now = () => Date.now(), log = console }) {
+ if (!outbox) return { enabled: false, finished: async () => {}, flush: async () => ({ queued: 0, invalid: 0 }), pending: async () => 0, status: async () => ({ enabled: false }) };
const q = {
const q = {
- add: db.prepare(`INSERT INTO tool_job_usage (project_id, env, capability, dimension, window_start, quantity, errors)
- VALUES (@project_id, @env, @capability, @dimension, @window_start, 1, @errors)
- ON CONFLICT(project_id, env, capability, dimension, window_start) DO UPDATE SET
- quantity = quantity + 1, errors = errors + excluded.errors,
- revision = revision + (CASE WHEN emitted_at IS NULL THEN 0 ELSE 1 END), emitted_at = NULL`),
- get: db.prepare('SELECT error_codes, samples FROM tool_job_usage WHERE project_id = ? AND env = ? AND capability = ? AND dimension = ? AND window_start = ?'),
- setErrors: db.prepare('UPDATE tool_job_usage SET error_codes = ?, samples = ? WHERE project_id = ? AND env = ? AND capability = ? AND dimension = ? AND window_start = ?'),
- due: db.prepare('SELECT * FROM tool_job_usage WHERE emitted_at IS NULL AND window_start <= ? ORDER BY window_start LIMIT 500'),
- sent: db.prepare('UPDATE tool_job_usage SET emitted_at = ?, event_id = ? WHERE project_id = ? AND env = ? AND capability = ? AND dimension = ? AND window_start = ? AND emitted_at IS NULL'),
- prune: db.prepare('DELETE FROM tool_job_usage WHERE emitted_at IS NOT NULL AND window_start < ?'),
- pending: db.prepare('SELECT COUNT(*) AS n FROM tool_job_usage WHERE emitted_at IS NULL'),
+ add: db.prepare(`INSERT INTO tool_job_usage (app, project_id, env, capability, dimension, window_start, quantity, errors)
+ VALUES (@app, @project_id, @env, @capability, @dimension, @window_start, 1, @errors)
+ ON CONFLICT(app, project_id, env, capability, dimension, window_start) DO UPDATE SET
+ quantity = tool_job_usage.quantity + 1, errors = tool_job_usage.errors + excluded.errors,
+ revision = tool_job_usage.revision + (CASE WHEN tool_job_usage.emitted_at IS NULL THEN 0 ELSE 1 END), emitted_at = NULL`),
+ get: db.prepare('SELECT error_codes, samples FROM tool_job_usage WHERE app = @app AND project_id = @project_id AND env = @env AND capability = @capability AND dimension = @dimension AND window_start = @window_start'),
+ setErrors: db.prepare('UPDATE tool_job_usage SET error_codes = @error_codes, samples = @samples WHERE app = @app AND project_id = @project_id AND env = @env AND capability = @capability AND dimension = @dimension AND window_start = @window_start'),
+ due: db.prepare('SELECT * FROM tool_job_usage WHERE app = @app AND emitted_at IS NULL AND window_start <= @at ORDER BY window_start LIMIT 500'),
+ sent: db.prepare('UPDATE tool_job_usage SET emitted_at = @at, event_id = @event_id WHERE app = @app AND project_id = @project_id AND env = @env AND capability = @capability AND dimension = @dimension AND window_start = @window_start AND emitted_at IS NULL'),
+ prune: db.prepare('DELETE FROM tool_job_usage WHERE app = @app AND emitted_at IS NOT NULL AND window_start < @at'),
+ pending: db.prepare('SELECT COUNT(*) AS n FROM tool_job_usage WHERE app = @app AND emitted_at IS NULL'),
};
};
const stats = { invalid: 0, lastInvalid: null };
const stats = { invalid: 0, lastInvalid: null };
/** INSIDE the transaction that records the job's end (store.finish), with the row read back. */
/** INSIDE the transaction that records the job's end (store.finish), with the row read back. */
- function finished(row) {
+ async function finished(row) {
const k = keyOf(row);
const k = keyOf(row);
if (!k) return;
if (!k) return;
const failed = row.state === 'failed';
const failed = row.state === 'failed';
- q.add.run({ ...k, errors: failed ? 1 : 0 });
+ await q.add.run({ app, ...k, errors: failed ? 1 : 0 });
if (!failed) return;
if (!failed) return;
const error = parse(row.error_json, null) || {};
const error = parse(row.error_json, null) || {};
const code = CODE_RE.test(String(error.code || '')) ? String(error.code) : 'tools.job.failed';
const code = CODE_RE.test(String(error.code || '')) ? String(error.code) : 'tools.job.failed';
- const cur = q.get.get(k.project_id, k.env, k.capability, k.dimension, k.window_start);
+ const cur = await q.get.get({ app, ...k });
const codes = parse(cur.error_codes, {});
const codes = parse(cur.error_codes, {});
if (codes[code] || Object.keys(codes).length < MAX_CODES) codes[code] = (codes[code] || 0) + 1;
if (codes[code] || Object.keys(codes).length < MAX_CODES) codes[code] = (codes[code] || 0) + 1;
const sample = { at: new Date(row.finished_at || now()).toISOString(), code, ref: row.id };
const sample = { at: new Date(row.finished_at || now()).toISOString(), code, ref: row.id };
if (Number.isInteger(error.status) && error.status >= 100 && error.status <= 599) sample.status = error.status;
if (Number.isInteger(error.status) && error.status >= 100 && error.status <= 599) sample.status = error.status;
if (TRACE_RE.test(String(row.trace_id || ''))) sample.trace_id = row.trace_id;
if (TRACE_RE.test(String(row.trace_id || ''))) sample.trace_id = row.trace_id;
const samples = [sample, ...parse(cur.samples, [])].slice(0, MAX_SAMPLES);
const samples = [sample, ...parse(cur.samples, [])].slice(0, MAX_SAMPLES);
- q.setErrors.run(JSON.stringify(codes), JSON.stringify(samples), k.project_id, k.env, k.capability, k.dimension, k.window_start);
+ await q.setErrors.run({ app, ...k, error_codes: JSON.stringify(codes), samples: JSON.stringify(samples) });
}
}
function payloadOf(row) {
function payloadOf(row) {
const p = {
const p = {
project_id: row.project_id, env: row.env, capability: row.capability, dimension: row.dimension, unit: 'jobs', window: 'hour',
project_id: row.project_id, env: row.env, capability: row.capability, dimension: row.dimension, unit: 'jobs', window: 'hour',
- window_start: new Date(row.window_start).toISOString(), window_end: new Date(row.window_start + HOUR_MS).toISOString(),
- quantity: row.quantity, errors: row.errors,
+ window_start: new Date(Number(row.window_start)).toISOString(), window_end: new Date(Number(row.window_start) + HOUR_MS).toISOString(),
+ quantity: Number(row.quantity), errors: Number(row.errors),
};
};
const codes = parse(row.error_codes, {});
const codes = parse(row.error_codes, {});
if (Object.keys(codes).length) p.error_codes = codes;
if (Object.keys(codes).length) p.error_codes = codes;
const samples = parse(row.samples, []);
const samples = parse(row.samples, []);
if (samples.length) p.samples = samples;
if (samples.length) p.samples = samples;
- if (row.revision > 1) p.revision = row.revision;
+ if (Number(row.revision) > 1) p.revision = Number(row.revision);
return p;
return p;
}
}
@@ -130,9 +112,9 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),
@@ -130,9 +112,9 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),
* Write every closed hour's rollups to the outbox (each in the transaction that marks it sent).
* Write every closed hour's rollups to the outbox (each in the transaction that marks it sent).
* A payload that fails its contract is logged and left unsent, never thrown.
* A payload that fails its contract is logged and left unsent, never thrown.
*/
*/
- function flush(at = now()) {
+ async function flush(at = now()) {
let queued = 0;
let queued = 0;
- for (const row of q.due.all(hourOf(at - GRACE_MS) - HOUR_MS)) {
+ for (const row of await q.due.all({ app, at: hourOf(at - GRACE_MS) - HOUR_MS })) {
const payload = payloadOf(row);
const payload = payloadOf(row);
const v = contracts.validate('tools.usage.recorded@1', payload);
const v = contracts.validate('tools.usage.recorded@1', payload);
if (!v.valid) {
if (!v.valid) {
@@ -141,8 +123,8 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),
@@ -141,8 +123,8 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),
log.error(`[Jobs] tools.usage.recorded not sent (payload does not match the contract): ${stats.lastInvalid}`);
log.error(`[Jobs] tools.usage.recorded not sent (payload does not match the contract): ${stats.lastInvalid}`);
continue;
continue;
}
}
- db.transaction(() => {
- const env = outbox.enqueue({
+ await db.tx(async (t) => {
+ const env = await outbox.enqueue(t, {
event_type: 'tools.usage.recorded',
event_type: 'tools.usage.recorded',
actor: { type: 'service', id: 'tools' },
actor: { type: 'service', id: 'tools' },
subject: { type: 'project', id: row.project_id },
subject: { type: 'project', id: row.project_id },
@@ -150,16 +132,20 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),
@@ -150,16 +132,20 @@ function createJobUsage({ db, contracts, outbox = null, now = () => Date.now(),
priority: 'low',
priority: 'low',
payload,
payload,
});
});
- q.sent.run(at, env.event_id, row.project_id, row.env, row.capability, row.dimension, row.window_start);
- })();
+ await q.sent.run({ app, ...row, event_id: env.event_id, at });
+ });
queued++;
queued++;
}
}
- q.prune.run(at - KEEP_SENT_MS);
+ await q.prune.run({ app, at: at - KEEP_SENT_MS });
if (queued) outbox.kick();
if (queued) outbox.kick();
return { queued, invalid: stats.invalid };
return { queued, invalid: stats.invalid };
}
}
- return { enabled: true, finished, flush, payloadOf, pending: () => q.pending.get().n, status: () => ({ pending: q.pending.get().n, invalid: stats.invalid, last_invalid: stats.lastInvalid }) };
+ return {
+ enabled: true, finished, flush, payloadOf,
+ pending: async () => (await q.pending.get({ app })).n,
+ status: async () => ({ pending: (await q.pending.get({ app })).n, invalid: stats.invalid, last_invalid: stats.lastInvalid }),
+ };
}
}
module.exports = { createJobUsage, keyOf, HOUR_MS, GRACE_MS };
module.exports = { createJobUsage, keyOf, HOUR_MS, GRACE_MS };
@@ -52,18 +52,18 @@ function observe(o) {
@@ -52,18 +52,18 @@ function observe(o) {
if (o.jobs) {
if (o.jobs) {
inst.registry.gauge({
inst.registry.gauge({
name: 'tools_jobs', help: 'Jobs in this satellite\'s job store by state', labelNames: ['app', 'state'],
name: 'tools_jobs', help: 'Jobs in this satellite\'s job store by state', labelNames: ['app', 'state'],
- collect: () => {
+ collect: async () => {
const sys = o.jobs();
const sys = o.jobs();
if (!sys) return null;
if (!sys) return null;
- return Object.entries(sys.store.counts()).map(([state, n]) => ({ labels: { app: appName, state }, value: n }));
+ return Object.entries(await sys.store.counts()).map(([state, n]) => ({ labels: { app: appName, state }, value: n }));
},
},
});
});
inst.registry.gauge({
inst.registry.gauge({
name: 'tools_jobs_executing', help: 'Jobs executing in this process now, and the concurrency limit', labelNames: ['app', 'kind'],
name: 'tools_jobs_executing', help: 'Jobs executing in this process now, and the concurrency limit', labelNames: ['app', 'kind'],
- collect: () => {
+ collect: async () => {
const sys = o.jobs();
const sys = o.jobs();
if (!sys) return null;
if (!sys) return null;
- const s = sys.stats();
+ const s = await sys.stats();
return [{ labels: { app: appName, kind: 'executing' }, value: s.executing }, { labels: { app: appName, kind: 'limit' }, value: s.concurrency }];
return [{ labels: { app: appName, kind: 'executing' }, value: s.executing }, { labels: { app: appName, kind: 'limit' }, value: s.concurrency }];
},
},
});
});
@@ -119,14 +119,14 @@ const checks = {
@@ -119,14 +119,14 @@ const checks = {
},
},
/** The job runtime: started and not stopped, its store answering (counts are a real query). */
/** The job runtime: started and not stopped, its store answering (counts are a real query). */
- jobRuntime(name, getSystem, { required = true, description = 'job runtime: worker started, jobs.db answering' } = {}) {
+ jobRuntime(name, getSystem, { required = true, description = 'job runtime: worker started, store answering' } = {}) {
return {
return {
name, required, description,
name, required, description,
- check: () => {
+ check: async () => {
const sys = getSystem();
const sys = getSystem();
if (!sys) return 'job runtime not set up yet';
if (!sys) return 'job runtime not set up yet';
if (!sys.isRunning()) return 'job runtime is not running';
if (!sys.isRunning()) return 'job runtime is not running';
- const s = sys.stats(); // store.counts() inside: a real query on jobs.db
+ const s = await sys.stats(); // store.counts() inside: a real query
return { ok: true, detail: { queued: s.queued, running: s.running, executing: s.executing, concurrency: s.concurrency, results: s.results } };
return { ok: true, detail: { queued: s.queued, running: s.running, executing: s.executing, concurrency: s.concurrency, results: s.results } };
},
},
};
};
@@ -46,7 +46,7 @@ const SVC = principal('svc:network');
@@ -46,7 +46,7 @@ const SVC = principal('svc:network');
const log = { log() {}, error() {}, warn: (m) => lines.push(String(m)) };
const log = { log() {}, error() {}, warn: (m) => lines.push(String(m)) };
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-actor-limits-'));
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-actor-limits-'));
const guard = createGuard({
const guard = createGuard({
- app: 'test', dataDir, Database: dep('better-sqlite3'), contracts, specs: [], issuer: ISSUER,
+ app: 'test', dataDir, contracts, specs: [], issuer: ISSUER,
keys: { get: () => publicKey, ensure: async () => publicKey }, env: { TOOLS_GUARD: 'report' }, log, pruneIntervalMs: 0,
keys: { get: () => publicKey, ensure: async () => publicKey }, env: { TOOLS_GUARD: 'report' }, log, pruneIntervalMs: 0,
});
});
const registry = metrics.createRegistry();
const registry = metrics.createRegistry();
@@ -1,181 +1,124 @@
@@ -1,181 +1,124 @@
'use strict';
'use strict';
-// ADR-021 analytics bounds for every satellite (openvibe-shared/analytics, scripts/analytics-prune.js):
-// a tracked request stores no IP, user id, city, raw user agent, raw referer or query string anywhere
-// in analytics.db; paths are route templates; uniques come from day-scoped hashes deleted after the
-// day's rollup; prune removes strictly-older raw rows in bounded batches and never touches rollups;
-// scrub rewrites legacy rows without changing a rollup counter; the CLI dry run changes nothing and
-// --apply needs --backup or --no-backup.
+// ADR-021 analytics bounds for every satellite, on PostgreSQL (plan T8, decision 5): a tracked request
+// stores no IP, user id, city, raw user agent, raw referer or query string anywhere in the analytics
+// tables; paths are route templates; uniques come from day-scoped hashes deleted after the day's rollup;
+// pruneRawEventsPg removes strictly-older raw rows and day hashes in bounded batches and never touches
+// rollups. (The SQLite tracker's scrub and file CLI are retired with better-sqlite3.)
const assert = require('assert');
const assert = require('assert');
-const crypto = require('crypto');
const fs = require('fs');
const fs = require('fs');
const os = require('os');
const os = require('os');
const path = require('path');
const path = require('path');
const { dep } = require('./deps');
const { dep } = require('./deps');
-const { AnalyticsTracker, privacy, retention } = dep('openvibe-shared/analytics');
+const { testDb, closeAllTestDbs } = require('./testdb');
+const { AnalyticsTrackerPg, privacy, pruneRawEventsPg } = dep('openvibe-shared/analytics/pg');
const { sqlTime } = dep('openvibe-shared/analytics/tracker');
const { sqlTime } = dep('openvibe-shared/analytics/tracker');
-const cli = require('../../../scripts/analytics-prune');
const express = dep('express');
const express = dep('express');
-const Database = dep('better-sqlite3');
+const DAY = 86400000;
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-tools-analytics-'));
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-tools-analytics-'));
const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';
const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';
-const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };
-const dumpAll = (db) => db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all()
- .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');
+const dayOf = (ms) => sqlTime(ms).slice(0, 10);
-function seedLegacy(db, nowMs) {
- new AnalyticsTracker(db, 'openvibe-yt', { timers: false }).destroy();
- const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)
- VALUES ('openvibe-yt', 'pageview', ?, 'GET', 200, ?, ?, ?, ?, ?, ?, ?)`);
- const cutoffMs = nowMs - 30 * 86400000;
- for (let i = 0; i < 10; i++) ins.run(`/watch/dQw4w9WgXc${i}?v=${i}`, 7, 'tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/x?y=1', sqlTime(nowMs - (40 + i) * 86400000));
- ins.run('/edge-older', null, null, '198.51.100.50', null, CHROME, '', sqlTime(cutoffMs - 1000));
- ins.run('/edge-exact', null, null, '198.51.100.51', null, CHROME, '', sqlTime(cutoffMs));
- ins.run('/api/info/abc123def?url=https://youtube.com/watch?v=x', 9, 'abcdef0123456789', '198.51.100.52', 'Oslo', FIREFOX, 'https://www.reddit.com/r/a', sqlTime(nowMs - 2 * 86400000));
- const tops = JSON.stringify([{ path: '/watch/dQw4w9WgXcQ', cnt: 3 }, { path: '/watch/9bZkp7q19f0', cnt: 2 }]);
- const refs = JSON.stringify([{ referer: 'https://www.google.com/search?q=x', cnt: 4 }]);
- db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
- .run('openvibe-yt', sqlTime(nowMs - 60 * 86400000).slice(0, 10), 100, 40, 30, 10, 2, tops, refs);
- db.prepare('INSERT INTO analytics_hourly (service, hour, pageviews, api_calls, unique_visitors, unique_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
- .run('openvibe-yt', sqlTime(nowMs - 3 * 86400000).slice(0, 13) + ':00:00', 10, 4, 3, 1, tops, refs);
- db.prepare("INSERT INTO analytics_rate_tracking (ip, window_start, hit_count) VALUES ('198.51.100.99', 1, 1)").run();
+/** Every analytics row as text, to prove a needle is nowhere. */
+async function dumpAll(db) {
+ const parts = [];
+ for (const t of ['analytics_events', 'analytics_hourly', 'analytics_daily', 'analytics_visitor_days', 'analytics_day_salts']) {
+ parts.push(JSON.stringify(await db.prepare(`SELECT * FROM ${t}`).all()));
+ }
+ return parts.join('\n');
}
}
(async () => {
(async () => {
- // ── Templates and reducers ──
- const n = privacy.normalisePath;
- assert.strictEqual(n('/watch/dQw4w9WgXcQ?v=1'), '/watch/:param');
- assert.strictEqual(n('/api/info?url=https://youtube.com/watch?v=abc'), '/api/info');
- assert.strictEqual(n('/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events'), '/api/jobs/:param/events');
- assert.strictEqual(n('/recipe/chicken-tikka'), '/recipe/:param');
- assert.strictEqual(n('/place/@52.37,4.89,14z'), '/place/@:user');
- assert.strictEqual(n('/@alex'), '/@:user');
- assert.strictEqual(n('/api/things/:id'), '/api/things/:id');
- assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=secret'), 'https://www.google.com');
- assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');
+ try {
+ // ── Templates and reducers ──
+ const n = privacy.normalisePath;
+ assert.strictEqual(n('/watch/dQw4w9WgXcQ?v=1'), '/watch/:param');
+ assert.strictEqual(n('/api/info?url=https://youtube.com/watch?v=abc'), '/api/info');
+ assert.strictEqual(n('/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events'), '/api/jobs/:param/events');
+ assert.strictEqual(n('/recipe/chicken-tikka'), '/recipe/:param');
+ assert.strictEqual(n('/place/@52.37,4.89,14z'), '/place/@:user');
+ assert.strictEqual(n('/@alex'), '/@:user');
+ assert.strictEqual(n('/api/things/:id'), '/api/things/:id');
+ assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=secret'), 'https://www.google.com');
+ assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');
- // ── A tracked request through express ──
- {
- const db = newDb('track.db');
- let clock = Date.parse('2026-09-23T10:15:00Z');
- const tracker = new AnalyticsTracker(db, 'openvibe-yt', { timers: false, now: () => clock });
- const app = express();
- app.set('trust proxy', true);
- app.use(tracker.middleware());
- app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT }; next(); });
- app.get('/api/jobs/:id', (req, res) => res.json({ ok: true }));
- app.get('*', (req, res) => res.send('ok'));
- const server = app.listen(0, '127.0.0.1');
- await new Promise((r) => server.once('listening', r));
- const base = `http://127.0.0.1:${server.address().port}`;
- const hdr = (x) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', ...x });
- try {
- await (await fetch(`${base}/api/jobs/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();
- await (await fetch(`${base}/watch/dQw4w9WgXcQ?v=1`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();
- await new Promise((r) => setTimeout(r, 50));
- } finally { server.close(); }
- tracker.flush();
- const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();
- assert.deepStrictEqual(rows.map((r) => r.path), ['/api/jobs/:id', '/watch/:param']);
- assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0]);
- for (const r of rows) {
- assert.ok(r.ip === null && r.user_id === null && r.city === null);
- assert.strictEqual(r.referer, 'https://www.google.com');
- assert.ok(/^[0-9a-f]{16}$/.test(r.session_id));
- }
- const all = dumpAll(db);
- for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'supersecret', 'secret-query', '98765', 'dQw4w9WgXcQ', 'Mozilla/5.0']) {
- assert.ok(!all.includes(needle), `found ${needle}`);
+ // ── A tracked request through express ──
+ {
+ const db = await testDb(path.join(tmp, 'track'));
+ let clock = Date.parse('2026-09-23T10:15:00Z');
+ const tracker = new AnalyticsTrackerPg(db, 'openvibe-yt', { timers: false, now: () => clock });
+ const app = express();
+ app.set('trust proxy', true);
+ app.use(tracker.middleware());
+ app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT }; next(); });
+ app.get('/api/jobs/:id', (req, res) => res.json({ ok: true }));
+ app.get('*', (req, res) => res.send('ok'));
+ const server = app.listen(0, '127.0.0.1');
+ await new Promise((r) => server.once('listening', r));
+ const base = `http://127.0.0.1:${server.address().port}`;
+ const hdr = (x) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', ...x });
+ try {
+ await (await fetch(`${base}/api/jobs/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();
+ await (await fetch(`${base}/watch/dQw4w9WgXcQ?v=1`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();
+ await new Promise((r) => setTimeout(r, 50));
+ } finally { server.close(); }
+ await tracker.flush();
+ const rows = await db.prepare('SELECT * FROM analytics_events ORDER BY id').all();
+ assert.deepStrictEqual(rows.map((r) => r.path), ['/api/jobs/:id', '/watch/:param']);
+ assert.deepStrictEqual(rows.map((r) => Number(r.authenticated)), [1, 0]);
+ for (const r of rows) {
+ assert.ok(r.ip === null && r.user_id === null && r.city === null);
+ assert.strictEqual(r.referer, 'https://www.google.com');
+ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id));
+ }
+ const all = await dumpAll(db);
+ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'supersecret', 'secret-query', '98765', 'dQw4w9WgXcQ', 'Mozilla/5.0']) {
+ assert.ok(!all.includes(needle), `found ${needle}`);
+ }
+ await tracker.aggregate();
+ const d = await db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();
+ assert.strictEqual(Number(d.unique_visitors), 2);
+ assert.strictEqual(Number(d.unique_users), 1);
+ clock = Date.parse('2026-09-24T00:10:00Z');
+ await tracker.aggregate();
+ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_visitor_days').get()).n), 0);
+ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_day_salts').get()).n), 0);
+ assert.strictEqual(Number((await db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").get()).unique_visitors), 2);
+ assert.ok((await tracker.getStats({ days: 30 })).summary);
+ assert.ok(Array.isArray((await tracker.getBotAnalysis(7)).topBotIPs));
+ await tracker.destroy();
}
}
- tracker.aggregate();
- const d = db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();
- assert.strictEqual(d.unique_visitors, 2);
- assert.strictEqual(d.unique_users, 1);
- clock = Date.parse('2026-09-24T00:10:00Z');
- tracker.aggregate();
- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 0);
- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 0);
- assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2);
- assert.ok(tracker.getStats({ days: 30 }).summary);
- assert.ok(Array.isArray(tracker.getBotAnalysis(7).topBotIPs));
- tracker.destroy();
- db.close();
- }
-
- // ── Prune boundaries, rollups untouched ──
- {
- const db = newDb('prune.db');
- const nowMs = Date.parse('2026-09-23T12:00:00Z');
- seedLegacy(db, nowMs);
- const totals = retention.rollupTotals(db);
- const part = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, maxBatches: 1, now: () => nowMs });
- assert.deepStrictEqual([part.deleted, part.complete], [4, false]);
- const rest = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, now: () => nowMs });
- assert.strictEqual(rest.deleted, 7);
- assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all().map((p) => p.split('?')[0]), ['/edge-exact', '/api/info/abc123def']);
- assert.deepStrictEqual(retention.rollupTotals(db), totals);
- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
-
- // Scrub the survivors and the rollup lists.
- await retention.scrubEvents(db, { batchSize: 1 });
- retention.scrubRollups(db);
- assert.deepStrictEqual(retention.rollupTotals(db), totals);
- const rows = db.prepare('SELECT * FROM analytics_events ORDER BY created_at').all();
- assert.deepStrictEqual(rows.map((r) => r.path), ['/edge-exact', '/api/info/:id']);
- assert.deepStrictEqual(rows.map((r) => r.authenticated), [0, 1]);
- assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()), [{ path: '/watch/:param', cnt: 5 }]);
- const all = dumpAll(db);
- for (const needle of ['198.51.100.', 'Lyon', 'Oslo', 'q=x', 'dQw4w9WgXcQ', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);
- db.close();
- }
-
- // ── CLI ──
- {
- const out = [];
- const log = (l) => out.push(l);
- const make = (name) => {
- const file = path.join(tmp, name, 'data', 'analytics.db');
- fs.mkdirSync(path.dirname(file), { recursive: true });
- const db = new Database(file);
- db.pragma('journal_mode = WAL');
- seedLegacy(db, Date.now() + 120000); // edge rows clear of the CLI's own cutoff: 10 older rows
- db.pragma('wal_checkpoint(TRUNCATE)');
- db.close();
- return file;
- };
- const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
- const yt = make('yt');
- const food = make('food');
- const hashes = [sha(yt), sha(food)];
-
- assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--scrub'], log), 0);
- assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'dry run changed a file');
- assert.ok(out.some((l) => /prune\s+10 rows/.test(l)), out.join('\n'));
- assert.ok(out.some((l) => /dry run: nothing changed/.test(l)));
- assert.strictEqual(await cli.main(['--db', yt, '--apply'], log), 2);
- assert.strictEqual(await cli.main(['--db', yt, '--apply', '--no-backup', '--days', '45'], log), 2);
- assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'a refused run changed a file');
+ // ── Prune boundaries: strictly-older raw rows and day hashes go; rollups stay ──
+ {
+ const db = await testDb(path.join(tmp, 'prune'));
+ const nowMs = Date.parse('2026-09-23T12:00:00Z');
+ const ins = db.prepare("INSERT INTO analytics_events (service, event_type, path, created_at) VALUES ('openvibe-yt', 'pageview', ?, ?)");
+ for (let i = 0; i < 5; i++) await ins.run(`/old/${i}`, sqlTime(nowMs - 40 * DAY));
+ await ins.run('/recent', sqlTime(nowMs - 20 * DAY));
+ const vd = db.prepare('INSERT INTO analytics_visitor_days (service, day, hour, vhash) VALUES (?, ?, ?, ?)');
+ await vd.run('openvibe-yt', dayOf(nowMs - 2 * DAY), dayOf(nowMs - 2 * DAY) + 'T00', 'a'.repeat(16));
+ await vd.run('openvibe-yt', dayOf(nowMs), dayOf(nowMs) + 'T00', 'b'.repeat(16));
+ const ds = db.prepare('INSERT INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)');
+ await ds.run('openvibe-yt', dayOf(nowMs - 2 * DAY), 'deadbeef');
+ await ds.run('openvibe-yt', dayOf(nowMs), 'cafebabe');
+ await db.prepare("INSERT INTO analytics_daily (service, date, pageviews) VALUES ('openvibe-yt', ?, 100)").run(dayOf(nowMs - 40 * DAY));
- const bdir = path.join(tmp, 'backups');
- assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--scrub', '--backup', bdir], log), 0, out.join('\n'));
- for (const [name, file] of [['yt', yt], ['food', food]]) {
- const b = new Database(path.join(bdir, `${name}.analytics.db`), { readonly: true });
- assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13);
- b.close();
- const db = new Database(file, { readonly: true });
- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 3);
- assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events WHERE ip IS NOT NULL OR user_id IS NOT NULL OR city IS NOT NULL').pluck().get(), 0);
- assert.strictEqual(db.prepare('SELECT SUM(pageviews) FROM analytics_daily').pluck().get(), 100);
- db.close();
+ const r = await pruneRawEventsPg(db, { days: 30, now: () => nowMs });
+ assert.strictEqual(r.removed, 5, 'only the strictly-older raw rows');
+ const left = (await db.prepare('SELECT path FROM analytics_events ORDER BY path').all()).map((x) => x.path);
+ assert.deepStrictEqual(left, ['/recent']);
+ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_visitor_days').get()).n), 1, "today's visitor day is kept");
+ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM analytics_day_salts').get()).n), 1, "today's salt is kept");
+ assert.strictEqual(Number((await db.prepare('SELECT pageviews FROM analytics_daily WHERE date = ?').get(dayOf(nowMs - 40 * DAY))).pageviews), 100, 'rollups are never pruned');
}
}
- // Existing backup targets are refused.
- assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--backup', bdir], log), 2);
- }
- fs.rmSync(tmp, { recursive: true, force: true });
- console.log('analytics ADR-021: tracked request, templates, rollups, prune, scrub, CLI ok');
+ console.log('analytics ADR-021 (PostgreSQL): tracked request, templates, rollups, prune ok');
+ } finally {
+ await closeAllTestDbs();
+ fs.rmSync(tmp, { recursive: true, force: true });
+ }
})().catch((e) => { console.error(e); process.exit(1); });
})().catch((e) => { console.error(e); process.exit(1); });
@@ -16,7 +16,6 @@ const { dep } = require('./deps');
@@ -16,7 +16,6 @@ const { dep } = require('./deps');
const express = dep('express');
const express = dep('express');
const cookieParser = dep('cookie-parser');
const cookieParser = dep('cookie-parser');
const multer = dep('multer');
const multer = dep('multer');
-const Database = dep('better-sqlite3'); // the job store is still SQLite (Phases 2–3 not done)
const contracts = dep('openvibe-contracts');
const contracts = dep('openvibe-contracts');
const metrics = dep('openvibe-shared/metrics');
const metrics = dep('openvibe-shared/metrics');
const guardLib = require('../guard');
const guardLib = require('../guard');
@@ -27,6 +26,7 @@ const { createGuardStore } = require('../guard/store');
@@ -27,6 +26,7 @@ const { createGuardStore } = require('../guard/store');
const { ipBucket } = require('../guard/ip');
const { ipBucket } = require('../guard/ip');
const sniff = require('../guard/sniff');
const sniff = require('../guard/sniff');
const jobs = require('../jobs');
const jobs = require('../jobs');
+const { testDb, closeAllTestDbs } = require('./testdb');
const ISSUER = 'https://openvibe.network';
const ISSUER = 'https://openvibe.network';
const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
@@ -494,10 +494,10 @@ async function serve(app) {
@@ -494,10 +494,10 @@ async function serve(app) {
// ── The job store's bounds: queued jobs and disk (system.busy) through the guard ──
// ── The job store's bounds: queued jobs and disk (system.busy) through the guard ──
for (const mode of ['enforce', 'report']) {
for (const mode of ['enforce', 'report']) {
const dataDir = tmp();
const dataDir = tmp();
- const db = new Database(':memory:');
+ const db = await testDb(dataDir);
const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, maxQueued: 2, diskBudgetBytes: 1_000_000, log: quiet });
const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, maxQueued: 2, diskBudgetBytes: 1_000_000, log: quiet });
system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });
system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });
- system.start();
+ await system.start();
const g = makeGuard({ mode });
const g = makeGuard({ mode });
const app = express();
const app = express();
app.set('trust proxy', TRUST_PROXY);
app.set('trust proxy', TRUST_PROXY);
@@ -509,7 +509,7 @@ async function serve(app) {
@@ -509,7 +509,7 @@ async function serve(app) {
try {
try {
assert.strictEqual((await submit()).status, 202);
assert.strictEqual((await submit()).status, 202);
assert.strictEqual((await submit()).status, 202);
assert.strictEqual((await submit()).status, 202);
- assert.strictEqual(system.busy().reason, 'queue');
+ assert.strictEqual((await system.busy()).reason, 'queue');
const r = await submit();
const r = await submit();
if (mode === 'enforce') {
if (mode === 'enforce') {
assert.strictEqual(r.status, 503, 'a third queued job is refused');
assert.strictEqual(r.status, 503, 'a third queued job is refused');
@@ -519,15 +519,15 @@ async function serve(app) {
@@ -519,15 +519,15 @@ async function serve(app) {
assert.strictEqual(r.status, 202, 'report mode accepts it');
assert.strictEqual(r.status, 202, 'report mode accepts it');
assert.ok(g.store.abuseRows().some(x => x.reason === 'busy.queue' && x.enforced === 0));
assert.ok(g.store.abuseRows().some(x => x.reason === 'busy.queue' && x.enforced === 0));
}
}
- } finally { await s.close(); g.close(); system.stop(); db.close(); fs.rmSync(dataDir, { recursive: true, force: true }); }
+ } finally { await s.close(); g.close(); system.stop(); fs.rmSync(dataDir, { recursive: true, force: true }); }
}
}
// Unfinished browser-session jobs are bounded per address too: a new cookie is no new allowance.
// Unfinished browser-session jobs are bounded per address too: a new cookie is no new allowance.
for (const mode of ['enforce', 'report']) {
for (const mode of ['enforce', 'report']) {
const dataDir = tmp();
const dataDir = tmp();
- const db = new Database(':memory:');
+ const db = await testDb(dataDir);
const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, maxActivePerOwner: 1, log: quiet });
const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, maxActivePerOwner: 1, log: quiet });
system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });
system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });
- system.start();
+ await system.start();
assert.strictEqual(system.bounds.maxActivePerAddress, 3, '3 × the per-owner bound by default');
assert.strictEqual(system.bounds.maxActivePerAddress, 3, '3 × the per-owner bound by default');
const g = makeGuard({ mode });
const g = makeGuard({ mode });
const app = express();
const app = express();
@@ -549,24 +549,25 @@ async function serve(app) {
@@ -549,24 +549,25 @@ async function serve(app) {
for (let i = 0; i < 3; i++) codes.push((await submit('198.51.100.60')).status); // a fresh cookie each time
for (let i = 0; i < 3; i++) codes.push((await submit('198.51.100.60')).status); // a fresh cookie each time
assert.deepStrictEqual(codes, mode === 'enforce' ? [202, 202, 429] : [202, 202, 202], `${mode}: three per address`);
assert.deepStrictEqual(codes, mode === 'enforce' ? [202, 202, 429] : [202, 202, 202], `${mode}: three per address`);
assert.strictEqual((await submit('198.51.100.61')).status, 202, 'another address is fine');
assert.strictEqual((await submit('198.51.100.61')).status, 202, 'another address is fine');
- const row = db.prepare('SELECT ip_key FROM tool_jobs LIMIT 1').get();
+ const row = await db.prepare('SELECT ip_key FROM tool_jobs LIMIT 1').get();
assert.match(row.ip_key, /^ip:[0-9a-f]{32}$/, 'the job keeps the hashed key, never the address');
assert.match(row.ip_key, /^ip:[0-9a-f]{32}$/, 'the job keeps the hashed key, never the address');
if (mode === 'report') assert.ok(g.store.abuseRows().some(x => x.reason === 'jobs.address' && x.enforced === 0));
if (mode === 'report') assert.ok(g.store.abuseRows().some(x => x.reason === 'jobs.address' && x.enforced === 0));
- } finally { await s.close(); g.close(); system.stop(); db.close(); fs.rmSync(dataDir, { recursive: true, force: true }); }
+ } finally { await s.close(); g.close(); system.stop(); fs.rmSync(dataDir, { recursive: true, force: true }); }
}
}
{
{
const dataDir = tmp();
const dataDir = tmp();
- const db = new Database(':memory:');
+ const db = await testDb(dataDir);
const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, diskBudgetBytes: 1000, log: quiet });
const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir, concurrency: 0, diskBudgetBytes: 1000, log: quiet });
system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });
system.define({ type: 'test.noop', maxFiles: 1, async run() { return { files: [], data: {} }; } });
- system.start();
+ await system.start();
await system.refreshDisk();
await system.refreshDisk();
- assert.strictEqual(system.busy(), null);
+ assert.strictEqual(await system.busy(), null);
await system.submit({ owner: 'session:x', type: 'test.noop', input: {}, files: [{ buffer: Buffer.alloc(1500), name: 'a.bin', mime: 'application/octet-stream', size: 1500 }] });
await system.submit({ owner: 'session:x', type: 'test.noop', input: {}, files: [{ buffer: Buffer.alloc(1500), name: 'a.bin', mime: 'application/octet-stream', size: 1500 }] });
- assert.strictEqual(system.busy().reason, 'disk', 'accepted inputs count against the disk budget at once');
+ assert.strictEqual((await system.busy()).reason, 'disk', 'accepted inputs count against the disk budget at once');
assert.strictEqual((await system.refreshDisk()) >= 1500, true, 'and the walk finds them');
assert.strictEqual((await system.refreshDisk()) >= 1500, true, 'and the walk finds them');
- system.stop(); db.close(); fs.rmSync(dataDir, { recursive: true, force: true });
+ system.stop(); fs.rmSync(dataDir, { recursive: true, force: true });
}
}
+ await closeAllTestDbs();
console.log('guard: callers (/64, one loopback hop, services by token, aud-checked users, sessions, sandbox), bucket math + cost, session sharing, report vs enforce, headers + problem+json, two processes one Valkey quota + outage fallback, sniffing, target throttle, port cap, abuse log + prune + metric, semaphore, job bounds (queue, disk, per address), challenge hook, Origin check (CSRF): all checks passed');
console.log('guard: callers (/64, one loopback hop, services by token, aud-checked users, sessions, sandbox), bucket math + cost, session sharing, report vs enforce, headers + problem+json, two processes one Valkey quota + outage fallback, sniffing, target throttle, port cap, abuse log + prune + metric, semaphore, job bounds (queue, disk, per address), challenge hook, Origin check (CSRF): all checks passed');
})().catch((err) => { console.error(err); process.exit(1); });
})().catch((err) => { console.error(err); process.exit(1); });
@@ -1,16 +1,16 @@
@@ -1,16 +1,16 @@
'use strict';
'use strict';
// tools.job.* → OpenVibe.Events (roadmap Wave 11): every transition writes exactly one event to the
// tools.job.* → OpenVibe.Events (roadmap Wave 11): every transition writes exactly one event to the
-// jobs database's outbox in the transaction that records it; a rolled-back transition writes none;
-// sandbox jobs and cancellations write none; payloads carry no input, file name, output data or
-// browser session and validate against openvibe-contracts; the relay posts to EVENTS_URL with the
-// tools service token; and without EVENTS_URL nothing is set up at all.
+// tools database's outbox (openvibe-sdk createPgOutbox) in the transaction that records it; a rolled-back
+// transition writes none; sandbox jobs and cancellations write none; payloads carry no input, file name,
+// output data or browser session and validate against openvibe-contracts; the relay posts to EVENTS_URL
+// with the tools service token; and without EVENTS_URL nothing is set up at all.
const assert = require('assert');
const assert = require('assert');
const fs = require('fs');
const fs = require('fs');
const os = require('os');
const os = require('os');
const path = require('path');
const path = require('path');
const { dep } = require('./deps');
const { dep } = require('./deps');
+const { testDb, closeAllTestDbs } = require('./testdb');
-const Database = dep('better-sqlite3');
const contracts = dep('openvibe-contracts');
const contracts = dep('openvibe-contracts');
const sdk = dep('openvibe-sdk');
const sdk = dep('openvibe-sdk');
const jobs = require('../jobs');
const jobs = require('../jobs');
@@ -100,17 +100,19 @@ function fakeNetwork() {
@@ -100,17 +100,19 @@ function fakeNetwork() {
const ENV = { EVENTS_URL: 'http://events.test/', OV_OAUTH_CLIENT_SECRET: 'tools-secret', OV_NETWORK_INTERNAL_URL: 'http://network.test' };
const ENV = { EVENTS_URL: 'http://events.test/', OV_OAUTH_CLIENT_SECRET: 'tools-secret', OV_NETWORK_INTERNAL_URL: 'http://network.test' };
-function satellite(dir, { wrap, concurrency = 1, net } = {}) {
+async function satellite(dir, { wrap, concurrency = 1, net } = {}) {
fs.mkdirSync(dir, { recursive: true });
fs.mkdirSync(dir, { recursive: true });
- const db = new Database(path.join(dir, 'jobs.db'));
+ const db = await testDb(dir);
const { outbox, reason } = outboxFromEnv({ db, sdk, env: ENV, fetch: net.fetch, log: silent });
const { outbox, reason } = outboxFromEnv({ db, sdk, env: ENV, fetch: net.fetch, log: silent });
assert.ok(outbox, `outbox configured (${reason})`);
assert.ok(outbox, `outbox configured (${reason})`);
+ await outbox.ready;
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, concurrency, outbox: wrap ? wrap(outbox) : outbox, progressThrottleMs: 0, log: silent });
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, concurrency, outbox: wrap ? wrap(outbox) : outbox, progressThrottleMs: 0, log: silent });
define(system);
define(system);
- system.start();
- const envelopes = (jobId) => db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all().map(r => JSON.parse(r.envelope))
+ await system.start();
+ const envelopes = async (jobId) => (await db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all())
+ .map(r => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope))
.filter(e => !jobId || e.subject.id === jobId);
.filter(e => !jobId || e.subject.id === jobId);
- return { db, system, outbox, envelopes, close() { system.stop(); outbox.stop(); db.close(); } };
+ return { db, system, outbox, envelopes, close: async () => { system.stop(); await outbox.stop(); } };
}
}
const types = (list) => list.map(e => e.event_type);
const types = (list) => list.map(e => e.event_type);
@@ -135,28 +137,28 @@ function noLeaks(list, extra = []) {
@@ -135,28 +137,28 @@ function noLeaks(list, extra = []) {
}
}
}
}
}
}
-const settled = (s, id, state) => until(() => { const r = s.system.get(id); return r && r.state === state && r; }, `${id} ${state}`);
+const settled = (s, id, state) => until(async () => { const r = await s.system.get(id); return r && r.state === state && r; }, `${id} ${state}`);
(async () => {
(async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-job-events-'));
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-job-events-'));
try {
try {
// ── Inert without EVENTS_URL ─────────────────────────────
// ── Inert without EVENTS_URL ─────────────────────────────
{
{
- const db = new Database(':memory:');
+ const db = await testDb(path.join(root, 'inert-db'));
assert.deepStrictEqual(outboxFromEnv({ db, sdk, env: {} }), { outbox: null, reason: 'EVENTS_URL is not set' });
assert.deepStrictEqual(outboxFromEnv({ db, sdk, env: {} }), { outbox: null, reason: 'EVENTS_URL is not set' });
assert.strictEqual(outboxFromEnv({ db, sdk, env: { EVENTS_URL: 'http://events.test' } }).outbox, null, 'no secret, no outbox');
assert.strictEqual(outboxFromEnv({ db, sdk, env: { EVENTS_URL: 'http://events.test' } }).outbox, null, 'no secret, no outbox');
assert.strictEqual(outboxFromEnv({ db, sdk, env: { ...ENV, EVENTS_PUBLISH: 'off' } }).outbox, null, 'EVENTS_PUBLISH=off');
assert.strictEqual(outboxFromEnv({ db, sdk, env: { ...ENV, EVENTS_PUBLISH: 'off' } }).outbox, null, 'EVENTS_PUBLISH=off');
assert.strictEqual(outboxFromEnv({ db, sdk: null, env: ENV }).outbox, null, 'no sdk, no outbox');
assert.strictEqual(outboxFromEnv({ db, sdk: null, env: ENV }).outbox, null, 'no sdk, no outbox');
const dir = path.join(root, 'inert');
const dir = path.join(root, 'inert');
- fs.mkdirSync(dir);
+ fs.mkdirSync(dir, { recursive: true });
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, log: silent });
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, log: silent });
define(system);
define(system);
- system.start();
+ await system.start();
const { job } = await system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'x' } });
const { job } = await system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'x' } });
- await until(() => system.get(job.id).state === 'succeeded', 'inert job');
- assert.strictEqual(db.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE name = 'event_outbox'").get().n, 0, 'no outbox table without EVENTS_URL');
- assert.deepStrictEqual(system.stats().events, { enabled: false });
- system.stop(); db.close();
+ await until(async () => (await system.get(job.id)).state === 'succeeded', 'inert job');
+ assert.strictEqual((await db.prepare("SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_name = 'event_outbox'").get()).n, 0, 'no outbox table without EVENTS_URL');
+ assert.deepStrictEqual((await system.stats()).events, { enabled: false });
+ system.stop();
}
}
const net = fakeNetwork();
const net = fakeNetwork();
@@ -164,18 +166,18 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
@@ -164,18 +166,18 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
let failOn = null;
let failOn = null;
const wrap = (outbox) => ({
const wrap = (outbox) => ({
...outbox,
...outbox,
- enqueue(envelope, opts) {
- const env = outbox.enqueue(envelope, opts);
+ async enqueue(t, envelope, opts) {
+ const env = await outbox.enqueue(t, envelope, opts);
if (failOn === envelope.event_type) throw new Error('disk I/O error (test)');
if (failOn === envelope.event_type) throw new Error('disk I/O error (test)');
return env;
return env;
},
},
});
});
- let s = satellite(path.join(root, 'a'), { net, wrap });
+ let s = await satellite(path.join(root, 'a'), { net, wrap });
// ── A signed-in person's job: created, started, succeeded — once each ──
// ── A signed-in person's job: created, started, succeeded — once each ──
let { job } = await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: SECRET_INPUT }, files: [{ buffer: Buffer.from('hello'), name: SECRET_FILE, mime: 'text/plain' }] });
let { job } = await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: SECRET_INPUT }, files: [{ buffer: Buffer.from('hello'), name: SECRET_FILE, mime: 'text/plain' }] });
await settled(s, job.id, 'succeeded');
await settled(s, job.id, 'succeeded');
- let list = s.envelopes(job.id);
+ let list = await s.envelopes(job.id);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
list.forEach(checkContracts);
list.forEach(checkContracts);
noLeaks(list);
noLeaks(list);
@@ -195,12 +197,20 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
@@ -195,12 +197,20 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
assert.strictEqual(done.result.files[0].storage, 'local');
assert.strictEqual(done.result.files[0].storage, 'local');
assert.strictEqual(done.result.files[0].media, null);
assert.strictEqual(done.result.files[0].media, null);
assert.strictEqual(done.result.files[0].size, 5);
assert.strictEqual(done.result.files[0].size, 5);
- assert.deepStrictEqual(s.system.stats().events.enabled, true);
+ assert.deepStrictEqual((await s.system.stats()).events.enabled, true);
+
+ // ── A stream that dropped mid-job resumes with Last-Event-ID: the log from there is exactly the
+ // tail of the whole sequence, seq for seq — no gap, no repeat (the SSE reconnect path) ──
+ const log = await s.system.eventsAfter(job.id, 0);
+ assert.deepStrictEqual(log.map(e => e.event), ['job.queued', 'job.running', 'job.succeeded']);
+ const resumed = await s.system.eventsAfter(job.id, log[0].seq);
+ assert.deepStrictEqual(resumed.map(e => e.event), ['job.running', 'job.succeeded'], 'exactly the events after the last id the client saw');
+ assert.deepStrictEqual(resumed.map(e => e.seq), log.slice(1).map(e => e.seq), 'the same per-job seq values, in order');
// ── A failure whose message names the input file: failed once, name and path scrubbed ──
// ── A failure whose message names the input file: failed once, name and path scrubbed ──
({ job } = await s.system.submit({ owner: `user:${USER}`, type: 'test.boom', input: { text: SECRET_INPUT }, files: [{ buffer: Buffer.from('x'), name: SECRET_FILE, mime: 'text/plain' }] }));
({ job } = await s.system.submit({ owner: `user:${USER}`, type: 'test.boom', input: { text: SECRET_INPUT }, files: [{ buffer: Buffer.from('x'), name: SECRET_FILE, mime: 'text/plain' }] }));
await settled(s, job.id, 'failed');
await settled(s, job.id, 'failed');
- list = s.envelopes(job.id);
+ list = await s.envelopes(job.id);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.failed']);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.failed']);
list.forEach(checkContracts);
list.forEach(checkContracts);
noLeaks(list);
noLeaks(list);
@@ -214,7 +224,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
@@ -214,7 +224,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
// ── A browser session's job: owner null, the service acts, the session never appears ──
// ── A browser session's job: owner null, the service acts, the session never appears ──
({ job } = await s.system.submit({ owner: SESSION, type: 'test.upper', input: { text: SECRET_INPUT } }));
({ job } = await s.system.submit({ owner: SESSION, type: 'test.upper', input: { text: SECRET_INPUT } }));
await settled(s, job.id, 'succeeded');
await settled(s, job.id, 'succeeded');
- list = s.envelopes(job.id);
+ list = await s.envelopes(job.id);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
list.forEach(checkContracts);
list.forEach(checkContracts);
noLeaks(list);
noLeaks(list);
@@ -223,7 +233,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
@@ -223,7 +233,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
// ── Principals: a service and a production app are subjects ──
// ── Principals: a service and a production app are subjects ──
({ job } = await s.system.submit({ owner: 'svc:live', type: 'test.upper', input: {} }));
({ job } = await s.system.submit({ owner: 'svc:live', type: 'test.upper', input: {} }));
await settled(s, job.id, 'succeeded');
await settled(s, job.id, 'succeeded');
- list = s.envelopes(job.id);
+ list = await s.envelopes(job.id);
list.forEach(checkContracts);
list.forEach(checkContracts);
assert.deepStrictEqual(list[0].actor, { type: 'service', id: 'live' });
assert.deepStrictEqual(list[0].actor, { type: 'service', id: 'live' });
assert.deepStrictEqual(list[0].payload.owner, { type: 'service', id: 'live' });
assert.deepStrictEqual(list[0].payload.owner, { type: 'service', id: 'live' });
@@ -231,90 +241,90 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
@@ -231,90 +241,90 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
assert.strictEqual(ownerRef(contracts, 'app:nope'), null);
assert.strictEqual(ownerRef(contracts, 'app:nope'), null);
// ── Sandbox app jobs are never announced ─────────────────
// ── Sandbox app jobs are never announced ─────────────────
- const before = s.envelopes().length;
+ const before = (await s.envelopes()).length;
({ job } = await s.system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: 'sandbox' }, env: 'sandbox' }));
({ job } = await s.system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: 'sandbox' }, env: 'sandbox' }));
await settled(s, job.id, 'succeeded');
await settled(s, job.id, 'succeeded');
const sandboxFail = (await s.system.submit({ owner: `app:${APP}`, type: 'test.boom', files: [{ buffer: Buffer.from('x'), name: 'a.txt' }], env: 'sandbox' })).job;
const sandboxFail = (await s.system.submit({ owner: `app:${APP}`, type: 'test.boom', files: [{ buffer: Buffer.from('x'), name: 'a.txt' }], env: 'sandbox' })).job;
await settled(s, sandboxFail.id, 'failed');
await settled(s, sandboxFail.id, 'failed');
- assert.strictEqual(s.envelopes(job.id).length + s.envelopes(sandboxFail.id).length, 0, 'sandbox jobs emit nothing');
- assert.strictEqual(s.envelopes().length, before);
+ assert.strictEqual((await s.envelopes(job.id)).length + (await s.envelopes(sandboxFail.id)).length, 0, 'sandbox jobs emit nothing');
+ assert.strictEqual((await s.envelopes()).length, before);
// ── Retry: a new created (retry_of) + its run; the failed job gets nothing new ──
// ── Retry: a new created (retry_of) + its run; the failed job gets nothing new ──
flaky.add('t1');
flaky.add('t1');
const first = (await s.system.submit({ owner: `user:${USER}`, type: 'test.flaky', input: { tag: 't1' }, files: [{ buffer: Buffer.from('abc'), name: SECRET_FILE }] })).job;
const first = (await s.system.submit({ owner: `user:${USER}`, type: 'test.flaky', input: { tag: 't1' }, files: [{ buffer: Buffer.from('abc'), name: SECRET_FILE }] })).job;
await settled(s, first.id, 'failed');
await settled(s, first.id, 'failed');
- assert.strictEqual(s.envelopes(first.id).at(-1).payload.retryable, true);
+ assert.strictEqual((await s.envelopes(first.id)).at(-1).payload.retryable, true);
flaky.delete('t1');
flaky.delete('t1');
- const retried = s.system.retry(first.id).job;
+ const retried = (await s.system.retry(first.id)).job;
await settled(s, retried.id, 'succeeded');
await settled(s, retried.id, 'succeeded');
- assert.deepStrictEqual(types(s.envelopes(first.id)), ['tools.job.created', 'tools.job.started', 'tools.job.failed'], 'the retried job gets no new event');
- list = s.envelopes(retried.id);
+ assert.deepStrictEqual(types(await s.envelopes(first.id)), ['tools.job.created', 'tools.job.started', 'tools.job.failed'], 'the retried job gets no new event');
+ list = await s.envelopes(retried.id);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
assert.deepStrictEqual(types(list), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
list.forEach(checkContracts);
list.forEach(checkContracts);
noLeaks(list);
noLeaks(list);
assert.strictEqual(list[0].payload.retry_of, first.id);
assert.strictEqual(list[0].payload.retry_of, first.id);
- assert.strictEqual(s.system.retry(first.id).replayed, true);
- assert.strictEqual(s.envelopes(retried.id).length, 3, 'a replayed retry creates no event');
+ assert.strictEqual((await s.system.retry(first.id)).replayed, true);
+ assert.strictEqual((await s.envelopes(retried.id)).length, 3, 'a replayed retry creates no event');
// ── An Idempotency-Key replay creates no job and no event ──
// ── An Idempotency-Key replay creates no job and no event ──
const k1 = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'k' }, idempotencyKey: 'key-00000001' })).job;
const k1 = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'k' }, idempotencyKey: 'key-00000001' })).job;
await settled(s, k1.id, 'succeeded');
await settled(s, k1.id, 'succeeded');
const k2 = await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'k' }, idempotencyKey: 'key-00000001' });
const k2 = await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'k' }, idempotencyKey: 'key-00000001' });
assert.strictEqual(k2.replayed, true);
assert.strictEqual(k2.replayed, true);
- assert.strictEqual(s.envelopes(k1.id).length, 3);
+ assert.strictEqual((await s.envelopes(k1.id)).length, 3);
// ── Cancelled jobs are not announced (neither queued nor running) ──
// ── Cancelled jobs are not announced (neither queued nor running) ──
const blocker = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { gate: 'g1' } })).job;
const blocker = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { gate: 'g1' } })).job;
await settled(s, blocker.id, 'running');
await settled(s, blocker.id, 'running');
const queued = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'q' } })).job;
const queued = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'q' } })).job;
- assert.strictEqual(s.system.cancel(queued.id).changed, true);
- assert.deepStrictEqual(types(s.envelopes(queued.id)), ['tools.job.created']);
- s.system.cancel(blocker.id);
+ assert.strictEqual((await s.system.cancel(queued.id)).changed, true);
+ assert.deepStrictEqual(types(await s.envelopes(queued.id)), ['tools.job.created']);
+ await s.system.cancel(blocker.id);
await settled(s, blocker.id, 'cancelled');
await settled(s, blocker.id, 'cancelled');
- assert.deepStrictEqual(types(s.envelopes(blocker.id)), ['tools.job.created', 'tools.job.started']);
+ assert.deepStrictEqual(types(await s.envelopes(blocker.id)), ['tools.job.created', 'tools.job.started']);
// ── Rollbacks: a failed outbox write undoes the transition, and no event is left ──
// ── Rollbacks: a failed outbox write undoes the transition, and no event is left ──
- const count = () => s.envelopes().length;
- let n = count();
+ const count = async () => (await s.envelopes()).length;
+ let n = await count();
failOn = 'tools.job.created';
failOn = 'tools.job.created';
await assert.rejects(s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'rollback' } }), /disk I\/O error/);
await assert.rejects(s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'rollback' } }), /disk I\/O error/);
failOn = null;
failOn = null;
- assert.strictEqual(count(), n, 'rolled-back submit: no event');
- assert.strictEqual(s.db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE input_json LIKE '%rollback%'").get().n, 0, 'rolled-back submit: no job');
+ assert.strictEqual(await count(), n, 'rolled-back submit: no event');
+ assert.strictEqual((await s.db.prepare("SELECT COUNT(*) AS n FROM tool_jobs WHERE input_json LIKE '%rollback%'").get()).n, 0, 'rolled-back submit: no job');
// claim(): the job stays queued and nothing says it started.
// claim(): the job stays queued and nothing says it started.
failOn = 'tools.job.started';
failOn = 'tools.job.started';
const stuck = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'claim' } })).job;
const stuck = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'claim' } })).job;
await sleep(50);
await sleep(50);
- assert.strictEqual(s.system.get(stuck.id).state, 'queued', 'rolled-back claim: still queued');
- assert.strictEqual(s.system.get(stuck.id).attempts, 0);
- assert.deepStrictEqual(types(s.envelopes(stuck.id)), ['tools.job.created'], 'rolled-back claim: no started event');
+ assert.strictEqual((await s.system.get(stuck.id)).state, 'queued', 'rolled-back claim: still queued');
+ assert.strictEqual((await s.system.get(stuck.id)).attempts, 0);
+ assert.deepStrictEqual(types(await s.envelopes(stuck.id)), ['tools.job.created'], 'rolled-back claim: no started event');
failOn = null;
failOn = null;
// The next kick (any submit) starts it: exactly one started.
// The next kick (any submit) starts it: exactly one started.
const kicker = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'kick' } })).job;
const kicker = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'kick' } })).job;
await settled(s, stuck.id, 'succeeded');
await settled(s, stuck.id, 'succeeded');
await settled(s, kicker.id, 'succeeded');
await settled(s, kicker.id, 'succeeded');
- assert.deepStrictEqual(types(s.envelopes(stuck.id)), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
+ assert.deepStrictEqual(types(await s.envelopes(stuck.id)), ['tools.job.created', 'tools.job.started', 'tools.job.succeeded']);
// finish(): the job stays running (recovered on the next start) and nothing says it ended.
// finish(): the job stays running (recovered on the next start) and nothing says it ended.
failOn = 'tools.job.succeeded';
failOn = 'tools.job.succeeded';
const unfinished = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'finish' } })).job;
const unfinished = (await s.system.submit({ owner: `user:${USER}`, type: 'test.upper', input: { text: 'finish' } })).job;
- await until(() => s.envelopes(unfinished.id).length === 2 && s.system.stats().executing === 0, 'finish attempt');
+ await until(async () => (await s.envelopes(unfinished.id)).length === 2 && (await s.system.stats()).executing === 0, 'finish attempt');
await sleep(30);
await sleep(30);
- assert.strictEqual(s.system.get(unfinished.id).state, 'running', 'rolled-back finish: still running');
- assert.deepStrictEqual(types(s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started']);
+ assert.strictEqual((await s.system.get(unfinished.id)).state, 'running', 'rolled-back finish: still running');
+ assert.deepStrictEqual(types(await s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started']);
failOn = null;
failOn = null;
// ── Restart: the stuck job is re-queued (not announced) and started again (announced again);
// ── Restart: the stuck job is re-queued (not announced) and started again (announced again);
// a running onRestart:'fail' job fails once with tools.job.interrupted ──
// a running onRestart:'fail' job fails once with tools.job.interrupted ──
const fragile = (await s.system.submit({ owner: `user:${USER}`, type: 'test.fragile', input: { gate: 'never' } })).job;
const fragile = (await s.system.submit({ owner: `user:${USER}`, type: 'test.fragile', input: { gate: 'never' } })).job;
await settled(s, fragile.id, 'running');
await settled(s, fragile.id, 'running');
- s.close();
- s = satellite(path.join(root, 'a'), { net });
+ await s.close();
+ s = await satellite(path.join(root, 'a'), { net });
await settled(s, unfinished.id, 'succeeded');
await settled(s, unfinished.id, 'succeeded');
- assert.deepStrictEqual(types(s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started', 'tools.job.started', 'tools.job.succeeded']);
- assert.strictEqual(s.envelopes(unfinished.id)[2].payload.attempts, 2);
- const fr = s.envelopes(fragile.id);
+ assert.deepStrictEqual(types(await s.envelopes(unfinished.id)), ['tools.job.created', 'tools.job.started', 'tools.job.started', 'tools.job.succeeded']);
+ assert.strictEqual((await s.envelopes(unfinished.id))[2].payload.attempts, 2);
+ const fr = await s.envelopes(fragile.id);
assert.deepStrictEqual(types(fr), ['tools.job.created', 'tools.job.started', 'tools.job.failed']);
assert.deepStrictEqual(types(fr), ['tools.job.created', 'tools.job.started', 'tools.job.failed']);
fr.forEach(checkContracts);
fr.forEach(checkContracts);
assert.strictEqual(fr[2].payload.error.code, 'tools.job.interrupted');
assert.strictEqual(fr[2].payload.error.code, 'tools.job.interrupted');
@@ -322,21 +332,21 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
@@ -322,21 +332,21 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
assert.ok(fr[2].payload.started_at);
assert.ok(fr[2].payload.started_at);
// ── Every event so far: valid, and never a secret ──
// ── Every event so far: valid, and never a secret ──
- const all = s.envelopes();
+ const all = await s.envelopes();
all.forEach(checkContracts);
all.forEach(checkContracts);
noLeaks(all);
noLeaks(all);
assert.strictEqual(new Set(all.map(e => e.event_id)).size, all.length, 'unique event ids');
assert.strictEqual(new Set(all.map(e => e.event_id)).size, all.length, 'unique event ids');
// ── The relay: POST EVENTS_URL/api/v1/events with the tools token for openvibe.events ──
// ── The relay: POST EVENTS_URL/api/v1/events with the tools token for openvibe.events ──
- assert.strictEqual(s.outbox.pending(), all.length);
+ assert.strictEqual(await s.outbox.pending(), all.length);
const r = await s.outbox.flush();
const r = await s.outbox.flush();
assert.strictEqual(r.sent, all.length);
assert.strictEqual(r.sent, all.length);
- assert.strictEqual(s.outbox.pending(), 0);
+ assert.strictEqual(await s.outbox.pending(), 0);
const published = net.calls.publish.flatMap(c => c.events);
const published = net.calls.publish.flatMap(c => c.events);
assert.deepStrictEqual(published.map(e => e.event_id), all.map(e => e.event_id), 'published in order, once');
assert.deepStrictEqual(published.map(e => e.event_id), all.map(e => e.event_id), 'published in order, once');
assert.ok(net.calls.publish.every(c => c.authorization === 'Bearer events-token'));
assert.ok(net.calls.publish.every(c => c.authorization === 'Bearer events-token'));
assert.deepStrictEqual(net.calls.token[0], { grant_type: 'client_credentials', client_id: 'tools', client_secret: 'tools-secret', audience: 'openvibe.events', scope: 'events.event.publish' });
assert.deepStrictEqual(net.calls.token[0], { grant_type: 'client_credentials', client_id: 'tools', client_secret: 'tools-secret', audience: 'openvibe.events', scope: 'events.event.publish' });
- s.close();
+ await s.close();
// ── Payload builder: a result in Media is { media_id, role } only ──
// ── Payload builder: a result in Media is { media_id, role } only ──
const row = {
const row = {
@@ -353,6 +363,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
@@ -353,6 +363,7 @@ const settled = (s, id, state) => until(() => { const r = s.system.get(id); retu
console.log('job events (tools.job.* outbox): all checks passed');
console.log('job events (tools.job.* outbox): all checks passed');
} finally {
} finally {
+ await closeAllTestDbs();
fs.rmSync(root, { recursive: true, force: true });
fs.rmSync(root, { recursive: true, force: true });
}
}
})().catch((err) => { console.error(err); process.exit(1); });
})().catch((err) => { console.error(err); process.exit(1); });
@@ -4,14 +4,14 @@
@@ -4,14 +4,14 @@
// failures by code with their job id and submit trace id; nobody else's jobs and no cancelled job
// failures by code with their job id and submit trace id; nobody else's jobs and no cancelled job
// count; each closed hour is written to the outbox once as tools.usage.recorded (valid against
// count; each closed hour is written to the outbox once as tools.usage.recorded (valid against
// openvibe-contracts, no owner, input or file name); a job ending in an hour already sent re-sends it
// openvibe-contracts, no owner, input or file name); a job ending in an hour already sent re-sends it
-// as revision 2; without an outbox nothing is counted.
+// as revision 2; without an outbox nothing is counted. On PostgreSQL (plan T8): openvibe-sdk/db.
const assert = require('assert');
const assert = require('assert');
const fs = require('fs');
const fs = require('fs');
const os = require('os');
const os = require('os');
const path = require('path');
const path = require('path');
const { dep } = require('./deps');
const { dep } = require('./deps');
+const { testDb, closeAllTestDbs } = require('./testdb');
-const Database = dep('better-sqlite3');
const contracts = dep('openvibe-contracts');
const contracts = dep('openvibe-contracts');
const sdk = dep('openvibe-sdk');
const sdk = dep('openvibe-sdk');
const jobs = require('../jobs');
const jobs = require('../jobs');
@@ -60,29 +60,30 @@ function define(system) {
@@ -60,29 +60,30 @@ function define(system) {
try {
try {
// ── Without an outbox: nothing counted, no table ──
// ── Without an outbox: nothing counted, no table ──
{
{
- const db = new Database(':memory:');
+ const db = await testDb(path.join(root, 'inert-db'));
const dir = path.join(root, 'inert');
const dir = path.join(root, 'inert');
- fs.mkdirSync(dir);
+ fs.mkdirSync(dir, { recursive: true });
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, log: silent });
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, log: silent });
define(system);
define(system);
- system.start();
+ await system.start();
const { job } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: 'x' }, project: PRJ });
const { job } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: 'x' }, project: PRJ });
- await until(() => system.get(job.id).state === 'succeeded', 'inert job');
- assert.strictEqual(db.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE name = 'tool_job_usage'").get().n, 0);
- assert.deepStrictEqual(system.stats().usage, { enabled: false });
- system.stop(); db.close();
+ await until(async () => (await system.get(job.id)).state === 'succeeded', 'inert job');
+ assert.strictEqual(Number((await db.prepare('SELECT COUNT(*) AS n FROM tool_job_usage').get()).n), 0, 'inert: nothing counted');
+ assert.deepStrictEqual((await system.stats()).usage, { enabled: false });
+ system.stop();
}
}
const dir = path.join(root, 'a');
const dir = path.join(root, 'a');
- fs.mkdirSync(dir);
- const db = new Database(path.join(dir, 'jobs.db'));
+ fs.mkdirSync(dir, { recursive: true });
+ const db = await testDb(dir);
const { outbox } = outboxFromEnv({ db, sdk, env: ENV, fetch: fakeFetch, log: silent });
const { outbox } = outboxFromEnv({ db, sdk, env: ENV, fetch: fakeFetch, log: silent });
+ await outbox.ready;
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, concurrency: 1, outbox, progressThrottleMs: 0, log: silent });
const system = jobs.createJobSystem({ db, contracts, service: 'img', dataDir: dir, concurrency: 1, outbox, progressThrottleMs: 0, log: silent });
define(system);
define(system);
- system.start();
+ await system.start();
const run = async (o, state) => {
const run = async (o, state) => {
const { job } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: SECRET_INPUT }, ...o });
const { job } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { text: SECRET_INPUT }, ...o });
- await until(() => { const r = system.get(job.id); return r && r.state === state; }, `${job.id} ${state}`);
+ await until(async () => { const r = await system.get(job.id); return r && r.state === state; }, `${job.id} ${state}`);
return job.id;
return job.id;
};
};
@@ -93,13 +94,13 @@ function define(system) {
@@ -93,13 +94,13 @@ function define(system) {
await run({ project: PRJ, env: 'sandbox' }, 'succeeded');
await run({ project: PRJ, env: 'sandbox' }, 'succeeded');
await run({ project: PRJ, tool: 'image-resize' }, 'succeeded');
await run({ project: PRJ, tool: 'image-resize' }, 'succeeded');
await run({ project: null, owner: `user:${USER}` }, 'succeeded'); // a person's job: no project, not counted
await run({ project: null, owner: `user:${USER}` }, 'succeeded'); // a person's job: no project, not counted
- assert.strictEqual(system.get(failedId).trace_id, TRACE, 'the submit request\'s trace id is kept with the job');
+ assert.strictEqual((await system.get(failedId)).trace_id, TRACE, 'the submit request\'s trace id is kept with the job');
const { job: held } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { hold: true }, project: PRJ });
const { job: held } = await system.submit({ owner: `app:${APP}`, type: 'test.upper', input: { hold: true }, project: PRJ });
- await until(() => system.get(held.id).state === 'running', 'held job running');
- system.cancel(held.id);
- await until(() => system.get(held.id).state === 'cancelled', 'held job cancelled');
+ await until(async () => (await system.get(held.id)).state === 'running', 'held job running');
+ await system.cancel(held.id);
+ await until(async () => (await system.get(held.id)).state === 'cancelled', 'held job cancelled');
- const rows = db.prepare('SELECT * FROM tool_job_usage ORDER BY capability, dimension, env').all();
+ const rows = await db.prepare('SELECT * FROM tool_job_usage ORDER BY capability, dimension, env').all();
const view = rows.map(r => [r.capability, r.dimension, r.env, r.quantity, r.errors]);
const view = rows.map(r => [r.capability, r.dimension, r.env, r.quantity, r.errors]);
assert.deepStrictEqual(view, [
assert.deepStrictEqual(view, [
['tools.job.create', 'test.boom', 'production', 1, 1],
['tools.job.create', 'test.boom', 'production', 1, 1],
@@ -111,18 +112,18 @@ function define(system) {
@@ -111,18 +112,18 @@ function define(system) {
assert.deepStrictEqual(JSON.parse(boom.error_codes), { 'tools.test.unreadable': 1 });
assert.deepStrictEqual(JSON.parse(boom.error_codes), { 'tools.test.unreadable': 1 });
const [sample] = JSON.parse(boom.samples);
const [sample] = JSON.parse(boom.samples);
assert.deepStrictEqual([sample.code, sample.status, sample.trace_id, sample.ref], ['tools.test.unreadable', 422, TRACE, failedId]);
assert.deepStrictEqual([sample.code, sample.status, sample.trace_id, sample.ref], ['tools.test.unreadable', 422, TRACE, failedId]);
- assert.strictEqual(keyOf({ ...system.get(failedId), state: 'cancelled' }), null);
+ assert.strictEqual(keyOf({ ...(await system.get(failedId)), state: 'cancelled' }), null);
// ── Nothing leaves before the hour closes ──
// ── Nothing leaves before the hour closes ──
- const before = db.prepare("SELECT COUNT(*) AS n FROM event_outbox WHERE envelope LIKE '%tools.usage.recorded%'").get().n;
- assert.strictEqual(before, 0);
- assert.strictEqual(system.flushUsage().queued, 0);
+ const before = (await db.prepare("SELECT COUNT(*) AS n FROM event_outbox WHERE envelope->>'event_type' = 'tools.usage.recorded'").get()).n;
+ assert.strictEqual(Number(before), 0);
+ assert.strictEqual((await system.flushUsage()).queued, 0);
// ── After the hour: one tools.usage.recorded per rollup, once ──
// ── After the hour: one tools.usage.recorded per rollup, once ──
- const later = rows[0].window_start + HOUR_MS + 2 * 60 * 1000;
- assert.strictEqual(system.flushUsage(later).queued, 4);
- assert.strictEqual(system.flushUsage(later).queued, 0, 'sent once');
- const sent = db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all().map(r => JSON.parse(r.envelope)).filter(e => e.event_type === 'tools.usage.recorded');
+ const later = Number(rows[0].window_start) + HOUR_MS + 2 * 60 * 1000;
+ assert.strictEqual((await system.flushUsage(later)).queued, 4);
+ assert.strictEqual((await system.flushUsage(later)).queued, 0, 'sent once');
+ const sent = (await db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all()).map(r => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope)).filter(e => e.event_type === 'tools.usage.recorded');
assert.strictEqual(sent.length, 4);
assert.strictEqual(sent.length, 4);
for (const env of sent) {
for (const env of sent) {
const v = contracts.validate('tools.usage.recorded@1', env.payload);
const v = contracts.validate('tools.usage.recorded@1', env.payload);
@@ -138,16 +139,17 @@ function define(system) {
@@ -138,16 +139,17 @@ function define(system) {
assert.deepStrictEqual([failedRollup.quantity, failedRollup.errors, failedRollup.samples[0].ref], [1, 1, failedId]);
assert.deepStrictEqual([failedRollup.quantity, failedRollup.errors, failedRollup.samples[0].ref], [1, 1, failedId]);
// ── A job that ends in an hour already sent: re-sent as revision 2 with the new totals ──
// ── A job that ends in an hour already sent: re-sent as revision 2 with the new totals ──
- db.transaction(() => system.usage.finished({ ...system.get(failedId), state: 'succeeded' }))();
- assert.strictEqual(system.flushUsage(later).queued, 1);
- const resent = db.prepare('SELECT envelope FROM event_outbox ORDER BY id DESC LIMIT 1').get();
- const p = JSON.parse(resent.envelope).payload;
+ await db.tx(async () => system.usage.finished({ ...(await system.get(failedId)), state: 'succeeded' }));
+ assert.strictEqual((await system.flushUsage(later)).queued, 1);
+ const resent = await db.prepare('SELECT envelope FROM event_outbox ORDER BY id DESC LIMIT 1').get();
+ const p = (typeof resent.envelope === 'string' ? JSON.parse(resent.envelope) : resent.envelope).payload;
assert.deepStrictEqual([p.dimension, p.quantity, p.errors, p.revision], ['test.boom', 2, 1, 2]);
assert.deepStrictEqual([p.dimension, p.quantity, p.errors, p.revision], ['test.boom', 2, 1, 2]);
- assert.deepStrictEqual(system.stats().usage, { pending: 0, invalid: 0, last_invalid: null });
+ assert.deepStrictEqual((await system.stats()).usage, { pending: 0, invalid: 0, last_invalid: null });
- system.stop(); outbox.stop(); db.close();
+ system.stop(); await outbox.stop();
console.log('job usage: all checks passed');
console.log('job usage: all checks passed');
} finally {
} finally {
+ await closeAllTestDbs();
fs.rmSync(root, { recursive: true, force: true });
fs.rmSync(root, { recursive: true, force: true });
}
}
})().catch((e) => { console.error(e); process.exit(1); });
})().catch((e) => { console.error(e); process.exit(1); });
@@ -14,9 +14,9 @@ const { dep } = require('./deps');
@@ -14,9 +14,9 @@ const { dep } = require('./deps');
const express = dep('express');
const express = dep('express');
const cookieParser = dep('cookie-parser');
const cookieParser = dep('cookie-parser');
-const Database = dep('better-sqlite3');
const contracts = dep('openvibe-contracts');
const contracts = dep('openvibe-contracts');
const jobs = require('../jobs');
const jobs = require('../jobs');
+const { testDb: dbFor, closeAllTestDbs: closeDbs, quiet } = require('./testdb');
const ISSUER = 'https://openvibe.network';
const ISSUER = 'https://openvibe.network';
const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
@@ -114,11 +114,11 @@ function fakeMedia() {
@@ -114,11 +114,11 @@ function fakeMedia() {
}
}
const fakeTokens = { authHeaders: async () => ({ Authorization: 'Bearer media-token' }), invalidate() {} };
const fakeTokens = { authHeaders: async () => ({ Authorization: 'Bearer media-token' }), invalidate() {} };
-// ── A satellite: express + the job routes over one SQLite file ──
-function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10 } = {}) {
+// ── A satellite: express + the job routes over the shared `tools` database ──
+async function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10 } = {}) {
fs.mkdirSync(dir, { recursive: true });
fs.mkdirSync(dir, { recursive: true });
- const db = new Database(path.join(dir, 'jobs.db'));
- const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir: dir, concurrency, media, maxActivePerOwner, progressThrottleMs: 0, log: { log() {}, warn() {}, error() {} } });
+ const db = await dbFor(dir);
+ const system = jobs.createJobSystem({ db, contracts, service: 'test', dataDir: dir, concurrency, media, maxActivePerOwner, progressThrottleMs: 0, log: quiet });
define(system);
define(system);
const app = express();
const app = express();
app.use(cookieParser());
app.use(cookieParser());
@@ -128,11 +128,11 @@ function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10
@@ -128,11 +128,11 @@ function satellite(dir, { concurrency = 2, media = null, maxActivePerOwner = 10
const upload = multer({ dest: path.join(dir, 'tmp') });
const upload = multer({ dest: path.join(dir, 'tmp') });
const resolveOwner = jobs.createOwnerResolver({ contracts, getPublicKey: () => publicKey, issuer: ISSUER, secureCookie: false });
const resolveOwner = jobs.createOwnerResolver({ contracts, getPublicKey: () => publicKey, issuer: ISSUER, secureCookie: false });
jobs.mountJobRoutes(app, { system, contracts, resolveOwner, receive: upload.any() });
jobs.mountJobRoutes(app, { system, contracts, resolveOwner, receive: upload.any() });
- system.start();
+ await system.start();
return new Promise(resolve => {
return new Promise(resolve => {
const server = app.listen(0, '127.0.0.1', () => resolve({
const server = app.listen(0, '127.0.0.1', () => resolve({
system, db, base: `http://127.0.0.1:${server.address().port}`,
system, db, base: `http://127.0.0.1:${server.address().port}`,
- close: () => new Promise(r => { system.stop(); server.closeAllConnections(); server.close(() => { db.close(); r(); }); }),
+ close: () => new Promise(r => { system.stop(); server.closeAllConnections(); server.close(() => r()); }),
}));
}));
});
});
}
}
@@ -246,7 +246,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -246,7 +246,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
assert.strictEqual((await guest1(`/api/v1/jobs/${guestJob}`)).status, 200, 'the session that made it can reattach');
assert.strictEqual((await guest1(`/api/v1/jobs/${guestJob}`)).status, 200, 'the session that made it can reattach');
assert.strictEqual((await guest2(`/api/v1/jobs/${guestJob}`)).status, 404, 'another browser cannot');
assert.strictEqual((await guest2(`/api/v1/jobs/${guestJob}`)).status, 404, 'another browser cannot');
assert.strictEqual((await alice(`/api/v1/jobs/${guestJob}`)).status, 404, 'nor can a signed-in stranger');
assert.strictEqual((await alice(`/api/v1/jobs/${guestJob}`)).status, 404, 'nor can a signed-in stranger');
- assert.ok(s.db.prepare('SELECT owner FROM tool_jobs').all().every(x => /^(user:usr_[0-9A-Z]{26}|session:[0-9a-f]{40})$/.test(x.owner)), 'owners are canonical subjects or hashed sessions, never raw cookies');
+ assert.ok((await s.db.prepare('SELECT owner FROM tool_jobs').all()).every(x => /^(user:usr_[0-9A-Z]{26}|session:[0-9a-f]{40})$/.test(x.owner)), 'owners are canonical subjects or hashed sessions, never raw cookies');
// ── SSE: events carry ids; Last-Event-ID resumes after them ──
// ── SSE: events carry ids; Last-Event-ID resumes after them ──
const all = await readSse(`${s.base}/api/v1/jobs/${first}/events`, { headers: { 'x-test-user': USER_A } });
const all = await readSse(`${s.base}/api/v1/jobs/${first}/events`, { headers: { 'x-test-user': USER_A } });
@@ -267,6 +267,19 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -267,6 +267,19 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
const live = await streaming;
const live = await streaming;
assert.strictEqual(live.events[live.events.length - 1].event, 'job.succeeded');
assert.strictEqual(live.events[live.events.length - 1].event, 'job.succeeded');
assert.strictEqual(live.events[live.events.length - 1].data.result.files.length, 1, 'the terminal event carries the result');
assert.strictEqual(live.events[live.events.length - 1].data.result.files.length, 1, 'the terminal event carries the result');
+ // A forced disconnect mid-stream: the reader drops after the first frame it sees, then reconnects
+ // with the last id it got. The two reads together are the exact log — no gap, no repeat.
+ r = await alice('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'cut', gate: 'cut' } }));
+ const cutId = r.body.id;
+ await until(async () => (await alice(`/api/v1/jobs/${cutId}`)).body.state === 'running', 'disconnect job running');
+ const head = await readSse(`${s.base}/api/v1/jobs/${cutId}/events`, { headers: { 'x-test-user': USER_A }, stop: (ev) => ev.length >= 1 });
+ assert.ok(head.events.length >= 1, 'the stream was cut mid-job');
+ gate('cut').open();
+ const tail = await readSse(`${s.base}/api/v1/jobs/${cutId}/events`, { headers: { 'x-test-user': USER_A, 'last-event-id': String(head.events[head.events.length - 1].id) } });
+ const joined = [...head.events, ...tail.events];
+ const whole = await readSse(`${s.base}/api/v1/jobs/${cutId}/events`, { headers: { 'x-test-user': USER_A } });
+ assert.deepStrictEqual(joined.map(e => e.event), whole.events.map(e => e.event), 'the exact sequence across the disconnect');
+ assert.deepStrictEqual(joined.map(e => e.id), whole.events.map(e => e.id), 'the job\'s log, in order, once each');
// ── Cancel ──────────────────────────────────────────────
// ── Cancel ──────────────────────────────────────────────
await s.close();
await s.close();
@@ -294,14 +307,14 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -294,14 +307,14 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
const dave = client(s.base, { user: USER_B });
const dave = client(s.base, { user: USER_B });
const ids = [];
const ids = [];
for (let i = 0; i < 4; i++) ids.push((await dave('/api/v1/jobs', json({ type: 'test.upper', input: { gate: 'pool' } }))).body.id);
for (let i = 0; i < 4; i++) ids.push((await dave('/api/v1/jobs', json({ type: 'test.upper', input: { gate: 'pool' } }))).body.id);
- await until(async () => s.system.stats().running === 2, 'two running');
+ await until(async () => (await s.system.stats()).running === 2, 'two running');
await sleep(50);
await sleep(50);
- assert.strictEqual(s.system.stats().running, 2, 'never more than the concurrency limit');
- assert.strictEqual(s.system.stats().queued, 2);
+ assert.strictEqual((await s.system.stats()).running, 2, 'never more than the concurrency limit');
+ assert.strictEqual((await s.system.stats()).queued, 2);
r = await dave('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'one too many' } }));
r = await dave('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'one too many' } }));
assert.strictEqual(r.status, 429); assert.strictEqual(r.body.code, 'tools.job.too_many_active');
assert.strictEqual(r.status, 429); assert.strictEqual(r.body.code, 'tools.job.too_many_active');
gate('pool').open();
gate('pool').open();
- await until(async () => s.system.stats().succeeded === 4, 'pool drained');
+ await until(async () => (await s.system.stats()).succeeded === 4, 'pool drained');
// ── Restart: an accepted job is not lost ────────────────
// ── Restart: an accepted job is not lost ────────────────
await s.close();
await s.close();
@@ -339,7 +352,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -339,7 +352,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
assert.ok(pruned >= 3);
assert.ok(pruned >= 3);
assert.strictEqual((await erin(`/api/v1/jobs/${accepted}`)).status, 404, 'expired jobs are gone');
assert.strictEqual((await erin(`/api/v1/jobs/${accepted}`)).status, 404, 'expired jobs are gone');
assert.ok(!fs.existsSync(path.join(dir, 'jobs', accepted)), 'with their files');
assert.ok(!fs.existsSync(path.join(dir, 'jobs', accepted)), 'with their files');
- assert.strictEqual(s.db.prepare('SELECT COUNT(*) AS n FROM tool_job_events WHERE job_id = ?').get(accepted).n, 0, 'and their events');
+ assert.strictEqual((await s.db.prepare('SELECT COUNT(*) AS n FROM tool_job_events WHERE job_id = ?').get(accepted)).n, 0, 'and their events');
await s.close();
await s.close();
// ── Results as Media objects ────────────────────────────
// ── Results as Media objects ────────────────────────────
@@ -488,7 +501,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -488,7 +501,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
assert.strictEqual(r.headers.get('set-cookie'), null, 'principals get no session cookie');
assert.strictEqual(r.headers.get('set-cookie'), null, 'principals get no session cookie');
const devJob = r.body.id;
const devJob = r.body.id;
await until(async () => (await dev(`/api/v1/jobs/${devJob}`)).body.state === 'succeeded', 'app job');
await until(async () => (await dev(`/api/v1/jobs/${devJob}`)).body.state === 'succeeded', 'app job');
- assert.strictEqual(s.system.get(devJob).owner, 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC');
+ assert.strictEqual((await s.system.get(devJob)).owner, 'app:app_01JCCCCCCCCCCCCCCCCCCCCCCC');
r = await dev(`/api/v1/jobs/${devJob}`, { method: 'DELETE' });
r = await dev(`/api/v1/jobs/${devJob}`, { method: 'DELETE' });
assert.strictEqual(r.status, 403, 'cancel needs tools.job.cancel');
assert.strictEqual(r.status, 403, 'cancel needs tools.job.cancel');
r = await client(s.base, { bearer: token({ cap: ['tools.job.read'] }) })('/api/v1/jobs', json({ type: 'test.upper', input: {} }));
r = await client(s.base, { bearer: token({ cap: ['tools.job.read'] }) })('/api/v1/jobs', json({ type: 'test.upper', input: {} }));
@@ -504,8 +517,8 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -504,8 +517,8 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
assert.strictEqual(r.status, 202, 'a sandbox app token can submit');
assert.strictEqual(r.status, 202, 'a sandbox app token can submit');
const sbxJob = r.body.id;
const sbxJob = r.body.id;
await until(async () => (await sbx(`/api/v1/jobs/${sbxJob}`)).body.state === 'succeeded', 'sandbox job');
await until(async () => (await sbx(`/api/v1/jobs/${sbxJob}`)).body.state === 'succeeded', 'sandbox job');
- assert.strictEqual(s.system.get(sbxJob).env, 'sandbox', 'the job remembers its environment');
- assert.ok(s.system.get(sbxJob).ttl_ms <= 30 * 60 * 1000, 'sandbox jobs are kept briefly');
+ assert.strictEqual((await s.system.get(sbxJob)).env, 'sandbox', 'the job remembers its environment');
+ assert.ok((await s.system.get(sbxJob)).ttl_ms <= 30 * 60 * 1000, 'sandbox jobs are kept briefly');
r = await client(s.base, { bearer: token({ sub: 'app:app_01JDDDDDDDDDDDDDDDDDDDDDDD', env: 'sandbox' }) })(`/api/v1/jobs/${sbxJob}`);
r = await client(s.base, { bearer: token({ sub: 'app:app_01JDDDDDDDDDDDDDDDDDDDDDDD', env: 'sandbox' }) })(`/api/v1/jobs/${sbxJob}`);
assert.strictEqual(r.status, 404, 'another sandbox app cannot read it');
assert.strictEqual(r.status, 404, 'another sandbox app cannot read it');
r = await client(s.base, { bearer: contracts.serviceAuth.signServiceToken({
r = await client(s.base, { bearer: contracts.serviceAuth.signServiceToken({
@@ -524,7 +537,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -524,7 +537,7 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
const done = async (c, id) => until(async () => { const g = await c(`/api/v1/jobs/${id}`); return g.body.state === 'succeeded' && g.body; }, 'project job');
const done = async (c, id) => until(async () => { const g = await c(`/api/v1/jobs/${id}`); return g.body.state === 'succeeded' && g.body; }, 'project job');
const prod = client(s.base, { bearer: token({}) });
const prod = client(s.base, { bearer: token({}) });
let j = await done(prod, (await prod('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'prod' } }))).body.id);
let j = await done(prod, (await prod('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'prod' } }))).body.id);
- assert.strictEqual(s.system.get(j.id).project_id, PRJ, 'the job remembers its project');
+ assert.strictEqual((await s.system.get(j.id)).project_id, PRJ, 'the job remembers its project');
assert.strictEqual(j.result.files[0].storage, 'media');
assert.strictEqual(j.result.files[0].storage, 'media');
assert.strictEqual(fm2.objects.get(j.result.files[0].media.media_id).meta.namespace, `tools.app.${PRJ}`, 'production: tools.app.<project>');
assert.strictEqual(fm2.objects.get(j.result.files[0].media.media_id).meta.namespace, `tools.app.${PRJ}`, 'production: tools.app.<project>');
assert.strictEqual(j.result.files[0].media.namespace, `tools.app.${PRJ}`);
assert.strictEqual(j.result.files[0].media.namespace, `tools.app.${PRJ}`);
@@ -535,13 +548,14 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
@@ -535,13 +548,14 @@ async function readSse(url, { headers = {}, stop = () => false, ms = 5000 } = {}
const person = client(s.base, { user: USER_B });
const person = client(s.base, { user: USER_B });
j = await done(person, (await person('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'person' } }))).body.id);
j = await done(person, (await person('/api/v1/jobs', json({ type: 'test.upper', input: { text: 'person' } }))).body.id);
assert.strictEqual(fm2.objects.get(j.result.files[0].media.media_id).meta.namespace, undefined, "a person's job stays in the tools root");
assert.strictEqual(fm2.objects.get(j.result.files[0].media.media_id).meta.namespace, undefined, "a person's job stays in the tools root");
- assert.strictEqual(s.system.get(j.id).project_id, null);
+ assert.strictEqual((await s.system.get(j.id)).project_id, null);
await s.close();
await s.close();
ms2.close();
ms2.close();
}
}
console.log('jobs (shared runtime): all checks passed');
console.log('jobs (shared runtime): all checks passed');
} finally {
} finally {
+ await closeDbs();
fs.rmSync(root, { recursive: true, force: true });
fs.rmSync(root, { recursive: true, force: true });
}
}
})().catch((err) => { console.error(err); process.exit(1); });
})().catch((err) => { console.error(err); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
+'use strict';
+// The SQLite era is gone (plan T8): every Tools app serves from PostgreSQL (openvibe-sdk/db) and the
+// job store lives in the one `tools` database. Nothing under apps/ may require better-sqlite3, and no
+// apps/*/package.json may list it. This fails loudly if either comes back.
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+
+const APPS = path.resolve(__dirname, '..', '..');
+const SKIP = new Set(['node_modules', 'data', 'dist', 'build', 'coverage', '.git']);
+const REQUIRE_RE = /\b(?:require\s*\(\s*['"]better-sqlite3['"]|dep\s*\(\s*['"]better-sqlite3['"]|from\s+['"]better-sqlite3['"])/;
+
+/** Every file under dir (source and manifests), skipping node_modules and build output. */
+function walk(dir, out = []) {
+ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+ if (SKIP.has(e.name)) continue;
+ const p = path.join(dir, e.name);
+ if (e.isDirectory()) walk(p, out);
+ else if (/\.(?:js|mjs|cjs|ts|json)$/.test(e.name)) out.push(p);
+ }
+ return out;
+}
+
+const files = walk(APPS).filter((f) => !/\.test\.js$/.test(f) || f !== __filename);
+const offenders = files
+ .filter((f) => path.basename(f) !== 'no-sqlite.test.js')
+ .filter((f) => REQUIRE_RE.test(fs.readFileSync(f, 'utf8')))
+ .map((f) => path.relative(APPS, f));
+
+assert.deepStrictEqual(offenders, [], `these files still require better-sqlite3:\n ${offenders.join('\n ')}`);
+
+// The dependency must be gone from every app manifest (and the root, if it ever had one).
+const manifests = [path.join(APPS, '..', 'package.json'), ...fs.readdirSync(APPS, { withFileTypes: true })
+ .filter((e) => e.isDirectory() && !SKIP.has(e.name))
+ .map((e) => path.join(APPS, e.name, 'package.json'))
+ .filter((p) => fs.existsSync(p))];
+for (const m of manifests) {
+ const pkg = JSON.parse(fs.readFileSync(m, 'utf8'));
+ assert.ok(!(pkg.dependencies && pkg.dependencies['better-sqlite3']), `${path.relative(APPS, m)} still depends on better-sqlite3`);
+}
+
+console.log(`no sqlite (nothing under apps/ requires better-sqlite3): all checks passed (${files.length} files scanned)`);
@@ -84,7 +84,7 @@ function assertShape(body, service) {
@@ -84,7 +84,7 @@ function assertShape(body, service) {
let r = await json(`${img.base}/api/ready`);
let r = await json(`${img.base}/api/ready`);
assertShape(r.body, 'tools-img');
assertShape(r.body, 'tools-img');
assert.strictEqual(r.status, 200, JSON.stringify(r.body));
assert.strictEqual(r.status, 200, JSON.stringify(r.body));
- for (const n of ['jobs_db', 'job_runtime', 'data_dir', 'uploads_dir', 'output_dir']) {
+ for (const n of ['tools_db', 'job_runtime', 'data_dir', 'uploads_dir', 'output_dir']) {
assert.strictEqual(r.body.checks[n].required, true, `${n} is required`);
assert.strictEqual(r.body.checks[n].required, true, `${n} is required`);
assert.strictEqual(r.body.checks[n].status, 'ok', `${n} ok`);
assert.strictEqual(r.body.checks[n].status, 'ok', `${n} ok`);
}
}
@@ -24,6 +24,13 @@ async function startApp(app, env = {}, port, { readyPath = '/api/health' } = {})
@@ -24,6 +24,13 @@ async function startApp(app, env = {}, port, { readyPath = '/api/health' } = {})
port = port || await freePort();
port = port || await freePort();
const dir = path.join(__dirname, '..', '..', app);
const dir = path.join(__dirname, '..', '..', app);
let out = '';
let out = '';
+ // Without a DATABASE_URL the app serves from an embedded PGlite database. Point it at a directory
+ // under the test's own DATA_DIR, so it survives the kill a restart test does (production uses a real
+ // PostgreSQL, which outlives the process the same way).
+ const url = env.DATABASE_URL !== undefined ? env.DATABASE_URL : process.env.DATABASE_URL;
+ const pglite = (!String(url || '').trim() && env.DATA_DIR && env.TOOLS_PGLITE_DIR === undefined)
+ ? { TOOLS_PGLITE_DIR: path.join(env.DATA_DIR, 'pglite') }
+ : {};
const child = spawn(process.execPath, ['server/index.js'], {
const child = spawn(process.execPath, ['server/index.js'], {
cwd: dir,
cwd: dir,
env: {
env: {
@@ -35,6 +42,7 @@ async function startApp(app, env = {}, port, { readyPath = '/api/health' } = {})
@@ -35,6 +42,7 @@ async function startApp(app, env = {}, port, { readyPath = '/api/health' } = {})
OV_NETWORK_URL: 'http://127.0.0.1:9',
OV_NETWORK_URL: 'http://127.0.0.1:9',
OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',
OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',
TOOLS_JOB_RESULTS: 'local',
TOOLS_JOB_RESULTS: 'local',
+ ...pglite,
...env,
...env,
},
},
stdio: ['ignore', 'pipe', 'pipe'],
stdio: ['ignore', 'pipe', 'pipe'],
new file mode 100644
new file mode 100644
@@ -0,0 +1,29 @@
@@ -0,0 +1,29 @@
+'use strict';
+// The one `tools` database for a test run (plan T8): one PGlite database per test directory, migrated
+// with the repo's migrations. It is kept across a satellite restart (the data must survive it), so a
+// test that restarts a satellite calls testDb() with the same directory again; closeAllTestDbs() at the
+// end of the run. Nothing here needs better-sqlite3: PGlite is real PostgreSQL in-process.
+const fs = require('fs');
+const path = require('path');
+const { dep } = require('./deps');
+
+const MIGRATIONS = path.resolve(__dirname, '..', '..', '..', 'migrations');
+const quiet = { log() {}, warn() {}, error() {} };
+const dbs = new Map();
+
+async function testDb(dir) {
+ if (!dbs.has(dir)) {
+ fs.mkdirSync(path.join(dir, 'pglite'), { recursive: true });
+ const db = dep('openvibe-sdk/db').createDb({ pglite: path.join(dir, 'pglite'), service: 'tools-test', log: quiet });
+ await db.migrate({ dir: MIGRATIONS, log: quiet });
+ dbs.set(dir, db);
+ }
+ return dbs.get(dir);
+}
+
+async function closeAllTestDbs() {
+ for (const db of dbs.values()) await db.close().catch(() => {});
+ dbs.clear();
+}
+
+module.exports = { testDb, closeAllTestDbs, MIGRATIONS, quiet };
@@ -327,14 +327,14 @@ function createRunApi(o) {
@@ -327,14 +327,14 @@ function createRunApi(o) {
const notFound = () => new Refusal(404, 'tools.run.file_not_found', `${ref.media_id || `${ref.job_id} file ${ref.index}`} is not a file you can read here: a result of one of your own Tools jobs that has not expired.`);
const notFound = () => new Refusal(404, 'tools.run.file_not_found', `${ref.media_id || `${ref.job_id} file ${ref.index}`} is not a file you can read here: a result of one of your own Tools jobs that has not expired.`);
let row = null, index = ref.index;
let row = null, index = ref.index;
if (ref.job_id) {
if (ref.job_id) {
- row = system.get(ref.job_id);
+ row = await system.get(ref.job_id);
if (!row && o.jobs.fetchSibling) {
if (!row && o.jobs.fetchSibling) {
const got = await o.jobs.fetchSibling(ref, req, dir);
const got = await o.jobs.fetchSibling(ref, req, dir);
if (got) return got;
if (got) return got;
throw notFound();
throw notFound();
}
}
} else {
} else {
- const hit = system.findResultMedia(owner, ref.media_id);
+ const hit = await system.findResultMedia(owner, ref.media_id);
if (hit) { row = hit.row; index = hit.index; }
if (hit) { row = hit.row; index = hit.index; }
}
}
if (!row || row.owner !== owner || row.state !== 'succeeded') throw notFound();
if (!row || row.owner !== owner || row.state !== 'succeeded') throw notFound();
@@ -352,23 +352,22 @@ function createRunApi(o) {
@@ -352,23 +352,22 @@ function createRunApi(o) {
throw notFound();
throw notFound();
}
}
- function waitFor(system, id, ms, res) {
- return new Promise((resolve) => {
- const row = system.get(id);
- if (!row || TERMINAL.has(row.state) || ms <= 0) return resolve();
+ async function waitFor(system, id, ms, res) {
+ const row = await system.get(id);
+ if (!row || TERMINAL.has(row.state) || ms <= 0) return;
+ await new Promise((resolve) => {
let done = false;
let done = false;
const finish = () => { if (done) return; done = true; clearTimeout(timer); off(); res.removeListener('close', finish); resolve(); };
const finish = () => { if (done) return; done = true; clearTimeout(timer); off(); res.removeListener('close', finish); resolve(); };
const off = system.subscribe(id, (e) => { if (TERMINAL.has(e.data.state)) finish(); });
const off = system.subscribe(id, (e) => { if (TERMINAL.has(e.data.state)) finish(); });
const timer = setTimeout(finish, ms);
const timer = setTimeout(finish, ms);
res.on('close', finish);
res.on('close', finish);
// It may have finished between the first read and the subscription.
// It may have finished between the first read and the subscription.
- const again = system.get(id);
- if (!again || TERMINAL.has(again.state)) finish();
+ system.get(id).then((again) => { if (!again || TERMINAL.has(again.state)) finish(); }).catch(() => {});
});
});
}
}
- function runView(system, d, row) {
- const job = system.view(row);
+ async function runView(system, d, row) {
+ const job = await system.view(row);
const took = row.finished_at ? Math.max(0, row.finished_at - row.created_at) : 0;
const took = row.finished_at ? Math.max(0, row.finished_at - row.created_at) : 0;
if (row.state === 'succeeded') return { state: 'succeeded', tool: d.id, result: { data: (job.result && job.result.data) || {}, files: (job.result && job.result.files) || [] }, took_ms: took, job };
if (row.state === 'succeeded') return { state: 'succeeded', tool: d.id, result: { data: (job.result && job.result.data) || {}, files: (job.result && job.result.files) || [] }, took_ms: took, job };
if (row.state === 'failed' || row.state === 'cancelled') return { state: row.state, tool: d.id, error: job.error || contracts.http.problem(500, 'tools.job.failed', { detail: 'The job failed' }), took_ms: took, job };
if (row.state === 'failed' || row.state === 'cancelled') return { state: row.state, tool: d.id, error: job.error || contracts.http.problem(500, 'tools.job.failed', { detail: 'The job failed' }), took_ms: took, job };
@@ -400,9 +399,9 @@ function createRunApi(o) {
@@ -400,9 +399,9 @@ function createRunApi(o) {
// The bytes against files.accept (hard; a misleading name or type is corrected).
// The bytes against files.accept (hard; a misleading name or type is corrected).
if (spec && !guard.checkFiles(req, res, files, spec.accept, d.id)) return undefined;
if (spec && !guard.checkFiles(req, res, files, spec.accept, d.id)) return undefined;
// Sessions from one address share a bound on unfinished jobs; the store's queue and disk bounds.
// Sessions from one address share a bound on unfinished jobs; the store's queue and disk bounds.
- const full = system.addressFull ? system.addressFull(owner, who.ipKey) : null;
+ const full = system.addressFull ? await system.addressFull(owner, who.ipKey) : null;
if (full && guard.refuse(req, res, { status: 429, code: 'tools.job.too_many_active', reason: 'jobs.address', tool: d.id, retryAfter: 30, detail: `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish.`, extra: { scope: 'address' } })) return undefined;
if (full && guard.refuse(req, res, { status: 429, code: 'tools.job.too_many_active', reason: 'jobs.address', tool: d.id, retryAfter: 30, detail: `At most ${full.limit} unfinished jobs from one address at a time; wait for one to finish.`, extra: { scope: 'address' } })) return undefined;
- const busy = system.busy ? system.busy() : null;
+ const busy = system.busy ? await system.busy() : null;
if (busy && !guard.jobsBusy(req, res, busy, d.id)) return undefined;
if (busy && !guard.jobsBusy(req, res, busy, d.id)) return undefined;
let row, replayed;
let row, replayed;
try {
try {
@@ -421,8 +420,8 @@ function createRunApi(o) {
@@ -421,8 +420,8 @@ function createRunApi(o) {
}
}
files.length = 0; // the job owns them now
files.length = 0; // the job owns them now
if (wait > 0) await waitFor(system, row.id, wait, res);
if (wait > 0) await waitFor(system, row.id, wait, res);
- const fresh = system.get(row.id) || row;
- const body = runView(system, d, fresh);
+ const fresh = (await system.get(row.id)) || row;
+ const body = await runView(system, d, fresh);
const headers = replayed ? { 'Idempotent-Replayed': 'true' } : {};
const headers = replayed ? { 'Idempotent-Replayed': 'true' } : {};
if (body.location) headers.Location = body.location;
if (body.location) headers.Location = body.location;
return send(res, replayed || !body.location ? 200 : 202, body, headers);
return send(res, replayed || !body.location ? 200 : 202, body, headers);
@@ -12,7 +12,6 @@
@@ -12,7 +12,6 @@
"openvibeToolsShared": "^1.2.0",
"openvibeToolsShared": "^1.2.0",
"dependencies": {
"dependencies": {
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
- "better-sqlite3": "^11.0.0",
"cookie-parser": "^1.4.7",
"cookie-parser": "^1.4.7",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');
@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');
const sdk = require('openvibe-sdk');
const sdk = require('openvibe-sdk');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
-const Database = require('better-sqlite3');
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
const { requireInternalAccess } = require('../../_shared/internal-token');
const { requireInternalAccess } = require('../../_shared/internal-token');
@@ -72,9 +71,8 @@ const obs = observe({
@@ -72,9 +71,8 @@ const obs = observe({
service: 'tools-audio', release: release.release,
service: 'tools-audio', release: release.release,
checks: [
checks: [
ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),
ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),
- ready.sqlite('jobs_db', () => jobs.db, { sql: 'SELECT COUNT(*) AS n FROM tool_jobs', description: 'job store (jobs.db)' }),
ready.jobRuntime('job_runtime', () => jobs),
ready.jobRuntime('job_runtime', () => jobs),
- ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'jobs.db and job inputs/results' }),
+ ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'job inputs/results' }),
ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),
ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),
ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),
ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),
ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),
ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),
@@ -160,9 +158,9 @@ app.use((req, _res, next) => {
@@ -160,9 +158,9 @@ app.use((req, _res, next) => {
// ── API Routes ───────────────────────────────────────────────
// ── API Routes ───────────────────────────────────────────────
// Health check
// Health check
-app.get('/api/health', (_req, res) => {
+app.get('/api/health', async (_req, res) => {
const stats = retention.getStats();
const stats = retention.getStats();
- res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats, jobs: jobs.stats() });
+ res.json({ status: 'ok', service: 'openvibe-audio', version: '1.0.0', files: stats, jobs: await jobs.stats() });
});
});
// Domain context (frontend calls this on load to get branding). It also starts this browser's session
// Domain context (frontend calls this on load to get branding). It also starts this browser's session
@@ -204,11 +202,11 @@ app.post('/api/probe', guard.originCheck(), burstLimiter, processLimiter, guard.
@@ -204,11 +202,11 @@ app.post('/api/probe', guard.originCheck(), burstLimiter, processLimiter, guard.
});
});
// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
-// The audio operations, asynchronous and durable: accepted into data/jobs.db, followed over SSE
+// The audio operations, asynchronous and durable: accepted into the shared `tools` database, followed over SSE
// (ffmpeg's own progress), cancellable (ffmpeg is killed), reattachable by id after a reload or a
// (ffmpeg's own progress), cancellable (ffmpeg is killed), reattachable by id after a reload or a
// restart (apps/_shared/jobs).
// restart (apps/_shared/jobs).
const jobs = jobsRuntime.setupJobs({
const jobs = jobsRuntime.setupJobs({
- app, service: 'audio', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts, sdk,
+ app, service: 'audio', dataDir: path.resolve(__dirname, '..', config.dataDir), db: toolsDb.db, contracts, sdk,
getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,
getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,
define: defineJobs,
define: defineJobs,
receive: uploadAny,
receive: uploadAny,
@@ -341,7 +339,7 @@ require('../../_shared/graceful').gracefulStop({
@@ -341,7 +339,7 @@ require('../../_shared/graceful').gracefulStop({
close: [
close: [
() => require('../../_shared/usage').stopRecorder(800),
() => require('../../_shared/usage').stopRecorder(800),
() => analytics.destroy(),
() => analytics.destroy(),
- () => jobs.close(), // running jobs stay 'running' in data/jobs.db; the next boot re-queues them
+ () => jobs.close(), // running jobs stay 'running' in the tools database; the next boot re-queues them
() => guard.close(),
() => guard.close(),
() => toolsDb.db.close(),
() => toolsDb.db.close(),
() => toolsValkey && toolsValkey.close(),
() => toolsValkey && toolsValkey.close(),
@@ -12,7 +12,6 @@
@@ -12,7 +12,6 @@
"openvibeToolsShared": "^1.2.0",
"openvibeToolsShared": "^1.2.0",
"dependencies": {
"dependencies": {
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
- "better-sqlite3": "^11.0.0",
"cookie-parser": "^1.4.7",
"cookie-parser": "^1.4.7",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');
@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');
const sdk = require('openvibe-sdk');
const sdk = require('openvibe-sdk');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
-const Database = require('better-sqlite3');
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
const { requireInternalAccess } = require('../../_shared/internal-token');
const { requireInternalAccess } = require('../../_shared/internal-token');
@@ -89,9 +88,8 @@ const obs = observe({
@@ -89,9 +88,8 @@ const obs = observe({
service: 'tools-docs', release: release.release,
service: 'tools-docs', release: release.release,
checks: [
checks: [
ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),
ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),
- ready.sqlite('jobs_db', () => jobs.db, { sql: 'SELECT COUNT(*) AS n FROM tool_jobs', description: 'job store (jobs.db)' }),
ready.jobRuntime('job_runtime', () => jobs),
ready.jobRuntime('job_runtime', () => jobs),
- ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'jobs.db and job inputs/results' }),
+ ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'job inputs/results' }),
ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),
ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),
ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),
ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),
ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),
ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),
@@ -189,9 +187,9 @@ function sendToolError(req, res, err, label) {
@@ -189,9 +187,9 @@ function sendToolError(req, res, err, label) {
}
}
// Health check
// Health check
-app.get('/api/health', (_req, res) => {
+app.get('/api/health', async (_req, res) => {
const stats = retention.getStats();
const stats = retention.getStats();
- res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats, jobs: jobs.stats(), workers: pool.stats() });
+ res.json({ status: 'ok', service: 'openvibe-docs', version: '1.0.0', files: stats, jobs: await jobs.stats(), workers: pool.stats() });
});
});
// Domain context (frontend calls this on load to get branding). It also starts this browser's session
// Domain context (frontend calls this on load to get branding). It also starts this browser's session
@@ -232,10 +230,10 @@ app.post('/api/info', guard.originCheck(), burstLimiter, processLimiter, guard.t
@@ -232,10 +230,10 @@ app.post('/api/info', guard.originCheck(), burstLimiter, processLimiter, guard.t
});
});
// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
-// The document operations, asynchronous and durable: accepted into data/jobs.db, followed over SSE,
+// The document operations, asynchronous and durable: accepted into the shared `tools` database, followed over SSE,
// reattachable by id after a reload or a restart.
// reattachable by id after a reload or a restart.
const jobs = jobsRuntime.setupJobs({
const jobs = jobsRuntime.setupJobs({
- app, service: 'docs', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts, sdk,
+ app, service: 'docs', dataDir: path.resolve(__dirname, '..', config.dataDir), db: toolsDb.db, contracts, sdk,
getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,
getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,
define: (system) => defineJobs(system, pool),
define: (system) => defineJobs(system, pool),
receive: uploadAny,
receive: uploadAny,
@@ -341,7 +339,7 @@ require('../../_shared/graceful').gracefulStop({
@@ -341,7 +339,7 @@ require('../../_shared/graceful').gracefulStop({
close: [
close: [
() => require('../../_shared/usage').stopRecorder(800),
() => require('../../_shared/usage').stopRecorder(800),
() => analytics.destroy(),
() => analytics.destroy(),
- () => jobs.close(), // running jobs stay 'running' in data/jobs.db; the next boot re-queues them
+ () => jobs.close(), // running jobs stay 'running' in the tools database; the next boot re-queues them
() => pool.close(),
() => pool.close(),
() => guard.close(),
() => guard.close(),
() => toolsDb.db.close(),
() => toolsDb.db.close(),
@@ -15,7 +15,6 @@
@@ -15,7 +15,6 @@
"dotenv": "^16.3.1",
"dotenv": "^16.3.1",
"express": "^4.18.2",
"express": "^4.18.2",
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
- "better-sqlite3": "^11.0.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
"express-rate-limit": "^7.1.5",
"express-rate-limit": "^7.1.5",
@@ -12,7 +12,6 @@
@@ -12,7 +12,6 @@
"openvibeToolsShared": "^1.2.0",
"openvibeToolsShared": "^1.2.0",
"dependencies": {
"dependencies": {
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
- "better-sqlite3": "^11.10.0",
"cookie-parser": "^1.4.7",
"cookie-parser": "^1.4.7",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -12,7 +12,6 @@
@@ -12,7 +12,6 @@
"openvibeToolsShared": "^1.2.0",
"openvibeToolsShared": "^1.2.0",
"dependencies": {
"dependencies": {
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
- "better-sqlite3": "^11.0.0",
"cookie-parser": "^1.4.7",
"cookie-parser": "^1.4.7",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');
@@ -34,7 +34,6 @@ const contracts = require('openvibe-contracts');
const sdk = require('openvibe-sdk');
const sdk = require('openvibe-sdk');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
-const Database = require('better-sqlite3');
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
const { requireInternalAccess } = require('../../_shared/internal-token');
const { requireInternalAccess } = require('../../_shared/internal-token');
@@ -78,9 +77,8 @@ const obs = observe({
@@ -78,9 +77,8 @@ const obs = observe({
service: 'tools-img', release: release.release,
service: 'tools-img', release: release.release,
checks: [
checks: [
ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),
ready.postgres('tools_db', () => toolsDb.db, { description: 'the one tools database (guard_abuse, jobs, analytics, revocations)' }),
- ready.sqlite('jobs_db', () => jobs.db, { sql: 'SELECT COUNT(*) AS n FROM tool_jobs', description: 'job store (jobs.db)' }),
ready.jobRuntime('job_runtime', () => jobs),
ready.jobRuntime('job_runtime', () => jobs),
- ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'jobs.db and job inputs/results' }),
+ ready.writableDir('data_dir', path.resolve(__dirname, '..', config.dataDir), { description: 'job inputs/results' }),
ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),
ready.writableDir('uploads_dir', path.resolve(config.uploadsDir), { description: 'synchronous uploads' }),
ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),
ready.writableDir('output_dir', path.resolve(config.outputDir), { description: 'synchronous results' }),
ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),
ready.networkKey('network_key', guard.keys.get, { description: 'verifies signed-in users and service tokens on the job API; anonymous use works without it' }),
@@ -166,9 +164,9 @@ app.use((req, _res, next) => {
@@ -166,9 +164,9 @@ app.use((req, _res, next) => {
// ── API Routes ───────────────────────────────────────────────
// ── API Routes ───────────────────────────────────────────────
// Health check
// Health check
-app.get('/api/health', (_req, res) => {
+app.get('/api/health', async (_req, res) => {
const stats = retention.getStats();
const stats = retention.getStats();
- res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats, jobs: jobs.stats(), workers: pool.stats() });
+ res.json({ status: 'ok', service: 'openvibe-img', version: '1.0.0', files: stats, jobs: await jobs.stats(), workers: pool.stats() });
});
});
// Domain context (frontend calls this on load to get branding). It also starts this browser's session
// Domain context (frontend calls this on load to get branding). It also starts this browser's session
@@ -199,10 +197,10 @@ app.get('/api/tools', (_req, res) => {
@@ -199,10 +197,10 @@ app.get('/api/tools', (_req, res) => {
});
});
// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
// ── Jobs (/api/v1/jobs) ──────────────────────────────────────
-// The image operations, asynchronous and durable: accepted into data/jobs.db, followed over SSE,
-// reattachable by id after a reload or a restart (apps/_shared/jobs).
+// The image operations, asynchronous and durable: accepted into the shared `tools` database, followed
+// over SSE, reattachable by id after a reload or a restart (apps/_shared/jobs).
const jobs = jobsRuntime.setupJobs({
const jobs = jobsRuntime.setupJobs({
- app, service: 'img', dataDir: path.resolve(__dirname, '..', config.dataDir), Database, contracts, sdk,
+ app, service: 'img', dataDir: path.resolve(__dirname, '..', config.dataDir), db: toolsDb.db, contracts, sdk,
getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,
getPublicKey: guard.keys.get, issuer: config.networkUrl, guard,
define: (system) => defineJobs(system, pool),
define: (system) => defineJobs(system, pool),
receive: uploadSingle,
receive: uploadSingle,
@@ -306,7 +304,7 @@ require('../../_shared/graceful').gracefulStop({
@@ -306,7 +304,7 @@ require('../../_shared/graceful').gracefulStop({
close: [
close: [
() => require('../../_shared/usage').stopRecorder(800),
() => require('../../_shared/usage').stopRecorder(800),
() => analytics.destroy(),
() => analytics.destroy(),
- () => jobs.close(), // running jobs stay 'running' in data/jobs.db; the next boot re-queues them
+ () => jobs.close(), // running jobs stay 'running' in the tools database; the next boot re-queues them
() => pool.close(),
() => pool.close(),
() => guard.close(),
() => guard.close(),
() => toolsDb.db.close(),
() => toolsDb.db.close(),
@@ -11,7 +11,6 @@
@@ -11,7 +11,6 @@
"openvibeToolsShared": "^1.2.0",
"openvibeToolsShared": "^1.2.0",
"dependencies": {
"dependencies": {
"axios": "^1.7.0",
"axios": "^1.7.0",
- "better-sqlite3": "^11.0.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
@@ -14,7 +14,6 @@
@@ -14,7 +14,6 @@
"helmet": "^7.1.0",
"helmet": "^7.1.0",
"cors": "^2.8.5",
"cors": "^2.8.5",
"cookie-parser": "^1.4.6",
"cookie-parser": "^1.4.6",
- "better-sqlite3": "^11.0.0",
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -17,7 +17,6 @@
@@ -17,7 +17,6 @@
"express": "^4.21.2",
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
- "better-sqlite3": "^11.0.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",