openvibers/openvibe.tools Commits
Tools on PostgreSQL, the groundwork (plan T8): migrations 0001_tools (one tools database for the eight apps: the job store and guard_abuse with app in every key that could collide; guard counters and salt go to Valkey) and 0002_analytics; scripts/migrate-to-postgres.js imports every app's SQLite files into it with per-table counts and checksums (rehearsed on PGlite: every table reconciled); scripts/guard-abuse-report.js (Phase 0: only reason=quota was ever recorded in report mode); the img, docs and audio pages lose their dead /api/process fallback (they submit through OVJobs, as production already did). npm test 57/57 (verified).
Changes
@@@ -624,3 +624,19 @@ when the gateway's `/api/ready` or `/release.json` does not come up with every u- Nginx: satellites have specific `server_name` blocks; the gateway'swildcard `*.openvibe.tools` block catches everything else. TLS via`/etc/letsencrypt/live/openvibe.tools/`.++++## Migration to PostgreSQL + Valkey (plan T8)++++All eight apps move from `better-sqlite3` to **one `tools` PostgreSQL database** and to **Valkey** for the++guard's throttle state. The schema is in `migrations/0001_tools.sql` (job store, `guard_abuse`) and++`migrations/0002_analytics.sql` (request analytics, the DDL of `openvibe-shared/analytics/pg.js`); the event++outbox and the revocation store create their own tables at boot. `scripts/migrate-to-postgres.js` imports++every `apps/*/data/{guard,jobs,analytics}.db` into that one database (each file's rows tagged with its app):++run it with `--pglite` for an in-memory rehearsal, `--dry-run --url <scratch>` for a scratch database, or++with `DATABASE_DIRECT_URL` set for the cutover; it prints a per-table count and checksum report.++++After the move the apps read `DATABASE_URL` / `DATABASE_DIRECT_URL` (PostgreSQL, via PgBouncer) and++`VALKEY_URL` (Valkey); `guard_salt`, `guard_day` and the minute/burst buckets live in Valkey (a throttle,++never authoritative), while `guard_abuse`, the job store and the analytics tables stay in PostgreSQL.++`TOOLS_GUARD` (default `report`) and its meaning are unchanged. Until the cutover lands the apps still run++on `better-sqlite3` exactly as described above.@@@ -432,31 +432,12 @@processing.style.display = '';// Jobs: the work is accepted, followed live (ffmpeg's progress) and survives a reload.-- if (window.OVJobs) {-- const input = {};-- for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;-- try {-- const job = await OVJobs.submit({ type: 'audio.process', input, files, fileField: isMerge() ? 'files' : 'file' });-- OVJobs.remember(job.id);-- followJob(job);-- } catch (err) {-- showToast(err.message, 'error');-- resetUI();-- }-- return;-- }---- // Without the jobs helper: the synchronous endpoint, as before.++ const input = {};++ for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;try {-- const res = await fetch(isMerge() ? '/api/process/multi' : '/api/process', { method: 'POST', body: formData });-- const data = await res.json();---- if (!res.ok) {-- throw new Error(data.error || `Server error (${res.status})`);-- }---- resultId = data.download.id;-- showResult(data);++ const job = await OVJobs.submit({ type: 'audio.process', input, files, fileField: isMerge() ? 'files' : 'file' });++ OVJobs.remember(job.id);++ followJob(job);} catch (err) {showToast(err.message, 'error');resetUI();@@@ -460,46 +460,14 @@addToolOptions(formData);// Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).-- if (window.OVJobs) {-- const input = {};-- for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;-- try {-- const job = await OVJobs.submit({ type: 'docs.process', input, files: isMulti ? selectedFiles : [selectedFiles[0]], fileField: isMulti ? 'files' : 'file' });-- OVJobs.remember(job.id);-- followJob(job);-- } catch (err) {-- failProcessing(err.message);-- }-- return;-- }---- // Without the jobs helper: the synchronous endpoints, as before.++ const input = {};++ for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;try {-- const token = getCookie('ov_token') || localStorage.getItem('ov_token');-- const headers = {};-- if (token) headers['Authorization'] = `Bearer ${token}`;---- const endpoint = isMulti ? '/api/process/multi' : '/api/process';-- const res = await fetch(endpoint, { method: 'POST', body: formData, headers });---- if (!res.ok) {-- const err = await res.json().catch(() => ({ error: 'Processing failed' }));-- throw new Error(err.error || `Server error: ${res.status}`);-- }---- const data = await res.json();---- // Handle view-only results (metadata)-- if (data.viewOnly) {-- showMetadataResult(data);-- } else {-- showResult(data);-- }++ const job = await OVJobs.submit({ type: 'docs.process', input, files: isMulti ? selectedFiles : [selectedFiles[0]], fileField: isMulti ? 'files' : 'file' });++ OVJobs.remember(job.id);++ followJob(job);} catch (err) {-- processingEl.style.display = 'none';-- optionsPanel.style.display = '';-- processBtn.disabled = false;-- alert(`Error: ${err.message}`);++ failProcessing(err.message);}}@@@ -374,39 +374,14 @@}// Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).-- if (window.OVJobs) {-- const input = {};-- for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;-- try {-- const job = await OVJobs.submit({ type: 'img.process', input, files: [selectedFile] });-- OVJobs.remember(job.id);-- followJob(job);-- } catch (err) {-- failProcessing(err.message);-- }-- return;-- }---- // Without the jobs helper: the synchronous endpoint, as before.++ const input = {};++ for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;try {-- const token = getCookie('ov_token') || localStorage.getItem('ov_token');-- const headers = {};-- if (token) headers['Authorization'] = `Bearer ${token}`;---- const res = await fetch('/api/process', { method: 'POST', body: formData, headers });---- if (!res.ok) {-- const err = await res.json().catch(() => ({ error: 'Processing failed' }));-- throw new Error(err.error || `Server error: ${res.status}`);-- }---- const data = await res.json();-- showResult(data);++ const job = await OVJobs.submit({ type: 'img.process', input, files: [selectedFile] });++ OVJobs.remember(job.id);++ followJob(job);} catch (err) {-- processingEl.style.display = 'none';-- optionsPanel.style.display = '';-- processBtn.disabled = false;-- alert(`Error: ${err.message}`);++ failProcessing(err.message);}}nnew file mode 100644@@@ -0,0 +1,55 @@++-- phase: expand++-- One `tools` database for all eight apps (plan T8, decision 3). Every key that could collide across++-- apps carries `app`. The runtime role creates no tables: this file is applied by migrations.++--++-- Not here on purpose (plan T8, decision 4):++-- guard_salt, guard_day, the minute/burst token buckets -> Valkey (throttle state, worthless after a day)++-- the event outbox -> openvibe-sdk createPgOutbox creates its own table at boot++-- token_revocations -> openvibe-sdk/auth createPgRevocationStore creates its own table at boot++--++-- `app` (the owning app, one row per app in the one database) has DEFAULT '' so scripts/migrate-to-postgres.js++-- can import a file that has no such column; the script sets the real value in the same pass and the runtime++-- always writes it explicitly. A default of '' is never a real owner.++-- analytics tables (analytics_events, analytics_hourly, analytics_daily, analytics_visitor_days,++-- analytics_day_salts and their indexes) are the analyticsSchema() DDL from++-- openvibe-shared/analytics/pg.js, copied verbatim into 0002_analytics.sql.++++CREATE TABLE IF NOT EXISTS tool_jobs (++ id text PRIMARY KEY, app text NOT NULL DEFAULT '', type text NOT NULL, type_version integer NOT NULL DEFAULT 1,++ owner text NOT NULL, state text NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled')),++ progress double precision, progress_message text, input_json text NOT NULL DEFAULT '{}',++ files_json text NOT NULL DEFAULT '[]', result_json text, error_json text,++ retryable integer NOT NULL DEFAULT 0, idempotency_key text, request_hash text,++ attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL DEFAULT 1,++ cancel_requested integer NOT NULL DEFAULT 0, ttl_ms integer NOT NULL DEFAULT 3600000,++ created_at bigint NOT NULL, updated_at bigint NOT NULL, started_at bigint, finished_at bigint, expires_at bigint,++ env text NOT NULL DEFAULT 'production' CHECK (env IN ('production','sandbox')),++ retry_of text, retried_by text, ip_key text, tool text, project_id text, trace_id text);++CREATE UNIQUE INDEX IF NOT EXISTS tool_jobs_idem ON tool_jobs(app, owner, idempotency_key) WHERE idempotency_key IS NOT NULL;++CREATE INDEX IF NOT EXISTS tool_jobs_state ON tool_jobs(app, state, id);++CREATE INDEX IF NOT EXISTS tool_jobs_expiry ON tool_jobs(app, expires_at) WHERE expires_at IS NOT NULL;++CREATE INDEX IF NOT EXISTS tool_jobs_owner ON tool_jobs(app, owner, state);++CREATE INDEX IF NOT EXISTS tool_jobs_ip ON tool_jobs(app, ip_key, state) WHERE ip_key IS NOT NULL;++++CREATE TABLE IF NOT EXISTS tool_job_events (++ seq bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, job_id text NOT NULL,++ event text NOT NULL, data text NOT NULL, at bigint NOT NULL);++CREATE INDEX IF NOT EXISTS tool_job_events_job ON tool_job_events(job_id, seq);++++CREATE TABLE IF NOT EXISTS tool_job_references (++ job_id text NOT NULL, ref text NOT NULL, created_at bigint NOT NULL, PRIMARY KEY (job_id, ref));++++CREATE TABLE IF NOT EXISTS tool_job_usage (++ app text NOT NULL DEFAULT '', project_id text NOT NULL, env text NOT NULL, capability text NOT NULL,++ dimension text NOT NULL, window_start bigint NOT NULL, quantity integer NOT NULL DEFAULT 0,++ errors integer NOT NULL DEFAULT 0, error_codes text NOT NULL DEFAULT '{}', samples text NOT NULL DEFAULT '[]',++ revision integer NOT NULL DEFAULT 1, event_id text, emitted_at bigint,++ PRIMARY KEY (app, project_id, env, capability, dimension, window_start));++CREATE INDEX IF NOT EXISTS tool_job_usage_open ON tool_job_usage(app, emitted_at, window_start);++++CREATE TABLE IF NOT EXISTS guard_abuse (++ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, app text NOT NULL DEFAULT '', at bigint NOT NULL,++ minute bigint NOT NULL, ip_hash text NOT NULL DEFAULT '', principal text NOT NULL DEFAULT '',++ tool text NOT NULL DEFAULT '', reason text NOT NULL, enforced integer NOT NULL, count integer NOT NULL DEFAULT 1);++CREATE UNIQUE INDEX IF NOT EXISTS guard_abuse_once ON guard_abuse(app, minute, ip_hash, principal, tool, reason, enforced);++CREATE INDEX IF NOT EXISTS guard_abuse_at ON guard_abuse(app, at);nnew file mode 100644@@@ -0,0 +1,60 @@++-- phase: expand++-- Request analytics on PostgreSQL (plan T8, decision 5). The DDL is analyticsSchema() from++-- openvibe-shared/analytics/pg.js, copied verbatim. One set of `service`-keyed tables serves the seven++-- tracking apps (img, audio, docs, text, yt, maps, food); service = app. The gateway runs no tracker.++-- analytics_rate_tracking is deliberately absent (the PG twin omits it; the SQLite one is emptied and++-- nothing reads it).++++CREATE TABLE IF NOT EXISTS analytics_events (++ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,++ service text NOT NULL,++ event_type text NOT NULL DEFAULT 'pageview',++ path text,++ method text DEFAULT 'GET',++ status_code integer,++ response_time_ms integer,++ user_id bigint,++ session_id text,++ ip text,++ country text,++ city text,++ user_agent text,++ referer text,++ is_bot integer DEFAULT 0,++ bot_type text,++ device_type text,++ browser text,++ os text,++ authenticated integer NOT NULL DEFAULT 0,++ created_at text NOT NULL++);++CREATE TABLE IF NOT EXISTS analytics_hourly (++ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,++ service text NOT NULL, hour text NOT NULL,++ pageviews bigint DEFAULT 0, api_calls bigint DEFAULT 0, unique_visitors bigint DEFAULT 0, unique_users bigint DEFAULT 0,++ bot_hits bigint DEFAULT 0, avg_response_ms integer DEFAULT 0, error_count bigint DEFAULT 0, bandwidth_bytes bigint DEFAULT 0,++ top_paths text, top_referers text,++ UNIQUE (service, hour)++);++CREATE TABLE IF NOT EXISTS analytics_daily (++ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,++ service text NOT NULL, date text NOT NULL,++ pageviews bigint DEFAULT 0, api_calls bigint DEFAULT 0, unique_visitors bigint DEFAULT 0, unique_users bigint DEFAULT 0,++ new_users bigint DEFAULT 0, bot_hits bigint DEFAULT 0, avg_response_ms integer DEFAULT 0, error_count bigint DEFAULT 0,++ top_paths text, top_referers text, top_countries text, device_breakdown text, browser_breakdown text,++ UNIQUE (service, date)++);++CREATE TABLE IF NOT EXISTS analytics_visitor_days (++ service text NOT NULL, day text NOT NULL, hour text NOT NULL, vhash text NOT NULL,++ authenticated integer NOT NULL DEFAULT 0,++ PRIMARY KEY (service, hour, vhash)++);++CREATE TABLE IF NOT EXISTS analytics_day_salts (++ service text NOT NULL, day text NOT NULL, salt text NOT NULL,++ PRIMARY KEY (service, day)++);++CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events (created_at);++CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events (service, created_at);++CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events (service, path, created_at);++CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events (is_bot, created_at);++CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days (service, day);nnew file mode 100644@@@ -0,0 +1,174 @@++#!/usr/bin/env node++/**++ * Phase 0 of plan T8 — enforce-mode readiness for the tools guard.++ *++ * In report mode (`TOOLS_GUARD` unset) the guard records every refusal it *would* make into++ * `guard_abuse` with `enforced = 0`. Before flipping `TOOLS_GUARD=enforce`, read those rows and++ * confirm no reason fires on ordinary traffic: a false positive that report mode merely logged++ * becomes a 429 for a real visitor in enforce mode.++ *++ * This groups `guard_abuse WHERE enforced = 0` by (reason, tool) and prints, per reason/tool, how++ * many rows (and how many repeats the `count` column folds in), across how many hashed addresses,++ * and the window it happened in. The PowerShell with `enforced = 1` rows is printed too, for++ * context. It reads copies — never a live file it also writes.++ *++ * node scripts/guard-abuse-report.js # every apps/<app>/data/guard.db++ * node scripts/guard-abuse-report.js --app maps --app gateway++ * node scripts/guard-abuse-report.js --db /backups/maps/guard.db++ * node scripts/guard-abuse-report.js --json # machine-readable, one JSON document++ * node scripts/guard-abuse-report.js --pg # the migrated database (DATABASE_URL)++ * node scripts/guard-abuse-report.js --pg --url postgres://… # a specific PostgreSQL database++ *++ * --app <name> only this app (repeatable); default: every app that has data/guard.db++ * --db <file> a guard database by path (repeatable; replaces the app discovery)++ * --pg read guard_abuse from PostgreSQL (DATABASE_URL, or --url) instead of SQLite files++ * --url <url> the PostgreSQL URL (implies --pg)++ * --json emit JSON instead of text++ *++ * After the T8 migration the guard's abuse log lives in the one `tools` database, so --pg is the++ * form to use on production once the rows have moved. Before that, the SQLite files are the source.++ */++'use strict';++const fs = require('fs');++const path = require('path');++++const ROOT = path.join(__dirname, '..');++const APPS = path.join(ROOT, 'apps');++++/** A package from the first app that has it installed (the repo root has no node_modules). */++function fromApps(name) {++ for (const a of ['gateway', 'maps', 'img', 'docs', 'audio', 'text', 'yt', 'food']) {++ try { return require(require.resolve(name, { paths: [path.join(APPS, a)] })); } catch { /* next */ }++ }++ throw new Error(`${name} is not installed in any app (npm run install:all)`);++}++++function parseArgs(argv) {++ const out = { apps: [], dbs: [], pg: false, url: null, json: false };++ for (let i = 0; i < argv.length; i++) {++ const a = argv[i];++ if (a === '--app') out.apps.push(argv[++i]);++ else if (a === '--db') out.dbs.push(argv[++i]);++ else if (a === '--pg') out.pg = true;++ else if (a === '--url') { out.url = argv[++i]; out.pg = true; }++ else if (a === '--json') out.json = true;++ else if (a === '--help' || a === '-h') out.help = true;++ else throw new Error(`unknown option "${a}" (--help for usage)`);++ }++ return out;++}++++/** [{ name, file }] when no --db is given: the --app list, else every app with data/guard.db. */++function targets(args) {++ const names = args.apps.length ? args.apps : fs.readdirSync(APPS).filter((a) => !a.startsWith('_')).sort();++ return names.map((name) => ({ name, file: path.join(APPS, name, 'data', 'guard.db') }))++ .filter((t) => args.apps.length || fs.existsSync(t.file));++}++++// reason, tool, and how many rows/repeats/minutes/addresses a group covers, plus its window.++const SELECT = `++SELECT reason,++ tool,++ COUNT(*) AS rows,++ COALESCE(SUM(count), 0) AS events,++ COUNT(DISTINCT ip_hash) AS addresses,++ MIN(at) AS first_at,++ MAX(at) AS last_at++ FROM guard_abuse++ WHERE enforced = 0++ GROUP BY reason, tool++ ORDER BY events DESC, reason, tool`;++++// The enforced rows, only for context (what has actually been refused since the mode flag flipped).++const SELECT_ENFORCED = `++SELECT reason, tool, COUNT(*) AS rows, COALESCE(SUM(count), 0) AS events++ FROM guard_abuse++ WHERE enforced = 1++ GROUP BY reason, tool++ ORDER BY events DESC, reason, tool`;++++function tableExists(db, name) {++ return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));++}++++function reportSqlite(results, target, Database) {++ if (!fs.existsSync(target.file)) { results.push({ app: target.name, file: target.file, present: false }); return; }++ // Read-only: a report never writes to the file it reads (a live file may be in WAL).++ const db = new Database(target.file, { readonly: true, fileMustExist: true });++ try {++ if (!tableExists(db, 'guard_abuse')) { results.push({ app: target.name, file: target.file, present: true, hasTable: false }); return; }++ results.push({++ app: target.name,++ file: target.file,++ present: true,++ hasTable: true,++ mode: 'sqlite',++ reported: db.prepare(SELECT).all(),++ enforced: db.prepare(SELECT_ENFORCED).all(),++ });++ } finally { db.close(); }++}++++async function reportPg(results, url) {++ const { Client } = fromApps('pg');++ const client = new Client({ connectionString: url });++ await client.connect();++ try {++ await client.query('BEGIN READ ONLY');++ const reported = (await client.query(SELECT)).rows;++ const enforced = (await client.query(SELECT_ENFORCED)).rows;++ await client.query('ROLLBACK');++ results.push({ app: 'postgres', file: url.replace(/:\/\/[^@/]*@/, '://***@'), present: true, hasTable: true, mode: 'postgresql', reported, enforced });++ } finally { await client.end(); }++}++++const fmtTime = (ms) => (ms == null ? '—' : new Date(Number(ms)).toISOString());++++function printReport(results, log) {++ let any = false;++ for (const r of results) {++ if (!r.present) { log(`\n${r.app}: no guard database (${r.file}) — skipped`); continue; }++ if (r.hasTable === false) { log(`\n${r.app}: ${r.file} has no guard_abuse table — skipped`); continue; }++ any = true;++ log(`\n${r.app} (${r.file})$ ${r.reported.length} reason/tool group${r.reported.length === 1 ? '' : 's'} recorded but not enforced`);++ if (!r.reported.length) { log(' (nothing was reported: no would-be refusal on this copy)'); }++ for (const row of r.reported) {++ log(` ${String(row.events).padStart(6)} event${Number(row.events) === 1 ? ' ' : 's'} reason=${row.reason} tool=${row.tool || '(none)'}`++ + ` rows=${row.rows} addresses=${row.addresses} ${fmtTime(row.first_at)} … ${fmtTime(row.last_at)}`);++ }++ if (r.enforced.length) {++ log(' — already enforced (context):');++ for (const row of r.enforced) log(` ${String(row.events).padStart(6)} event${Number(row.events) === 1 ? ' ' : 's'} reason=${row.reason} tool=${row.tool || '(none)'}`);++ }++ }++ if (!any) log('\nNo guard database with a guard_abuse table was found.');++ log('\nReview each reason/tool pair above: any one that fires on ordinary traffic is a false');++ log('positive that TOOLS_GUARD=enforce would turn into a 429 for a real visitor.');++}++++async function main(argv, log = console.log) {++ const args = parseArgs(argv);++ if (args.help) {++ const usage = fs.readFileSync(__filename, 'utf8').split('\n').filter((l) => /^ \*/.test(l)).slice(0, 22).join('\n');++ log(usage.replace(/^ \* ?/gm, ''));++ return;++ }++ const results = [];++ if (args.pg) {++ const url = args.url || process.env.DATABASE_URL;++ if (!url) throw new Error('--pg needs DATABASE_URL or --url');++ await reportPg(results, url);++ } else {++ const Database = fromApps('better-sqlite3');++ const list = args.dbs.length ? args.dbs.map((f) => ({ name: path.basename(path.dirname(path.dirname(f))) || 'db', file: path.resolve(f) })) : targets(args);++ for (const t of list) reportSqlite(results, t, Database);++ }++ if (args.json) log(JSON.stringify({ generated_at: new Date().toISOString(), results }, null, 2));++ else printReport(results, log);++}++++if (require.main === module) {++ main(process.argv.slice(2)).catch((err) => { console.error(err.message); process.exit(1); });++}++++module.exports = { main, targets, parseArgs, SELECT, SELECT_ENFORCED, fromApps, printReport };nnew file mode 100644@@@ -0,0 +1,162 @@++'use strict';++// ═══════════════════════════════════════════════════════════════++// scripts/migrate-to-postgres.js — the one-time move of every Tools SQLite file into the ONE `tools`++// PostgreSQL database (plan T8, decision 3/6; ADR-035). The operator (Opus) runs it at cutover.++//++// node scripts/migrate-to-postgres.js # production: DATABASE_DIRECT_URL (owner role)++// node scripts/migrate-to-postgres.js --pglite # an in-memory rehearsal (report only)++// node scripts/migrate-to-postgres.js --dry-run --url postgres://…/tools_scratch # a scratch database++// node scripts/migrate-to-postgres.js --app maps --app gateway [--json]++//++// Every app's data/<guard|jobs|analytics>.db is imported into the shared tables, tagged with its app id++// (`app` = the app name; the analytics tables already carry `service`). guard_salt and guard_day are not++// copied (Valkey, decision 4); the event outbox and token_revocations are left behind (the SDK re-creates++// its own tables at boot and a revocation cutoff is re-derivable). The SQLite files are opened read-only.++//++// It relies on the newer openvibe-sdk/db (0.25+) and better-sqlite3; OV_SDK_DIR points at an openvibe-sdk++// checkout, else the gateway's node_modules is used. Not part of `npm test`.++// ═══════════════════════════════════════════════════════════════++const fs = require('fs');++const path = require('path');++const { createRequire } = require('module');++++const ROOT = path.join(__dirname, '..');++const APPS = path.join(ROOT, 'apps');++const MIGRATIONS = path.join(ROOT, 'migrations');++++// One row per kind of file, in every app. `tables` = the target tables that file fills; `app` = the ones++// that carry the app id; `skip` = source tables deliberately not imported (importSqlite would otherwise++// report them as having no target); `omit` = identity columns the target must assign itself (each app's++// ids are per-app and would collide in the one shared table: the import reads a temp table of the same++// name without them, so PostgreSQL issues fresh, globally unique ids — nothing references these ids).++const KINDS = {++ 'guard.db': { tables: ['guard_abuse'], app: ['guard_abuse'], skip: ['guard_salt', 'guard_day'], omit: { guard_abuse: ['id'] } },++ 'jobs.db': { tables: ['tool_jobs', 'tool_job_events', 'tool_job_references', 'tool_job_usage'], app: ['tool_jobs', 'tool_job_usage'], skip: ['event_outbox'], omit: { tool_job_events: ['seq'] } },++ 'analytics.db': { tables: ['analytics_events', 'analytics_hourly', 'analytics_daily', 'analytics_visitor_days', 'analytics_day_salts'], app: [], skip: ['analytics_rate_tracking'], omit: { analytics_events: ['id'], analytics_hourly: ['id'], analytics_daily: ['id'] } },++};++const TARGETS = [...new Set(Object.values(KINDS).flatMap((k) => k.tables))];++++const argv = process.argv.slice(2);++const has = (f) => argv.includes(`--${f}`);++const opt = (n) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? argv[i + 1] || null : null; };++const repeat = (n) => argv.flatMap((a, i) => (a === `--${n}` && argv[i + 1] ? [argv[i + 1]] : []));++const quiet = { log() {}, warn: console.warn, error: console.error };++++if (has('help')) {++ console.log('usage: node scripts/migrate-to-postgres.js [--pglite | --url <DATABASE_DIRECT_URL>] [--dry-run] [--app <name>]… [--json]');++ process.exit(0);++}++++/** Resolve openvibe-sdk/db from OV_SDK_DIR or an app's node_modules (so better-sqlite3 resolves too). */++function sdkRequire() {++ const dir = process.env.OV_SDK_DIR || path.join(APPS, 'gateway', 'node_modules', 'openvibe-sdk');++ try {++ const req = createRequire(path.join(dir, 'package.json'));++ return req('openvibe-sdk/db');++ } catch (err) {++ throw new Error(`openvibe-sdk/db could not be loaded from ${dir} (${err.message}); it needs openvibe-sdk 0.25+ (OV_SDK_DIR overrides)`);++ }++}++++/** Every app's data/<name>.db, in a deterministic order, optionally narrowed with --app. */++function discover(apps) {++ const out = [];++ for (const app of fs.readdirSync(APPS).sort()) {++ if (app.startsWith('_') || (apps.length && !apps.includes(app))) continue;++ const dir = path.join(APPS, app, 'data');++ if (!fs.existsSync(dir)) continue;++ for (const name of Object.keys(KINDS).sort()) {++ const file = path.join(dir, name);++ if (fs.existsSync(file) && fs.statSync(file).isFile() && fs.statSync(file).size > 0) out.push({ app, kind: name, file });++ }++ }++ return out;++}++++/** Require a module as one app would (so better-sqlite3 resolves from that app's node_modules). */++function appRequire(app, name) {++ return createRequire(path.join(APPS, app, 'package.json'))(name);++}++++const q = (x) => `"${String(x).replace(/"/g, '""')}"`;++++/**++ * Import one SQLite file. Identity-keyed tables (guard_abuse.id, tool_job_events.seq, the analytics ids)++ * are read through a temp table of the same name without the identity column, so PostgreSQL assigns a++ * fresh, globally unique id (per-app ids would collide in the one shared database). Nothing references++ * those ids. The file is opened read-only.++ */++async function importFile({ app, file, spec, db, importSqlite, first }) {++ const Database = appRequire(app, 'better-sqlite3');++ const src = new Database(file, { readonly: true, fileMustExist: true });++ try {++ for (const [table, drop] of Object.entries(spec.omit || {})) {++ const have = src.prepare(`PRAGMA table_info(${q(table)})`).all().map((c) => c.name);++ if (!have.length) continue; // the source has no such table (a lazily created one): nothing to import++ const keep = have.filter((c) => !drop.includes(c));++ src.exec(`CREATE TEMP TABLE ${q(table)} AS SELECT ${keep.map(q).join(', ')} FROM main.${q(table)}`);++ }++ return await importSqlite({ sqlite: src, db, only: spec.tables, tables: {}, skipSource: spec.skip, truncate: false, verify: first, log: quiet });++ } finally { src.close(); }++}++++async function main() {++ const apps = repeat('app');++ const sdk = sdkRequire();++ const { createDb, importSqlite } = sdk;++++ let db;++ if (has('pglite') || (has('dry-run') && !opt('url'))) {++ db = createDb({ pglite: true, service: 'tools-import', log: quiet });++ } else {++ const url = opt('url') || process.env.DATABASE_DIRECT_URL;++ if (!url) throw new Error('set DATABASE_DIRECT_URL (the owner, direct connection), or pass --pglite / --dry-run --url <scratch>');++ if (has('dry-run') && !opt('url')) throw new Error('--dry-run needs an explicit scratch --url (or --pglite) so production is never touched');++ db = createDb({ url, service: 'tools-import', max: 2, log: quiet });++ }++++ const files = discover(apps);++ const report = { store: db.store, files: [], tables: {}, problems: [] };++ const sourceCount = {}; // target table → rows seen in the SQLite files++ const doneKind = new Set();++++ try {++ await db.migrate({ dir: MIGRATIONS, log: quiet });++ await db.query(`TRUNCATE ${TARGETS.map((t) => `"${t}"`).join(', ')} RESTART IDENTITY CASCADE`);++++ for (const { app, kind, file } of files) {++ const spec = KINDS[kind];++ const first = !doneKind.has(kind);++ doneKind.add(kind);++ const r = await importFile({ app, file, spec, db, importSqlite, first });++ const rows = {};++ for (const t of r.tables) { rows[t.table] = t.rows; sourceCount[t.table] = (sourceCount[t.table] || 0) + t.rows; }++ // Tag the app-scoped rows this file just inserted (they came in with the '' default).++ for (const t of spec.app) await db.query(`UPDATE "${t}" SET app = $1 WHERE app = ''`, [app]);++ if (!r.ok) for (const p of r.problems) report.problems.push({ app, kind, ...p });++ report.files.push({ app, kind, file: path.relative(ROOT, file), rows, verified: first, checksums: Object.fromEntries(r.tables.map((t) => [t.table, t.checksum]).filter(([, c]) => c)) });++ if (process.stdout.isTTY) process.stderr.write(`[import] ${app}/${kind}: ${JSON.stringify(rows)}\n`);++ }++++ // Reconcile counts across apps (per-file verification only covers the first file of a kind).++ for (const t of TARGETS) {++ const pg = Number((await db.maybe(`SELECT count(*) AS n FROM "${t}"`)).n);++ const src = sourceCount[t] || 0;++ report.tables[t] = { source: src, postgres: pg, ok: src === pg };++ if (src !== pg) report.problems.push({ table: t, problem: `count mismatch: ${src} SQLite rows, ${pg} PostgreSQL rows` });++ }++ report.ok = report.problems.length === 0;++ } finally {++ await db.close();++ }++++ if (has('json')) { console.log(JSON.stringify(report, null, 2)); }++ else {++ console.log(`import → ${report.store}: ${report.ok ? 'OK' : 'PROBLEMS'}`);++ for (const [t, c] of Object.entries(report.tables)) console.log(` ${t.padEnd(26)} ${String(c.source).padStart(7)} SQLite rows → ${String(c.postgres).padStart(7)} PostgreSQL rows${c.ok ? '' : ' MISMATCH'}`);++ for (const p of report.problems) console.log(` problem: ${p.app ? `${p.app}/${p.kind} ` : ''}${p.table}: ${p.problem}`);++ }++ process.exit(report.ok ? 0 : 1);++}++++main().catch((err) => { console.error(`migrate-to-postgres: ${err.message}`); process.exit(1); });
@@ -624,3 +624,19 @@ when the gateway's `/api/ready` or `/release.json` does not come up with every u
@@ -624,3 +624,19 @@ when the gateway's `/api/ready` or `/release.json` does not come up with every u
- Nginx: satellites have specific `server_name` blocks; the gateway's
- Nginx: satellites have specific `server_name` blocks; the gateway's
wildcard `*.openvibe.tools` block catches everything else. TLS via
wildcard `*.openvibe.tools` block catches everything else. TLS via
`/etc/letsencrypt/live/openvibe.tools/`.
`/etc/letsencrypt/live/openvibe.tools/`.
+
+## Migration to PostgreSQL + Valkey (plan T8)
+
+All eight apps move from `better-sqlite3` to **one `tools` PostgreSQL database** and to **Valkey** for the
+guard's throttle state. The schema is in `migrations/0001_tools.sql` (job store, `guard_abuse`) and
+`migrations/0002_analytics.sql` (request analytics, the DDL of `openvibe-shared/analytics/pg.js`); the event
+outbox and the revocation store create their own tables at boot. `scripts/migrate-to-postgres.js` imports
+every `apps/*/data/{guard,jobs,analytics}.db` into that one database (each file's rows tagged with its app):
+run it with `--pglite` for an in-memory rehearsal, `--dry-run --url <scratch>` for a scratch database, or
+with `DATABASE_DIRECT_URL` set for the cutover; it prints a per-table count and checksum report.
+
+After the move the apps read `DATABASE_URL` / `DATABASE_DIRECT_URL` (PostgreSQL, via PgBouncer) and
+`VALKEY_URL` (Valkey); `guard_salt`, `guard_day` and the minute/burst buckets live in Valkey (a throttle,
+never authoritative), while `guard_abuse`, the job store and the analytics tables stay in PostgreSQL.
+`TOOLS_GUARD` (default `report`) and its meaning are unchanged. Until the cutover lands the apps still run
+on `better-sqlite3` exactly as described above.
@@ -432,31 +432,12 @@
@@ -432,31 +432,12 @@
processing.style.display = '';
processing.style.display = '';
// Jobs: the work is accepted, followed live (ffmpeg's progress) and survives a reload.
// Jobs: the work is accepted, followed live (ffmpeg's progress) and survives a reload.
- if (window.OVJobs) {
- const input = {};
- for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;
- try {
- const job = await OVJobs.submit({ type: 'audio.process', input, files, fileField: isMerge() ? 'files' : 'file' });
- OVJobs.remember(job.id);
- followJob(job);
- } catch (err) {
- showToast(err.message, 'error');
- resetUI();
- }
- return;
- }
-
- // Without the jobs helper: the synchronous endpoint, as before.
+ const input = {};
+ for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;
try {
try {
- const res = await fetch(isMerge() ? '/api/process/multi' : '/api/process', { method: 'POST', body: formData });
- const data = await res.json();
-
- if (!res.ok) {
- throw new Error(data.error || `Server error (${res.status})`);
- }
-
- resultId = data.download.id;
- showResult(data);
+ const job = await OVJobs.submit({ type: 'audio.process', input, files, fileField: isMerge() ? 'files' : 'file' });
+ OVJobs.remember(job.id);
+ followJob(job);
} catch (err) {
} catch (err) {
showToast(err.message, 'error');
showToast(err.message, 'error');
resetUI();
resetUI();
@@ -460,46 +460,14 @@
@@ -460,46 +460,14 @@
addToolOptions(formData);
addToolOptions(formData);
// Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).
// Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).
- if (window.OVJobs) {
- const input = {};
- for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;
- try {
- const job = await OVJobs.submit({ type: 'docs.process', input, files: isMulti ? selectedFiles : [selectedFiles[0]], fileField: isMulti ? 'files' : 'file' });
- OVJobs.remember(job.id);
- followJob(job);
- } catch (err) {
- failProcessing(err.message);
- }
- return;
- }
-
- // Without the jobs helper: the synchronous endpoints, as before.
+ const input = {};
+ for (const [k, v] of formData.entries()) if (k !== 'file' && k !== 'files') input[k] = v;
try {
try {
- const token = getCookie('ov_token') || localStorage.getItem('ov_token');
- const headers = {};
- if (token) headers['Authorization'] = `Bearer ${token}`;
-
- const endpoint = isMulti ? '/api/process/multi' : '/api/process';
- const res = await fetch(endpoint, { method: 'POST', body: formData, headers });
-
- if (!res.ok) {
- const err = await res.json().catch(() => ({ error: 'Processing failed' }));
- throw new Error(err.error || `Server error: ${res.status}`);
- }
-
- const data = await res.json();
-
- // Handle view-only results (metadata)
- if (data.viewOnly) {
- showMetadataResult(data);
- } else {
- showResult(data);
- }
+ const job = await OVJobs.submit({ type: 'docs.process', input, files: isMulti ? selectedFiles : [selectedFiles[0]], fileField: isMulti ? 'files' : 'file' });
+ OVJobs.remember(job.id);
+ followJob(job);
} catch (err) {
} catch (err) {
- processingEl.style.display = 'none';
- optionsPanel.style.display = '';
- processBtn.disabled = false;
- alert(`Error: ${err.message}`);
+ failProcessing(err.message);
}
}
}
}
@@ -374,39 +374,14 @@
@@ -374,39 +374,14 @@
}
}
// Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).
// Jobs: the work is accepted, followed live and survives a reload (?job=<id> in the address).
- if (window.OVJobs) {
- const input = {};
- for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;
- try {
- const job = await OVJobs.submit({ type: 'img.process', input, files: [selectedFile] });
- OVJobs.remember(job.id);
- followJob(job);
- } catch (err) {
- failProcessing(err.message);
- }
- return;
- }
-
- // Without the jobs helper: the synchronous endpoint, as before.
+ const input = {};
+ for (const [k, v] of formData.entries()) if (k !== 'file') input[k] = v;
try {
try {
- const token = getCookie('ov_token') || localStorage.getItem('ov_token');
- const headers = {};
- if (token) headers['Authorization'] = `Bearer ${token}`;
-
- const res = await fetch('/api/process', { method: 'POST', body: formData, headers });
-
- if (!res.ok) {
- const err = await res.json().catch(() => ({ error: 'Processing failed' }));
- throw new Error(err.error || `Server error: ${res.status}`);
- }
-
- const data = await res.json();
- showResult(data);
+ const job = await OVJobs.submit({ type: 'img.process', input, files: [selectedFile] });
+ OVJobs.remember(job.id);
+ followJob(job);
} catch (err) {
} catch (err) {
- processingEl.style.display = 'none';
- optionsPanel.style.display = '';
- processBtn.disabled = false;
- alert(`Error: ${err.message}`);
+ failProcessing(err.message);
}
}
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,55 @@
@@ -0,0 +1,55 @@
+-- phase: expand
+-- One `tools` database for all eight apps (plan T8, decision 3). Every key that could collide across
+-- apps carries `app`. The runtime role creates no tables: this file is applied by migrations.
+--
+-- Not here on purpose (plan T8, decision 4):
+-- guard_salt, guard_day, the minute/burst token buckets -> Valkey (throttle state, worthless after a day)
+-- the event outbox -> openvibe-sdk createPgOutbox creates its own table at boot
+-- token_revocations -> openvibe-sdk/auth createPgRevocationStore creates its own table at boot
+--
+-- `app` (the owning app, one row per app in the one database) has DEFAULT '' so scripts/migrate-to-postgres.js
+-- can import a file that has no such column; the script sets the real value in the same pass and the runtime
+-- always writes it explicitly. A default of '' is never a real owner.
+-- analytics tables (analytics_events, analytics_hourly, analytics_daily, analytics_visitor_days,
+-- analytics_day_salts and their indexes) are the analyticsSchema() DDL from
+-- openvibe-shared/analytics/pg.js, copied verbatim into 0002_analytics.sql.
+
+CREATE TABLE IF NOT EXISTS tool_jobs (
+ id text PRIMARY KEY, app text NOT NULL DEFAULT '', type text NOT NULL, type_version integer NOT NULL DEFAULT 1,
+ owner text NOT NULL, state text NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled')),
+ progress double precision, progress_message text, input_json text NOT NULL DEFAULT '{}',
+ files_json text NOT NULL DEFAULT '[]', result_json text, error_json text,
+ retryable integer NOT NULL DEFAULT 0, idempotency_key text, request_hash text,
+ attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL DEFAULT 1,
+ cancel_requested integer NOT NULL DEFAULT 0, ttl_ms integer NOT NULL DEFAULT 3600000,
+ created_at bigint NOT NULL, updated_at bigint NOT NULL, started_at bigint, finished_at bigint, expires_at bigint,
+ env text NOT NULL DEFAULT 'production' CHECK (env IN ('production','sandbox')),
+ retry_of text, retried_by text, ip_key text, tool text, project_id text, trace_id text);
+CREATE UNIQUE INDEX IF NOT EXISTS tool_jobs_idem ON tool_jobs(app, owner, idempotency_key) WHERE idempotency_key IS NOT NULL;
+CREATE INDEX IF NOT EXISTS tool_jobs_state ON tool_jobs(app, state, id);
+CREATE INDEX IF NOT EXISTS tool_jobs_expiry ON tool_jobs(app, expires_at) WHERE expires_at IS NOT NULL;
+CREATE INDEX IF NOT EXISTS tool_jobs_owner ON tool_jobs(app, owner, state);
+CREATE INDEX IF NOT EXISTS tool_jobs_ip ON tool_jobs(app, ip_key, state) WHERE ip_key IS NOT NULL;
+
+CREATE TABLE IF NOT EXISTS tool_job_events (
+ seq bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, job_id text NOT NULL,
+ event text NOT NULL, data text NOT NULL, at bigint NOT NULL);
+CREATE INDEX IF NOT EXISTS tool_job_events_job ON tool_job_events(job_id, seq);
+
+CREATE TABLE IF NOT EXISTS tool_job_references (
+ job_id text NOT NULL, ref text NOT NULL, created_at bigint NOT NULL, PRIMARY KEY (job_id, ref));
+
+CREATE TABLE IF NOT EXISTS tool_job_usage (
+ app text NOT NULL DEFAULT '', project_id text NOT NULL, env text NOT NULL, capability text NOT NULL,
+ dimension text NOT NULL, window_start bigint NOT NULL, quantity integer NOT NULL DEFAULT 0,
+ errors integer NOT NULL DEFAULT 0, error_codes text NOT NULL DEFAULT '{}', samples text NOT NULL DEFAULT '[]',
+ revision integer NOT NULL DEFAULT 1, event_id text, emitted_at bigint,
+ PRIMARY KEY (app, project_id, env, capability, dimension, window_start));
+CREATE INDEX IF NOT EXISTS tool_job_usage_open ON tool_job_usage(app, emitted_at, window_start);
+
+CREATE TABLE IF NOT EXISTS guard_abuse (
+ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, app text NOT NULL DEFAULT '', at bigint NOT NULL,
+ minute bigint NOT NULL, ip_hash text NOT NULL DEFAULT '', principal text NOT NULL DEFAULT '',
+ tool text NOT NULL DEFAULT '', reason text NOT NULL, enforced integer NOT NULL, count integer NOT NULL DEFAULT 1);
+CREATE UNIQUE INDEX IF NOT EXISTS guard_abuse_once ON guard_abuse(app, minute, ip_hash, principal, tool, reason, enforced);
+CREATE INDEX IF NOT EXISTS guard_abuse_at ON guard_abuse(app, at);
new file mode 100644
new file mode 100644
@@ -0,0 +1,60 @@
@@ -0,0 +1,60 @@
+-- phase: expand
+-- Request analytics on PostgreSQL (plan T8, decision 5). The DDL is analyticsSchema() from
+-- openvibe-shared/analytics/pg.js, copied verbatim. One set of `service`-keyed tables serves the seven
+-- tracking apps (img, audio, docs, text, yt, maps, food); service = app. The gateway runs no tracker.
+-- analytics_rate_tracking is deliberately absent (the PG twin omits it; the SQLite one is emptied and
+-- nothing reads it).
+
+CREATE TABLE IF NOT EXISTS analytics_events (
+ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ service text NOT NULL,
+ event_type text NOT NULL DEFAULT 'pageview',
+ path text,
+ method text DEFAULT 'GET',
+ status_code integer,
+ response_time_ms integer,
+ user_id bigint,
+ session_id text,
+ ip text,
+ country text,
+ city text,
+ user_agent text,
+ referer text,
+ is_bot integer DEFAULT 0,
+ bot_type text,
+ device_type text,
+ browser text,
+ os text,
+ authenticated integer NOT NULL DEFAULT 0,
+ created_at text NOT NULL
+);
+CREATE TABLE IF NOT EXISTS analytics_hourly (
+ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ service text NOT NULL, hour text NOT NULL,
+ pageviews bigint DEFAULT 0, api_calls bigint DEFAULT 0, unique_visitors bigint DEFAULT 0, unique_users bigint DEFAULT 0,
+ bot_hits bigint DEFAULT 0, avg_response_ms integer DEFAULT 0, error_count bigint DEFAULT 0, bandwidth_bytes bigint DEFAULT 0,
+ top_paths text, top_referers text,
+ UNIQUE (service, hour)
+);
+CREATE TABLE IF NOT EXISTS analytics_daily (
+ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ service text NOT NULL, date text NOT NULL,
+ pageviews bigint DEFAULT 0, api_calls bigint DEFAULT 0, unique_visitors bigint DEFAULT 0, unique_users bigint DEFAULT 0,
+ new_users bigint DEFAULT 0, bot_hits bigint DEFAULT 0, avg_response_ms integer DEFAULT 0, error_count bigint DEFAULT 0,
+ top_paths text, top_referers text, top_countries text, device_breakdown text, browser_breakdown text,
+ UNIQUE (service, date)
+);
+CREATE TABLE IF NOT EXISTS analytics_visitor_days (
+ service text NOT NULL, day text NOT NULL, hour text NOT NULL, vhash text NOT NULL,
+ authenticated integer NOT NULL DEFAULT 0,
+ PRIMARY KEY (service, hour, vhash)
+);
+CREATE TABLE IF NOT EXISTS analytics_day_salts (
+ service text NOT NULL, day text NOT NULL, salt text NOT NULL,
+ PRIMARY KEY (service, day)
+);
+CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events (created_at);
+CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events (service, created_at);
+CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events (service, path, created_at);
+CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events (is_bot, created_at);
+CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days (service, day);
new file mode 100644
new file mode 100644
@@ -0,0 +1,174 @@
@@ -0,0 +1,174 @@
+#!/usr/bin/env node
+/**
+ * Phase 0 of plan T8 — enforce-mode readiness for the tools guard.
+ *
+ * In report mode (`TOOLS_GUARD` unset) the guard records every refusal it *would* make into
+ * `guard_abuse` with `enforced = 0`. Before flipping `TOOLS_GUARD=enforce`, read those rows and
+ * confirm no reason fires on ordinary traffic: a false positive that report mode merely logged
+ * becomes a 429 for a real visitor in enforce mode.
+ *
+ * This groups `guard_abuse WHERE enforced = 0` by (reason, tool) and prints, per reason/tool, how
+ * many rows (and how many repeats the `count` column folds in), across how many hashed addresses,
+ * and the window it happened in. The PowerShell with `enforced = 1` rows is printed too, for
+ * context. It reads copies — never a live file it also writes.
+ *
+ * node scripts/guard-abuse-report.js # every apps/<app>/data/guard.db
+ * node scripts/guard-abuse-report.js --app maps --app gateway
+ * node scripts/guard-abuse-report.js --db /backups/maps/guard.db
+ * node scripts/guard-abuse-report.js --json # machine-readable, one JSON document
+ * node scripts/guard-abuse-report.js --pg # the migrated database (DATABASE_URL)
+ * node scripts/guard-abuse-report.js --pg --url postgres://… # a specific PostgreSQL database
+ *
+ * --app <name> only this app (repeatable); default: every app that has data/guard.db
+ * --db <file> a guard database by path (repeatable; replaces the app discovery)
+ * --pg read guard_abuse from PostgreSQL (DATABASE_URL, or --url) instead of SQLite files
+ * --url <url> the PostgreSQL URL (implies --pg)
+ * --json emit JSON instead of text
+ *
+ * After the T8 migration the guard's abuse log lives in the one `tools` database, so --pg is the
+ * form to use on production once the rows have moved. Before that, the SQLite files are the source.
+ */
+'use strict';
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+const APPS = path.join(ROOT, 'apps');
+
+/** A package from the first app that has it installed (the repo root has no node_modules). */
+function fromApps(name) {
+ for (const a of ['gateway', 'maps', 'img', 'docs', 'audio', 'text', 'yt', 'food']) {
+ try { return require(require.resolve(name, { paths: [path.join(APPS, a)] })); } catch { /* next */ }
+ }
+ throw new Error(`${name} is not installed in any app (npm run install:all)`);
+}
+
+function parseArgs(argv) {
+ const out = { apps: [], dbs: [], pg: false, url: null, json: false };
+ for (let i = 0; i < argv.length; i++) {
+ const a = argv[i];
+ if (a === '--app') out.apps.push(argv[++i]);
+ else if (a === '--db') out.dbs.push(argv[++i]);
+ else if (a === '--pg') out.pg = true;
+ else if (a === '--url') { out.url = argv[++i]; out.pg = true; }
+ else if (a === '--json') out.json = true;
+ else if (a === '--help' || a === '-h') out.help = true;
+ else throw new Error(`unknown option "${a}" (--help for usage)`);
+ }
+ return out;
+}
+
+/** [{ name, file }] when no --db is given: the --app list, else every app with data/guard.db. */
+function targets(args) {
+ const names = args.apps.length ? args.apps : fs.readdirSync(APPS).filter((a) => !a.startsWith('_')).sort();
+ return names.map((name) => ({ name, file: path.join(APPS, name, 'data', 'guard.db') }))
+ .filter((t) => args.apps.length || fs.existsSync(t.file));
+}
+
+// reason, tool, and how many rows/repeats/minutes/addresses a group covers, plus its window.
+const SELECT = `
+SELECT reason,
+ tool,
+ COUNT(*) AS rows,
+ COALESCE(SUM(count), 0) AS events,
+ COUNT(DISTINCT ip_hash) AS addresses,
+ MIN(at) AS first_at,
+ MAX(at) AS last_at
+ FROM guard_abuse
+ WHERE enforced = 0
+ GROUP BY reason, tool
+ ORDER BY events DESC, reason, tool`;
+
+// The enforced rows, only for context (what has actually been refused since the mode flag flipped).
+const SELECT_ENFORCED = `
+SELECT reason, tool, COUNT(*) AS rows, COALESCE(SUM(count), 0) AS events
+ FROM guard_abuse
+ WHERE enforced = 1
+ GROUP BY reason, tool
+ ORDER BY events DESC, reason, tool`;
+
+function tableExists(db, name) {
+ return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
+}
+
+function reportSqlite(results, target, Database) {
+ if (!fs.existsSync(target.file)) { results.push({ app: target.name, file: target.file, present: false }); return; }
+ // Read-only: a report never writes to the file it reads (a live file may be in WAL).
+ const db = new Database(target.file, { readonly: true, fileMustExist: true });
+ try {
+ if (!tableExists(db, 'guard_abuse')) { results.push({ app: target.name, file: target.file, present: true, hasTable: false }); return; }
+ results.push({
+ app: target.name,
+ file: target.file,
+ present: true,
+ hasTable: true,
+ mode: 'sqlite',
+ reported: db.prepare(SELECT).all(),
+ enforced: db.prepare(SELECT_ENFORCED).all(),
+ });
+ } finally { db.close(); }
+}
+
+async function reportPg(results, url) {
+ const { Client } = fromApps('pg');
+ const client = new Client({ connectionString: url });
+ await client.connect();
+ try {
+ await client.query('BEGIN READ ONLY');
+ const reported = (await client.query(SELECT)).rows;
+ const enforced = (await client.query(SELECT_ENFORCED)).rows;
+ await client.query('ROLLBACK');
+ results.push({ app: 'postgres', file: url.replace(/:\/\/[^@/]*@/, '://***@'), present: true, hasTable: true, mode: 'postgresql', reported, enforced });
+ } finally { await client.end(); }
+}
+
+const fmtTime = (ms) => (ms == null ? '—' : new Date(Number(ms)).toISOString());
+
+function printReport(results, log) {
+ let any = false;
+ for (const r of results) {
+ if (!r.present) { log(`\n${r.app}: no guard database (${r.file}) — skipped`); continue; }
+ if (r.hasTable === false) { log(`\n${r.app}: ${r.file} has no guard_abuse table — skipped`); continue; }
+ any = true;
+ log(`\n${r.app} (${r.file})$ ${r.reported.length} reason/tool group${r.reported.length === 1 ? '' : 's'} recorded but not enforced`);
+ if (!r.reported.length) { log(' (nothing was reported: no would-be refusal on this copy)'); }
+ for (const row of r.reported) {
+ log(` ${String(row.events).padStart(6)} event${Number(row.events) === 1 ? ' ' : 's'} reason=${row.reason} tool=${row.tool || '(none)'}`
+ + ` rows=${row.rows} addresses=${row.addresses} ${fmtTime(row.first_at)} … ${fmtTime(row.last_at)}`);
+ }
+ if (r.enforced.length) {
+ log(' — already enforced (context):');
+ for (const row of r.enforced) log(` ${String(row.events).padStart(6)} event${Number(row.events) === 1 ? ' ' : 's'} reason=${row.reason} tool=${row.tool || '(none)'}`);
+ }
+ }
+ if (!any) log('\nNo guard database with a guard_abuse table was found.');
+ log('\nReview each reason/tool pair above: any one that fires on ordinary traffic is a false');
+ log('positive that TOOLS_GUARD=enforce would turn into a 429 for a real visitor.');
+}
+
+async function main(argv, log = console.log) {
+ const args = parseArgs(argv);
+ if (args.help) {
+ const usage = fs.readFileSync(__filename, 'utf8').split('\n').filter((l) => /^ \*/.test(l)).slice(0, 22).join('\n');
+ log(usage.replace(/^ \* ?/gm, ''));
+ return;
+ }
+ const results = [];
+ if (args.pg) {
+ const url = args.url || process.env.DATABASE_URL;
+ if (!url) throw new Error('--pg needs DATABASE_URL or --url');
+ await reportPg(results, url);
+ } else {
+ const Database = fromApps('better-sqlite3');
+ const list = args.dbs.length ? args.dbs.map((f) => ({ name: path.basename(path.dirname(path.dirname(f))) || 'db', file: path.resolve(f) })) : targets(args);
+ for (const t of list) reportSqlite(results, t, Database);
+ }
+ if (args.json) log(JSON.stringify({ generated_at: new Date().toISOString(), results }, null, 2));
+ else printReport(results, log);
+}
+
+if (require.main === module) {
+ main(process.argv.slice(2)).catch((err) => { console.error(err.message); process.exit(1); });
+}
+
+module.exports = { main, targets, parseArgs, SELECT, SELECT_ENFORCED, fromApps, printReport };
new file mode 100644
new file mode 100644
@@ -0,0 +1,162 @@
@@ -0,0 +1,162 @@
+'use strict';
+// ═══════════════════════════════════════════════════════════════
+// scripts/migrate-to-postgres.js — the one-time move of every Tools SQLite file into the ONE `tools`
+// PostgreSQL database (plan T8, decision 3/6; ADR-035). The operator (Opus) runs it at cutover.
+//
+// node scripts/migrate-to-postgres.js # production: DATABASE_DIRECT_URL (owner role)
+// node scripts/migrate-to-postgres.js --pglite # an in-memory rehearsal (report only)
+// node scripts/migrate-to-postgres.js --dry-run --url postgres://…/tools_scratch # a scratch database
+// node scripts/migrate-to-postgres.js --app maps --app gateway [--json]
+//
+// Every app's data/<guard|jobs|analytics>.db is imported into the shared tables, tagged with its app id
+// (`app` = the app name; the analytics tables already carry `service`). guard_salt and guard_day are not
+// copied (Valkey, decision 4); the event outbox and token_revocations are left behind (the SDK re-creates
+// its own tables at boot and a revocation cutoff is re-derivable). The SQLite files are opened read-only.
+//
+// It relies on the newer openvibe-sdk/db (0.25+) and better-sqlite3; OV_SDK_DIR points at an openvibe-sdk
+// checkout, else the gateway's node_modules is used. Not part of `npm test`.
+// ═══════════════════════════════════════════════════════════════
+const fs = require('fs');
+const path = require('path');
+const { createRequire } = require('module');
+
+const ROOT = path.join(__dirname, '..');
+const APPS = path.join(ROOT, 'apps');
+const MIGRATIONS = path.join(ROOT, 'migrations');
+
+// One row per kind of file, in every app. `tables` = the target tables that file fills; `app` = the ones
+// that carry the app id; `skip` = source tables deliberately not imported (importSqlite would otherwise
+// report them as having no target); `omit` = identity columns the target must assign itself (each app's
+// ids are per-app and would collide in the one shared table: the import reads a temp table of the same
+// name without them, so PostgreSQL issues fresh, globally unique ids — nothing references these ids).
+const KINDS = {
+ 'guard.db': { tables: ['guard_abuse'], app: ['guard_abuse'], skip: ['guard_salt', 'guard_day'], omit: { guard_abuse: ['id'] } },
+ 'jobs.db': { tables: ['tool_jobs', 'tool_job_events', 'tool_job_references', 'tool_job_usage'], app: ['tool_jobs', 'tool_job_usage'], skip: ['event_outbox'], omit: { tool_job_events: ['seq'] } },
+ 'analytics.db': { tables: ['analytics_events', 'analytics_hourly', 'analytics_daily', 'analytics_visitor_days', 'analytics_day_salts'], app: [], skip: ['analytics_rate_tracking'], omit: { analytics_events: ['id'], analytics_hourly: ['id'], analytics_daily: ['id'] } },
+};
+const TARGETS = [...new Set(Object.values(KINDS).flatMap((k) => k.tables))];
+
+const argv = process.argv.slice(2);
+const has = (f) => argv.includes(`--${f}`);
+const opt = (n) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? argv[i + 1] || null : null; };
+const repeat = (n) => argv.flatMap((a, i) => (a === `--${n}` && argv[i + 1] ? [argv[i + 1]] : []));
+const quiet = { log() {}, warn: console.warn, error: console.error };
+
+if (has('help')) {
+ console.log('usage: node scripts/migrate-to-postgres.js [--pglite | --url <DATABASE_DIRECT_URL>] [--dry-run] [--app <name>]… [--json]');
+ process.exit(0);
+}
+
+/** Resolve openvibe-sdk/db from OV_SDK_DIR or an app's node_modules (so better-sqlite3 resolves too). */
+function sdkRequire() {
+ const dir = process.env.OV_SDK_DIR || path.join(APPS, 'gateway', 'node_modules', 'openvibe-sdk');
+ try {
+ const req = createRequire(path.join(dir, 'package.json'));
+ return req('openvibe-sdk/db');
+ } catch (err) {
+ throw new Error(`openvibe-sdk/db could not be loaded from ${dir} (${err.message}); it needs openvibe-sdk 0.25+ (OV_SDK_DIR overrides)`);
+ }
+}
+
+/** Every app's data/<name>.db, in a deterministic order, optionally narrowed with --app. */
+function discover(apps) {
+ const out = [];
+ for (const app of fs.readdirSync(APPS).sort()) {
+ if (app.startsWith('_') || (apps.length && !apps.includes(app))) continue;
+ const dir = path.join(APPS, app, 'data');
+ if (!fs.existsSync(dir)) continue;
+ for (const name of Object.keys(KINDS).sort()) {
+ const file = path.join(dir, name);
+ if (fs.existsSync(file) && fs.statSync(file).isFile() && fs.statSync(file).size > 0) out.push({ app, kind: name, file });
+ }
+ }
+ return out;
+}
+
+/** Require a module as one app would (so better-sqlite3 resolves from that app's node_modules). */
+function appRequire(app, name) {
+ return createRequire(path.join(APPS, app, 'package.json'))(name);
+}
+
+const q = (x) => `"${String(x).replace(/"/g, '""')}"`;
+
+/**
+ * Import one SQLite file. Identity-keyed tables (guard_abuse.id, tool_job_events.seq, the analytics ids)
+ * are read through a temp table of the same name without the identity column, so PostgreSQL assigns a
+ * fresh, globally unique id (per-app ids would collide in the one shared database). Nothing references
+ * those ids. The file is opened read-only.
+ */
+async function importFile({ app, file, spec, db, importSqlite, first }) {
+ const Database = appRequire(app, 'better-sqlite3');
+ const src = new Database(file, { readonly: true, fileMustExist: true });
+ try {
+ for (const [table, drop] of Object.entries(spec.omit || {})) {
+ const have = src.prepare(`PRAGMA table_info(${q(table)})`).all().map((c) => c.name);
+ if (!have.length) continue; // the source has no such table (a lazily created one): nothing to import
+ const keep = have.filter((c) => !drop.includes(c));
+ src.exec(`CREATE TEMP TABLE ${q(table)} AS SELECT ${keep.map(q).join(', ')} FROM main.${q(table)}`);
+ }
+ return await importSqlite({ sqlite: src, db, only: spec.tables, tables: {}, skipSource: spec.skip, truncate: false, verify: first, log: quiet });
+ } finally { src.close(); }
+}
+
+async function main() {
+ const apps = repeat('app');
+ const sdk = sdkRequire();
+ const { createDb, importSqlite } = sdk;
+
+ let db;
+ if (has('pglite') || (has('dry-run') && !opt('url'))) {
+ db = createDb({ pglite: true, service: 'tools-import', log: quiet });
+ } else {
+ const url = opt('url') || process.env.DATABASE_DIRECT_URL;
+ if (!url) throw new Error('set DATABASE_DIRECT_URL (the owner, direct connection), or pass --pglite / --dry-run --url <scratch>');
+ if (has('dry-run') && !opt('url')) throw new Error('--dry-run needs an explicit scratch --url (or --pglite) so production is never touched');
+ db = createDb({ url, service: 'tools-import', max: 2, log: quiet });
+ }
+
+ const files = discover(apps);
+ const report = { store: db.store, files: [], tables: {}, problems: [] };
+ const sourceCount = {}; // target table → rows seen in the SQLite files
+ const doneKind = new Set();
+
+ try {
+ await db.migrate({ dir: MIGRATIONS, log: quiet });
+ await db.query(`TRUNCATE ${TARGETS.map((t) => `"${t}"`).join(', ')} RESTART IDENTITY CASCADE`);
+
+ for (const { app, kind, file } of files) {
+ const spec = KINDS[kind];
+ const first = !doneKind.has(kind);
+ doneKind.add(kind);
+ const r = await importFile({ app, file, spec, db, importSqlite, first });
+ const rows = {};
+ for (const t of r.tables) { rows[t.table] = t.rows; sourceCount[t.table] = (sourceCount[t.table] || 0) + t.rows; }
+ // Tag the app-scoped rows this file just inserted (they came in with the '' default).
+ for (const t of spec.app) await db.query(`UPDATE "${t}" SET app = $1 WHERE app = ''`, [app]);
+ if (!r.ok) for (const p of r.problems) report.problems.push({ app, kind, ...p });
+ report.files.push({ app, kind, file: path.relative(ROOT, file), rows, verified: first, checksums: Object.fromEntries(r.tables.map((t) => [t.table, t.checksum]).filter(([, c]) => c)) });
+ if (process.stdout.isTTY) process.stderr.write(`[import] ${app}/${kind}: ${JSON.stringify(rows)}\n`);
+ }
+
+ // Reconcile counts across apps (per-file verification only covers the first file of a kind).
+ for (const t of TARGETS) {
+ const pg = Number((await db.maybe(`SELECT count(*) AS n FROM "${t}"`)).n);
+ const src = sourceCount[t] || 0;
+ report.tables[t] = { source: src, postgres: pg, ok: src === pg };
+ if (src !== pg) report.problems.push({ table: t, problem: `count mismatch: ${src} SQLite rows, ${pg} PostgreSQL rows` });
+ }
+ report.ok = report.problems.length === 0;
+ } finally {
+ await db.close();
+ }
+
+ if (has('json')) { console.log(JSON.stringify(report, null, 2)); }
+ else {
+ console.log(`import → ${report.store}: ${report.ok ? 'OK' : 'PROBLEMS'}`);
+ for (const [t, c] of Object.entries(report.tables)) console.log(` ${t.padEnd(26)} ${String(c.source).padStart(7)} SQLite rows → ${String(c.postgres).padStart(7)} PostgreSQL rows${c.ok ? '' : ' MISMATCH'}`);
+ for (const p of report.problems) console.log(` problem: ${p.app ? `${p.app}/${p.kind} ` : ''}${p.table}: ${p.problem}`);
+ }
+ process.exit(report.ok ? 0 : 1);
+}
+
+main().catch((err) => { console.error(`migrate-to-postgres: ${err.message}`); process.exit(1); });