openvibers/openvibe.network Commits
Analytics within ADR-021: no IP, user id or city in raw events; route templates, referer origins and user-agent classes; rotating session ids; raw events pruned after 30 days (nightly job, bounded batches), rollups kept
Changes
@@@ -285,6 +285,35 @@ Google-style account management supporting up to 5 accounts:- [docs/patches/live-observability.diff](docs/patches/live-observability.diff) is the same change forOpenVibe.Live, to be applied there.++## Analytics (ADR-021)++++Network records one row per finished request in the `analytics_*` tables of `network.db` and rolls++them up hourly and daily for the admin analytics pages (`/api/admin/analytics`, which also gathers++Live, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (OpenVibe.Contracts++`docs/adr/ADR-021-analytics.md`). `server/analytics/` holds the same module as OpenVibe.Live++(`server/analytics/`) and OpenVibe.Tools (`apps/_shared/analytics/`); Network's wiring is++`server/analytics/network.js`.++++- **Stored:** event type, service, **route template** (the matched Express route, else the path++ without its query and with ids, usernames and tokens replaced by `:id` / `:param`), method, status,++ response time, a **rotating session id**, country (CDN header), **user-agent class**++ (`chrome/windows/desktop`, `bot:curl`), referer **origin** only, bot flags, a signed-in flag.++- **Never stored:** IP address, user or subject id, city, the user-agent string, full referer URLs.++ The `ip`, `user_id` and `city` columns stay for compatibility and are always NULL. Per-IP counters++ for the bot rate check live in memory only. Unique visitors come from a daily-salted hash kept only++ until that day's final rollup. "New vs returning visitors" is no longer measured.++- **Admin panel:** the bot tables list user-agent classes and high-volume session ids, not IPs.++- **Connection:** the tracker opens its own connection to `network.db`, so its settings++ (`busy_timeout` 250, `secure_delete`) never apply to the identity connection.++- **Retention:** raw events older than 30 days are deleted every night in batches of 5000 (the++ `analytics-prune` job: first run 5 minutes after boot, then every 24 h). Rollups are kept.++- **Operator CLI:** `scripts/analytics-prune.js` runs a dry run by default and changes nothing.++ `--apply` needs `--backup <new file>` (a verified, owner-only copy of all of `network.db`) or an++ explicit `--no-backup`. `--scrub` also rewrites the rows the old tracker wrote and the rollups'++ top-path and referer lists. Rollup totals are compared before and after the run. VACUUM rewrites++ all of `network.db`, so run it with the service stopped, or pass `--no-vacuum` while it is up. Run++ the CLI as the service user.++## Shared Client LibrariesThe `openvibe-shared` package ([OpenVibers/OpenVibe.Shared](https://github.com/OpenVibers/OpenVibe.Shared), pinned in package.json) provides drop-in vanilla JS components served at `/shared/` (and `/shared/v1/`). Only the package's browser files are served; a `?v=` equal to a file's content hash is cached for a year, anything else for five minutes:@@@ -363,6 +392,8 @@ Accessible to users with `role = 'admin'`. All endpoints under `/api/admin/`.| `dev_apps`, `dev_credentials`, `dev_auth_codes` | Developer apps, hashed client secrets, app authorization codes || `dev_grants`, `dev_quotas` | App capability grants; per-project quotas (enforced by the owning service) || `dev_audit` | Append-only developer audit; rows that are platform events carry an event envelope |++| `analytics_events`, `analytics_hourly`, `analytics_daily` | Request analytics (raw ≤ 30 days) and rollups, within ADR-021; see [Analytics](#analytics-adr-021) |++| `analytics_visitor_days`, `analytics_day_salts` | The current day's salted visitor hashes and salt, deleted after the day's final rollup |---@@@ -6,7 +6,7 @@"scripts": {"start": "node server/index.js","dev": "NODE_ENV=development node server/index.js",-- "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/security-session.test.js && node test/security-redirects.test.js",++ "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/security-session.test.js && node test/security-redirects.test.js && node test/analytics-privacy.test.js","reset-db": "node server/reset-db.js","grant-admin": "node server/grant-admin.js"},@@@ -3186,8 +3186,9 @@ async function loadServiceAnalytics(serviceName) {{ icon: 'fa-circle-exclamation', label: 'Errors', value: fmtNum(s.total_errors) },{ icon: 'fa-gauge-high', label: 'Avg Response', value: (s.avg_response_ms || 0) + 'ms' },{ icon: 'fa-arrow-up-from-bracket', label: 'Est. Bandwidth', value: fmtBytes(bw.estimated_bytes) },-- { icon: 'fa-user-plus', label: 'New Visitors', value: fmtNum(vis.new_visitors) },-- ].map(s => `<div class="stat-card"><div class="value">${s.value}</div><div class="label"><i class="fa-solid ${s.icon}"></i> ${s.label}</div></div>`).join('');++ // New vs returning needs a visitor identity that outlives a day; ADR-021 analytics keeps none.++ vis.new_visitors != null ? { icon: 'fa-user-plus', label: 'New Visitors', value: fmtNum(vis.new_visitors) } : null,++ ].filter(Boolean).map(s => `<div class="stat-card"><div class="value">${s.value}</div><div class="label"><i class="fa-solid ${s.icon}"></i> ${s.label}</div></div>`).join('');// For sub-day ranges, use timeBuckets for the "daily" chartconst isSubDay = period.hours && period.hours < 24;@@@ -3322,7 +3323,9 @@ async function loadServiceAnalytics(serviceName) {<div><strong style="color:var(--info)">${fmtNum(bw.page_requests)}</strong><br><span class="muted">Page Loads</span></div><div><strong style="color:var(--info)">${fmtNum(bw.api_requests)}</strong><br><span class="muted">API Requests</span></div><div><strong style="color:var(--success)">${fmtNum(a.sessionCount || 0)}</strong><br><span class="muted">Sessions</span></div>-- <div><strong style="color:var(--success)">${fmtNum(vis.new_visitors || 0)}</strong> new / <strong>${fmtNum(vis.returning_visitors || 0)}</strong> returning<br><span class="muted">Visitors</span></div>++ ${vis.new_visitors != null++ ? `<div><strong style="color:var(--success)">${fmtNum(vis.new_visitors)}</strong> new / <strong>${fmtNum(vis.returning_visitors)}</strong> returning<br><span class="muted">Visitors</span></div>`++ : '<div><strong class="muted">Not measured</strong><br><span class="muted" title="ADR-021: no visitor identity outlives a day">New / returning</span></div>'}</div>`;// Auth vs anonymous chart@@@ -3438,30 +3441,31 @@ async function loadBotAnalysis() {const types = svcBots.botTypes || [];if (types.length) {html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Bot Types</h4>-- <table class="admin-table"><thead><tr><th>Type</th><th>Hits</th><th>Unique IPs</th></tr></thead>-- <tbody>${types.map(t => `<tr><td><span class="badge badge-danger">${esc(t.bot_type || 'unknown')}</span></td><td>${fmtNum(t.hits)}</td><td>${fmtNum(t.unique_ips)}</td></tr>`).join('')}</tbody></table>`;++ <table class="admin-table"><thead><tr><th>Type</th><th>Hits</th><th>Sessions</th></tr></thead>++ <tbody>${types.map(t => `<tr><td><span class="badge badge-danger">${esc(t.bot_type || 'unknown')}</span></td><td>${fmtNum(t.hits)}</td><td>${t.unique_sessions != null ? fmtNum(t.unique_sessions) : '-'}</td></tr>`).join('')}</tbody></table>`;}-- // Top bot IPs-- const topIPs = svcBots.topBotIPs || [];-- if (topIPs.length) {-- html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Top Bot IPs</h4>-- <table class="admin-table"><thead><tr><th>IP</th><th>Hits</th><th>Type</th><th>First Seen</th><th>Last Seen</th></tr></thead>-- <tbody>${topIPs.slice(0, 15).map(ip => `-- <tr><td style="font-family:monospace;font-size:11px">${esc(ip.ip)}</td><td>${fmtNum(ip.hits)}</td>-- <td><span class="badge badge-warning">${esc(ip.bot_type || '?')}</span></td>-- <td style="font-size:11px">${ip.first_seen ? timeAgo(ip.first_seen) : '-'}</td>-- <td style="font-size:11px">${ip.last_seen ? timeAgo(ip.last_seen) : '-'}</td></tr>++ // Top bots by user-agent class (ADR-021: analytics stores no IP addresses)++ const topBots = svcBots.topBotIPs || [];++ if (topBots.length) {++ html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Top Bots (user-agent class)</h4>++ <table class="admin-table"><thead><tr><th>User-agent class</th><th>Hits</th><th>Sessions</th><th>Type</th><th>First Seen</th><th>Last Seen</th></tr></thead>++ <tbody>${topBots.slice(0, 15).map(b => `++ <tr><td style="font-family:monospace;font-size:11px">${esc(b.ua_class || '-')}</td><td>${fmtNum(b.hits)}</td>++ <td>${b.sessions != null ? fmtNum(b.sessions) : '-'}</td>++ <td><span class="badge badge-warning">${esc(b.bot_type || '?')}</span></td>++ <td style="font-size:11px">${b.first_seen ? timeAgo(b.first_seen) : '-'}</td>++ <td style="font-size:11px">${b.last_seen ? timeAgo(b.last_seen) : '-'}</td></tr>`).join('')}</tbody></table>`;}-- // Suspicious human IPs++ // High-volume sessions classified as human (rotating session ids, never an IP or user id)const suspicious = svcBots.suspiciousIPs || [];if (suspicious.length) {-- html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Suspicious IPs (high volume, classified as human)</h4>-- <table class="admin-table"><thead><tr><th>IP</th><th>Total Hits</th><th>Errors</th><th>Unique Paths</th><th>Period</th></tr></thead>++ html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Suspicious Sessions (high volume, classified as human)</h4>++ <table class="admin-table"><thead><tr><th>Session</th><th>Total Hits</th><th>Errors</th><th>Unique Paths</th><th>Period</th></tr></thead><tbody>${suspicious.map(s => `-- <tr><td style="font-family:monospace;font-size:11px">${esc(s.ip)}</td><td>${fmtNum(s.total_hits)}</td>++ <tr><td style="font-family:monospace;font-size:11px">${esc(s.session_id || '-')}</td><td>${fmtNum(s.total_hits)}</td><td style="color:var(--danger)">${fmtNum(s.error_hits)}</td><td>${fmtNum(s.unique_paths)}</td><td style="font-size:11px">${s.first_seen ? timeAgo(s.first_seen) : '-'} — ${s.last_seen ? timeAgo(s.last_seen) : '-'}</td></tr>`).join('')}</tbody></table>`;nnew file mode 100644@@@ -0,0 +1,161 @@++#!/usr/bin/env node++/**++ * Raw analytics retention and one-time scrub for Network's analytics tables (ADR-021). They live in++ * network.db, Network's identity database, next to users and sessions.++ *++ * node scripts/analytics-prune.js # dry run: counts only, changes nothing++ * node scripts/analytics-prune.js --scrub # dry run including what the scrub would rewrite++ * node scripts/analytics-prune.js --apply --backup <file> # online backup to <file>, then prune++ * node scripts/analytics-prune.js --apply --scrub --backup <file>++ * node scripts/analytics-prune.js --apply --no-backup # prune without a backup (explicit)++ *++ * Options:++ * --db <file> database (default: $DB_PATH, else data/network.db; relative paths from the repo root)++ * --days <n> keep raw events newer than n days, 1..30 (default 30)++ * --scrub after pruning, rewrite the remaining rows: ip/user_id/city → NULL, path → route++ * template (with Network's path rules: /avatar/<name>, /users/by-username/<name>,++ * /api/auth/anon/<token>), referer → origin, user_agent → class, legacy session++ * ids → NULL; same path/referer reduction in the rollups' top lists (counts untouched)++ * --no-vacuum skip the VACUUM that follows an --apply (VACUUM rewrites the file so pruned and++ * scrubbed values do not survive in free pages; it needs ~2x the database size free).++ * VACUUM rewrites all of network.db and holds its write lock meanwhile: Network's own++ * writes (sign-ins, sessions) wait 5 s and then fail. Vacuum with the service stopped,++ * or pass --no-vacuum while it is up.++ * --batch <n> rows per write batch (default 5000)++ *++ * --apply refuses to run without --backup <file> (a new file, mode 0600 — it is a full copy of++ * network.db; sqlite online backup, verified) or an explicit --no-backup. Rollup totals are compared++ * before and after; any difference exits 1. Prune and scrub are safe while the server is up (short++ * batches; the tracker retries a busy flush). Run as the service user so no file changes owner.++ */++'use strict';++const fs = require('fs');++const path = require('path');++const Database = require('better-sqlite3');++const retention = require('../server/analytics/retention');++const networkAnalytics = require('../server/analytics/network');++++const ROOT = path.join(__dirname, '..');++const HELP_LINES = 29; // header lines 3..29++const { paramPrefixes } = networkAnalytics.PATH_OPTS;++++function parseArgs(argv) {++ const a = { apply: false, scrub: false, vacuum: true, backup: null, noBackup: false, days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, db: null };++ for (let i = 0; i < argv.length; i++) {++ const k = argv[i];++ const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };++ if (k === '--apply') a.apply = true;++ else if (k === '--scrub') a.scrub = true;++ else if (k === '--no-vacuum') a.vacuum = false;++ else if (k === '--no-backup') a.noBackup = true;++ else if (k === '--backup') a.backup = val();++ else if (k === '--days') a.days = Number(val());++ else if (k === '--batch') a.batch = Number(val());++ else if (k === '--db') a.db = val();++ else if (k === '-h' || k === '--help') a.help = true;++ else throw new Error(`unknown option ${k}`);++ }++ retention.checkDays(a.days);++ if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');++ if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');++ return a;++}++++/** The database the server uses: --db, else $DB_PATH, else data/network.db (relative to the repo root). */++function defaultDbPath(args, env = process.env) {++ return path.resolve(ROOT, args.db || env.DB_PATH || path.join('data', 'network.db'));++}++++const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;++function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }++function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }++const sameTotals = (a, b) => JSON.stringify(a) === JSON.stringify(b);++++async function main(argv, log = console.log) {++ let args;++ try { args = parseArgs(argv); } catch (e) { log(`error: ${e.message}`); return 2; }++ if (args.help) { log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, HELP_LINES).join('\n')); return 0; }++ const dbPath = defaultDbPath(args);++ if (!fs.existsSync(dbPath)) { log(`error: ${dbPath} does not exist`); return 2; }++++ if (args.apply && !args.backup && !args.noBackup) {++ log('refusing --apply without a backup: pass --backup <new file> (sqlite online backup) or --no-backup');++ return 2;++ }++++ const db = new Database(dbPath, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });++ try {++ db.pragma('busy_timeout = 5000');++ const walBytes = fileBytes(dbPath + '-wal');++ const before = retention.inspect(db, { days: args.days, paramPrefixes });++ log(`database ${dbPath}`);++ log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);++ log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);++ log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);++ if (before.toScrub) {++ const t = before.toScrub;++ log(`scrub of the rows that stay: ${t.personal} with ip/user_id/city, ${t.paths} paths to template, ${t.referers} referers to origin, ${t.user_agents} user agents to class, ${t.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);++ }++ log(`rate rows ${before.rateRows || 0} (IP counters; the server keeps them in memory now)`);++ for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);++++ if (!args.apply) {++ log('\ndry run: nothing changed. Re-run with --apply --backup <file> (or --no-backup).');++ return 0;++ }++++ if (args.backup) {++ const target = path.resolve(args.backup);++ if (fs.existsSync(target)) { log(`error: backup target ${target} already exists; choose a new file`); return 2; }++ const need = before.bytes + walBytes;++ const free = freeBytes(path.dirname(target));++ if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(target)}, the backup needs about ${mb(need)}`); return 2; }++ // A full copy of network.db (accounts, sessions): owner-only from the first byte.++ const umask = process.umask(0o077);++ try { await db.backup(target); } finally { process.umask(umask); }++ fs.chmodSync(target, 0o600);++ const b = new Database(target, { readonly: true });++ try {++ const ok = b.pragma('quick_check', { simple: true });++ const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()++ ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;++ if (ok !== 'ok' || n < before.events) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }++ } finally { b.close(); }++ log(`backup ${target} (${mb(fileBytes(target))}, mode 0600, quick_check ok)`);++ }++++ db.pragma('secure_delete = ON');++ const totalsBefore = retention.rollupTotals(db);++ const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });++ log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);++ if (args.scrub) {++ await networkAnalytics.prescrub(db, { batchSize: args.batch });++ const s = await retention.scrubEvents(db, { batchSize: args.batch, paramPrefixes });++ const r = retention.scrubRollups(db, { paramPrefixes });++ log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);++ }++ const totalsAfter = retention.rollupTotals(db);++ if (!sameTotals(totalsBefore, totalsAfter)) {++ log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);++ return 1;++ }++ log('rollups totals unchanged');++ if (args.vacuum && (pruned.deleted || args.scrub)) {++ db.pragma('wal_checkpoint(TRUNCATE)');++ db.exec('VACUUM');++ db.pragma('wal_checkpoint(TRUNCATE)');++ log(`vacuumed ${mb(fileBytes(dbPath))}`);++ }++ const after = retention.inspect(db, { days: args.days, paramPrefixes });++ log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);++ return 0;++ } finally {++ db.close();++ }++}++++if (require.main === module) {++ main(process.argv.slice(2)).then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });++}++++module.exports = { main, parseArgs, defaultDbPath };@@@ -287,10 +287,11 @@ module.exports = function createAnalyticsRoutes(analytics, requireAuth, config)const fiveMinAgo = new Date(Date.now() - 300000).toISOString().slice(0, 19).replace('T', ' ');const db = analytics.db;++ // ADR-021: raw events carry no IP; visitors are distinct rotating session ids.const realtime = db.prepare(`SELECTCOUNT(*) as requests,-- COUNT(DISTINCT ip) as visitors,++ COUNT(DISTINCT session_id) as visitors,COUNT(*) FILTER (WHERE is_bot = 1) as bots,COUNT(*) FILTER (WHERE event_type = 'pageview') as pageviews,COUNT(*) FILTER (WHERE event_type = 'api_call') as api_callsnnew file mode 100644@@@ -0,0 +1,8 @@++'use strict';++// Request analytics within ADR-021 (see tracker.js, privacy.js, retention.js).++const { AnalyticsTracker } = require('./tracker');++const privacy = require('./privacy');++const retention = require('./retention');++const { ensureSchema } = require('./schema');++++module.exports = { AnalyticsTracker, privacy, retention, ensureSchema };nnew file mode 100644@@@ -0,0 +1,114 @@++'use strict';++/**++ * OpenVibe.Network's wiring of the ADR-021 analytics module. The other files in this directory are++ * the module shared with OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);++ * keep them identical to those copies and put anything Network-specific here.++ *++ * openAnalytics(dbPath) the tracker for service 'openvibe-network' (NetworkAnalyticsTracker).++ * Network's analytics tables (analytics_events / _hourly / _daily …) live++ * in network.db, as they always++ * have; the tracker gets a better-sqlite3 connection of its own so its++ * per-connection settings (busy_timeout 250, secure_delete) never apply to++ * the identity connection (busy_timeout 5000).++ * schedulePrune(tracker) the nightly `analytics-prune` job: raw events older than 30 days are++ * deleted in bounded batches (retention.pruneRawEvents); rollups are kept.++ * First run 5 minutes after boot, then every 24 hours; timers are unref'd.++ * prescrub(db) Network's part of the one-time scrub (scripts/analytics-prune.js --scrub):++ * the path rules below applied to legacy rows and rollup top lists, before++ * the shared scrub templates them.++ * PATH_OPTS the normaliser options (param prefixes) the CLI passes to the shared scrub.++ */++++const path = require('path');++const Database = require('better-sqlite3');++const { AnalyticsTracker } = require('./tracker');++const privacy = require('./privacy');++const retention = require('./retention');++++const SERVICE = 'openvibe-network';++++/**++ * Words whose next path segment is a parameter on Network (on top of privacy.DEFAULT_PARAM_PREFIXES).++ * They matter when no Express route matched (a rate-limited or rejected request is templated from++ * its raw path), and to the one-time scrub of rows the old tracker stored raw: /avatar/<username>,++ * /api/auth/anon/<token>, /api/v1/projects/<slug>.++ */++const PARAM_PREFIXES = ['avatar', 'anon', 'projects'];++const PATH_OPTS = { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...PARAM_PREFIXES]) };++++/**++ * A parameter behind two fixed words, which the shared normaliser cannot see: it turns `users` into a++ * prefix, so `/users/by-username/alex` would keep `alex`. Applied to the raw path before it.++ */++const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];++function preReducePath(p) {++ let s = String(p);++ for (const [re, to] of PATH_RULES) s = s.replace(re, to);++ return s;++}++++class NetworkAnalyticsTracker extends AnalyticsTracker {++ /** The raw path is only used when no Express route matched (a request refused before routing). */++ record(req, res, entryPath, responseTimeMs) {++ return super.record(req, res, preReducePath(entryPath || ''), responseTimeMs);++ }++}++++function openAnalytics(dbPath, opts = {}) {++ const db = new Database(path.resolve(dbPath));++ db.pragma('journal_mode = WAL');++ return new NetworkAnalyticsTracker(db, SERVICE, { retention: false, paramPrefixes: PARAM_PREFIXES, ...opts });++}++++const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);++const tick = () => new Promise((r) => setImmediate(r));++++/** PATH_RULES over legacy raw rows (id-range batches) and rollup top lists. Returns { rows, hourly, daily }. */++async function prescrub(db, { batchSize = retention.DEFAULT_BATCH } = {}) {++ const out = { rows: 0, hourly: 0, daily: 0 };++ if (!tableExists(db, 'analytics_events')) return out;++ db.function('ov_network_path', { deterministic: true }, (p) => (p == null ? null : preReducePath(p)));++ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();++ const upd = db.prepare("UPDATE analytics_events SET path = ov_network_path(path) WHERE id >= ? AND id < ? AND path LIKE '%/by-username/%' AND path IS NOT ov_network_path(path)");++ for (let from = lo; lo != null && from <= hi; from += batchSize) {++ out.rows += upd.run(from, from + batchSize).changes;++ await tick();++ }++ for (const table of ['analytics_hourly', 'analytics_daily']) {++ if (!tableExists(db, table)) continue;++ const rows = db.prepare(`SELECT id, top_paths FROM ${table} WHERE top_paths LIKE '%/by-username/%'`).all();++ const set = db.prepare(`UPDATE ${table} SET top_paths = ? WHERE id = ?`);++ db.transaction(() => {++ for (const r of rows) {++ const next = retention.reduceTopList(r.top_paths, 'path', preReducePath);++ if (next != null) { set.run(next, r.id); out[table === 'analytics_hourly' ? 'hourly' : 'daily']++; }++ }++ })();++ }++ return out;++}++++function schedulePrune(tracker, { initialDelayMs = 5 * 60 * 1000, intervalMs = 24 * 60 * 60 * 1000, log = console.log } = {}) {++ let running = false;++ const run = async () => {++ if (running) return null;++ running = true;++ try {++ const out = await retention.pruneRawEvents(tracker.db, { days: retention.MAX_DAYS });++ if (out.deleted) log(`[Analytics] pruned ${out.deleted} raw events older than ${out.cutoff}`);++ return out;++ } catch (err) {++ console.error('[Analytics] analytics-prune error:', err.message);++ return null;++ } finally {++ running = false;++ }++ };++ const first = setTimeout(run, initialDelayMs);++ const every = setInterval(run, intervalMs);++ first.unref();++ every.unref();++ return { run, stop() { clearTimeout(first); clearInterval(every); } };++}++++module.exports = { SERVICE, PARAM_PREFIXES, PATH_OPTS, NetworkAnalyticsTracker, preReducePath, openAnalytics, schedulePrune, prescrub };nnew file mode 100644@@@ -0,0 +1,211 @@++'use strict';++/**++ * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):++ * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a++ * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a++ * precise location or a user/subject id.++ *++ * This file holds the pure reducers that turn a request into those fields. It has no dependencies,++ * so the same file serves OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);++ * keep the two copies identical.++ */++++// ── Bots ──────────────────────────────────────────────────────++++const BOT_USER_AGENTS = [++ /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,++ /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,++ /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,++ /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,++ /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,++ /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,++ /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,++ /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,++ /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,++ /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,++ /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,++ /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,++ /statuscake/i, /monitoring/i, /health.?check/i,++];++++const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];++++/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */++const HIGH_REQUEST_RATE = 60;++++/** The family name of the first bot pattern the user agent matches, or null. */++function botName(ua) {++ if (!ua) return null;++ for (const re of BOT_USER_AGENTS) {++ if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';++ }++ return null;++}++++/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */++function classifyRequest(req) {++ const ua = (req.headers && req.headers['user-agent']) || '';++ if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };++ if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };++ const path = String(req.path || req.url || '').toLowerCase();++ if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };++ return { isBot: false, botType: null, confidence: 0.1 };++}++++// ── User agent → class ───────────────────────────────────────++++function parseUserAgent(ua) {++ if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };++ let device = 'desktop';++ if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';++ else if (/ipad|tablet/i.test(ua)) device = 'tablet';++++ let browser = 'other';++ if (/edg\//i.test(ua)) browser = 'edge';++ else if (/opr\/|opera/i.test(ua)) browser = 'opera';++ else if (/firefox\//i.test(ua)) browser = 'firefox';++ else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';++ else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';++ else if (/trident|msie/i.test(ua)) browser = 'ie';++++ let os = 'other';++ if (/windows/i.test(ua)) os = 'windows';++ else if (/macintosh|mac os/i.test(ua)) os = 'macos';++ else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';++ else if (/android/i.test(ua)) os = 'android';++ else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';++++ return { device, browser, os };++}++++const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;++++/**++ * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,++ * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps++ * to itself), so a scrub can run over rows that are already reduced.++ */++function uaClass(ua) {++ if (ua == null || ua === '') return 'none';++ const s = String(ua);++ if (UA_CLASS_RE.test(s)) return s;++ const bot = botName(s);++ if (bot) return `bot:${bot}`;++ const p = parseUserAgent(s);++ return `${p.browser}/${p.os}/${p.device}`;++}++++// ── Referer → origin ─────────────────────────────────────────++++/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */++function refererOrigin(ref) {++ if (!ref) return null;++ try {++ const u = new URL(String(ref));++ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;++ return u.origin;++ } catch {++ return null;++ }++}++++// ── Country ──────────────────────────────────────────────────++++/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */++function countryCode(v) {++ const s = String(v || '').trim().toUpperCase();++ return /^[A-Z]{2}$/.test(s) ? s : null;++}++++// ── Paths → route templates ──────────────────────────────────++++/**++ * Words after which the next path segment is a parameter even when it looks like a word++ * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.++ */++const DEFAULT_PARAM_PREFIXES = [++ 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',++ 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',++ 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',++ 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',++ 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',++];++++const MAX_SEGMENTS = 8;++const MAX_TEMPLATE_LENGTH = 200;++const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;++const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;++++/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */++function looksLikeId(s) {++ if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…++ if (/^\d+$/.test(s)) return true; // 123++ if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid++ if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes++ if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…++ if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid++ if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers++ if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names++ if (s.length > 32) return true;++ return false;++}++++/**++ * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by++ * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated++ * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.++ */++function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES } = {}) {++ if (raw == null || raw === '') return '/';++ let p = String(raw);++ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {++ try { p = new URL(p).pathname; } catch { return '/'; }++ }++ p = p.split(/[?#]/)[0];++ const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);++ const segs = p.split('/').filter(Boolean);++ const out = [];++ let expectParam = false;++ for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {++ let s = segs[i];++ try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }++ if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }++ if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }++ if (expectParam) { out.push(':param'); expectParam = false; continue; }++ if (looksLikeId(s)) { out.push(':id'); continue; }++ out.push(s);++ expectParam = prefixes.has(s.toLowerCase());++ }++ if (segs.length > MAX_SEGMENTS) out.push('*');++ const t = '/' + out.join('/');++ return t.length > MAX_TEMPLATE_LENGTH ? t.slice(0, MAX_TEMPLATE_LENGTH) : t;++}++++/**++ * The route template for a finished request: the matched Express route (mount path + route path)++ * when there is one, else the normalised original path. Both go through normalisePath, because a++ * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.++ */++function routeTemplate(req, fallbackPath, opts) {++ const r = req && req.route;++ if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {++ return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);++ }++ return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);++}++++module.exports = {++ BOT_USER_AGENTS,++ HONEYPOT_PATHS,++ HIGH_REQUEST_RATE,++ DEFAULT_PARAM_PREFIXES,++ botName,++ classifyRequest,++ parseUserAgent,++ uaClass,++ refererOrigin,++ countryCode,++ looksLikeId,++ normalisePath,++ routeTemplate,++};nnew file mode 100644@@@ -0,0 +1,209 @@++'use strict';++/**++ * Raw analytics retention and the one-time scrub (ADR-021).++ *++ * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded++ * batches, yielding to the event loop between batches. Rollups++ * (analytics_hourly / analytics_daily) are never touched. Also clears++ * leftovers: visitor hashes older than yesterday, salts of past days,++ * and analytics_rate_tracking (IP counters live in memory now).++ * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →++ * NULL (authenticated = 1 where a user id was present), path → route++ * template, referer → origin, user_agent → class, session ids that are++ * not rotating ids → NULL.++ * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /++ * top_referers JSON (counts merged, never changed).++ * inspect(db, { days }) read-only counts for a dry run.++ * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.++ *++ * Used by the server (scheduled prune) and scripts/analytics-prune.js (CLI).++ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).++ */++++const { ensureSchema } = require('./schema');++const privacy = require('./privacy');++++const MAX_DAYS = 30;++const DEFAULT_BATCH = 5000;++const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;++++const pad = (n) => String(n).padStart(2, '0');++function sqlTime(ms) {++ const d = new Date(ms);++ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;++}++++function checkDays(days) {++ const n = Number(days);++ if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {++ throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);++ }++ return n;++}++++/** The created_at bound: rows strictly older than this are pruned. */++function cutoffFor(days, nowMs = Date.now()) {++ return sqlTime(nowMs - checkDays(days) * 86400000);++}++++const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);++const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);++const tick = () => new Promise((r) => setImmediate(r));++++/**++ * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write++ * transaction). Returns { cutoff, deleted, batches, complete }.++ */++async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {++ const cutoff = cutoffFor(days, now());++ const out = { cutoff, deleted: 0, batches: 0, complete: true };++ if (!tableExists(db, 'analytics_events')) return out;++ const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');++ for (;;) {++ if (out.batches >= maxBatches) { out.complete = false; break; }++ const n = del.run(cutoff, batchSize).changes;++ out.batches++;++ out.deleted += n;++ if (n < batchSize) break;++ await tick();++ }++ const today = sqlTime(now()).slice(0, 10);++ const yesterday = sqlTime(now() - 86400000).slice(0, 10);++ if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);++ if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);++ if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();++ return out;++}++++/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */++function registerFunctions(db, opts) {++ db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));++ db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));++ db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));++ db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));++}++++/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */++async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes } = {}) {++ ensureSchema(db);++ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);++ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();++ const out = { rows: 0, batches: 0 };++ if (lo == null) return out;++ const upd = db.prepare(`++ UPDATE analytics_events SET++ authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,++ ip = NULL, user_id = NULL, city = NULL,++ session_id = ov_rotating_session(session_id),++ path = ov_route_template(path),++ referer = ov_referer_origin(referer),++ user_agent = ov_ua_class(user_agent)++ WHERE id >= ? AND id < ?++ `);++ for (let from = lo; from <= hi; from += batchSize) {++ out.rows += upd.run(from, from + batchSize).changes;++ out.batches++;++ await tick();++ }++ return out;++}++++/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */++function reduceTopList(json, key, fn) {++ let list;++ try { list = JSON.parse(json); } catch { return null; }++ if (!Array.isArray(list)) return null;++ const merged = new Map();++ for (const item of list) {++ if (!item || typeof item !== 'object') continue;++ const k = fn(item[key]);++ if (k == null) continue;++ merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));++ }++ const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);++ const text = JSON.stringify(next);++ return text === json ? null : text;++}++++/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */++function scrubRollups(db, { paramPrefixes } = {}) {++ const opts = paramPrefixes ? { paramPrefixes } : undefined;++ const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));++ const out = { hourly: 0, daily: 0 };++ for (const table of ['analytics_hourly', 'analytics_daily']) {++ if (!tableExists(db, table)) continue;++ const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();++ const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);++ db.transaction(() => {++ for (const r of rows) {++ const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;++ const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;++ if (paths == null && refs == null) continue;++ upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);++ out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;++ }++ })();++ }++ return out;++}++++/** Sums of every rollup counter; equal before and after a prune or scrub. */++function rollupTotals(db) {++ const out = {};++ if (tableExists(db, 'analytics_hourly')) {++ out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,++ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,++ TOTAL(error_count) AS error_count FROM analytics_hourly`).get();++ }++ if (tableExists(db, 'analytics_daily')) {++ out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,++ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,++ TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();++ }++ return out;++}++++/** Read-only counts for a dry run. */++function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes } = {}) {++ const cutoff = cutoffFor(days, now());++ const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };++ const pageSize = db.pragma('page_size', { simple: true });++ out.bytes = db.pragma('page_count', { simple: true }) * pageSize;++ out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;++ out.rollups = rollupTotals(db);++ if (!tableExists(db, 'analytics_events')) return out;++ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);++ const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();++ out.events = e.n;++ out.oldest = e.oldest;++ out.newest = e.newest;++ out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);++ out.remaining = out.events - out.older;++ const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))++ .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';++ const r = db.prepare(`++ SELECT++ COUNT(*) FILTER (WHERE ${personal}) AS personal,++ COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,++ COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,++ COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,++ COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions++ FROM analytics_events WHERE created_at >= ?++ `).get(cutoff);++ out.toScrub = r;++ out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;++ return out;++}++++module.exports = {++ MAX_DAYS,++ DEFAULT_BATCH,++ checkDays,++ cutoffFor,++ pruneRawEvents,++ scrubEvents,++ scrubRollups,++ rollupTotals,++ inspect,++ reduceTopList,++};nnew file mode 100644@@@ -0,0 +1,123 @@++'use strict';++/**++ * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the++ * existing analytics.db files and the admin dashboards keep working; the columns that used to hold++ * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and++ * user_agent / referer / path hold the reduced forms (class, origin, route template).++ *++ * Added:++ * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)++ * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;++ * rows are deleted once the day's rollup is final (≤ ~25 h)++ * analytics_day_salts the random salt of the current UTC day (so a restart does not++ * double-count); deleted together with that day's hashes++ * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)++ */++++const ANALYTICS_SCHEMA = `++ CREATE TABLE IF NOT EXISTS analytics_events (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ event_type TEXT NOT NULL DEFAULT 'pageview',++ path TEXT,++ method TEXT DEFAULT 'GET',++ status_code INTEGER,++ response_time_ms INTEGER,++ user_id INTEGER,++ session_id TEXT,++ ip TEXT,++ country TEXT,++ city TEXT,++ user_agent TEXT,++ referer TEXT,++ is_bot INTEGER DEFAULT 0,++ bot_type TEXT,++ device_type TEXT,++ browser TEXT,++ os TEXT,++ created_at DATETIME DEFAULT CURRENT_TIMESTAMP++ );++++ CREATE TABLE IF NOT EXISTS analytics_hourly (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ hour TEXT NOT NULL,++ pageviews INTEGER DEFAULT 0,++ api_calls INTEGER DEFAULT 0,++ unique_visitors INTEGER DEFAULT 0,++ unique_users INTEGER DEFAULT 0,++ bot_hits INTEGER DEFAULT 0,++ avg_response_ms INTEGER DEFAULT 0,++ error_count INTEGER DEFAULT 0,++ bandwidth_bytes INTEGER DEFAULT 0,++ top_paths TEXT,++ top_referers TEXT,++ UNIQUE(service, hour)++ );++++ CREATE TABLE IF NOT EXISTS analytics_daily (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ date TEXT NOT NULL,++ pageviews INTEGER DEFAULT 0,++ api_calls INTEGER DEFAULT 0,++ unique_visitors INTEGER DEFAULT 0,++ unique_users INTEGER DEFAULT 0,++ new_users INTEGER DEFAULT 0,++ bot_hits INTEGER DEFAULT 0,++ avg_response_ms INTEGER DEFAULT 0,++ error_count INTEGER DEFAULT 0,++ top_paths TEXT,++ top_referers TEXT,++ top_countries TEXT,++ device_breakdown TEXT,++ browser_breakdown TEXT,++ UNIQUE(service, date)++ );++++ CREATE TABLE IF NOT EXISTS analytics_rate_tracking (++ ip TEXT NOT NULL,++ window_start INTEGER NOT NULL,++ hit_count INTEGER DEFAULT 1,++ PRIMARY KEY (ip, window_start)++ );++++ CREATE TABLE IF NOT EXISTS analytics_visitor_days (++ service TEXT NOT NULL,++ day TEXT NOT NULL,++ hour TEXT NOT NULL,++ vhash TEXT NOT NULL,++ authenticated INTEGER NOT NULL DEFAULT 0,++ PRIMARY KEY (service, hour, vhash)++ );++++ CREATE TABLE IF NOT EXISTS analytics_day_salts (++ service TEXT NOT NULL,++ day TEXT NOT NULL,++ salt TEXT NOT NULL,++ PRIMARY KEY (service, day)++ );++++ CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);++ CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);++ CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);++++ -- Indexes over the retired personal columns.++ DROP INDEX IF EXISTS idx_analytics_events_ip;++ DROP INDEX IF EXISTS idx_analytics_events_user;++`;++++/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */++function ensureSchema(db) {++ db.exec(ANALYTICS_SCHEMA);++ const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);++ if (!cols.includes('authenticated')) {++ db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');++ }++}++++module.exports = { ANALYTICS_SCHEMA, ensureSchema };nnew file mode 100644@@@ -0,0 +1,728 @@++'use strict';++/**++ * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the openvibe-shared++ * AnalyticsTracker (same tables, same getStats/getOverview/getBotAnalysis shapes), minus the personal++ * data:++ *++ * raw row service, event_type, route template, method, status, response time, rotating session++ * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,++ * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.++ * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after++ * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.++ * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped++ * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).++ * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.++ * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as++ * the day is over); hashes go to analytics_visitor_days only, never to raw events,++ * and are deleted by the first aggregation after their day ends, right after that day's++ * final hourly/daily rollup — so no hash or salt outlives its day by more than about++ * one aggregation interval (1 h). Rollups keep only the counts.++ *++ * Retention of raw events (30 days) is ./retention.js; this tracker only writes and rolls up.++ *++ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).++ */++++const crypto = require('crypto');++const { ensureSchema } = require('./schema');++const privacy = require('./privacy');++const retention = require('./retention');++++const SESSION_IDLE_MS = 30 * 60 * 1000;++const MAX_TRACKED_CLIENTS = 200000;++const MAX_BUFFER = 5000;++const SESSION_ID_RE = /^[0-9a-f]{16}$/;++++const pad = (n) => String(n).padStart(2, '0');++/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */++function sqlTime(ms) {++ const d = new Date(ms);++ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;++}++const dayOf = (ms) => sqlTime(ms).slice(0, 10);++const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';++const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');++++function isStaticOrHealth(path) {++ return path === '/api/health' || path === '/health' ||++ /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);++}++++class AnalyticsTracker {++ /**++ * @param {object} db better-sqlite3 handle (its own analytics database)++ * @param {string} service service name ('live', 'openvibe-yt', …)++ * @param {object} [opts]++ * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter++ * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:++ * run the raw-event prune on a timer (Tools). Live passes false and schedules++ * retention.pruneRawEvents through server/utils/jobs.js instead.++ * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)++ * @param {function} [opts.now] clock (tests)++ */++ constructor(db, service, opts = {}) {++ this.db = db;++ this.service = service;++ this._now = opts.now || Date.now;++ this._pathOpts = opts.paramPrefixes++ ? { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...opts.paramPrefixes]) }++ : undefined;++ this._buffer = [];++ this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }++ this._sessions = new Map(); // vhash -> { sid, last }++ this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)++ this._saltDay = null;++ this._salt = null;++ this._timers = [];++++ ensureSchema(db);++ try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }++ try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }++ db.exec('DELETE FROM analytics_rate_tracking');++++ this._insertEvent = db.prepare(`++ INSERT INTO analytics_events++ (service, event_type, path, method, status_code, response_time_ms,++ user_id, session_id, ip, country, city, user_agent, referer,++ is_bot, bot_type, device_type, browser, os, authenticated, created_at)++ VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ `);++ this._upsertVisitor = db.prepare(`++ INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)++ ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)++ `);++ this._write = db.transaction((events, visitors) => {++ for (const e of events) {++ this._insertEvent.run(++ e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,++ e.session_id, e.country, e.user_agent, e.referer,++ e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,++ );++ }++ for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);++ });++++ if (opts.timers !== false) {++ const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };++ const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };++ every(() => { this.flush(); this._sweep(); }, 5000);++ every(() => this.aggregate(), 60 * 60 * 1000);++ once(() => this.aggregate(), 10_000);++ if (opts.retention) {++ const r = opts.retention;++ const run = () => retention.pruneRawEvents(db, { days: r.days || retention.MAX_DAYS })++ .then((out) => { if (out.deleted) console.log(`[Analytics:${service}] pruned ${out.deleted} raw events older than ${out.cutoff}`); })++ .catch((err) => console.error(`[Analytics:${service}] prune error:`, err.message));++ once(run, r.initialDelayMs != null ? r.initialDelayMs : 5 * 60 * 1000);++ every(run, r.intervalMs || 24 * 60 * 60 * 1000);++ }++ }++ }++++ // ── Per-request reducers ──────────────────────────────────++++ _daySalt(day) {++ if (this._saltDay === day) return this._salt;++ let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);++ if (!row) {++ this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')++ .run(this.service, day, crypto.randomBytes(32).toString('hex'));++ row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);++ }++ // Earlier days' salts are never needed again: their hashes are already written.++ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);++ this._saltDay = day;++ this._salt = row.salt;++ this._sessions.clear();++ return this._salt;++ }++++ _visitorHash(ip, ua, day) {++ return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);++ }++++ _sessionId(vhash, nowMs) {++ let s = this._sessions.get(vhash);++ if (!s || nowMs - s.last > SESSION_IDLE_MS) {++ s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };++ this._sessions.set(vhash, s);++ }++ s.last = nowMs;++ return s.sid;++ }++++ /** Hits from this client in the current and previous minute (memory only). */++ _rateHit(ip, nowMs) {++ const minute = Math.floor(nowMs / 60000);++ let r = this._rates.get(ip);++ if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }++ if (r.minute !== minute) {++ r.prev = r.minute === minute - 1 ? r.count : 0;++ r.count = 0;++ r.minute = minute;++ }++ r.count++;++ return r.count + r.prev;++ }++++ _sweep() {++ const nowMs = this._now();++ const minute = Math.floor(nowMs / 60000);++ for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);++ for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);++ if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();++ if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();++ }++++ _push(event, visitor) {++ if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();++ this._buffer.push(event);++ if (visitor) {++ const key = `${visitor.hour}|${visitor.vhash}`;++ const prev = this._visitors.get(key);++ if (!prev) this._visitors.set(key, visitor);++ else if (visitor.authenticated) prev.authenticated = true;++ }++ if (this._buffer.length >= 100) this.flush();++ }++++ /**++ * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted++ * router is relative by the time the response finishes).++ */++ record(req, res, entryPath, responseTimeMs) {++ const nowMs = this._now();++ const headers = req.headers || {};++ const ua = String(headers['user-agent'] || '');++ const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');++ const bot = privacy.classifyRequest({ headers, path: entryPath });++ if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {++ bot.isBot = true;++ bot.botType = 'rate_limit';++ bot.confidence = 0.80;++ }++ const parsed = privacy.parseUserAgent(ua);++ const day = dayOf(nowMs);++ const vhash = this._visitorHash(ip, ua, day);++ const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));++ const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');++ this._push({++ service: this.service,++ event_type: isApi ? 'api_call' : 'pageview',++ path: privacy.routeTemplate(req, entryPath, this._pathOpts),++ method: req.method,++ status_code: res.statusCode,++ response_time_ms: responseTimeMs,++ session_id: this._sessionId(vhash, nowMs),++ country: privacy.countryCode(headers['cf-ipcountry']),++ user_agent: privacy.uaClass(ua),++ referer: privacy.refererOrigin(headers.referer || headers.referrer),++ is_bot: bot.isBot,++ bot_type: bot.botType,++ device_type: parsed.device,++ browser: parsed.browser,++ os: parsed.os,++ authenticated,++ created_at: sqlTime(nowMs),++ }, { day, hour: hourOf(nowMs), vhash, authenticated });++ }++++ /** Express middleware: one row per finished request (health checks and static assets skipped). */++ middleware() {++ return (req, res, next) => {++ const entryPath = req.path || '';++ if (isStaticOrHealth(entryPath)) return next();++ const started = Date.now();++ let done = false;++ const finish = () => {++ if (done) return;++ done = true;++ try { this.record(req, res, entryPath, Date.now() - started); }++ catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }++ };++ res.once('finish', finish);++ res.once('close', finish);++ next();++ };++ }++++ /**++ * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,++ * referer and user_agent are reduced like a request's; session_id is kept only if it is one of++ * this tracker's rotating ids.++ */++ trackEvent(eventType, data = {}) {++ const nowMs = this._now();++ const ua = data.user_agent ? String(data.user_agent) : '';++ const parsed = ua ? privacy.parseUserAgent(ua) : {};++ this._push({++ service: this.service,++ event_type: String(eventType || 'event').slice(0, 64),++ path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,++ method: data.method || null,++ status_code: data.status_code || null,++ response_time_ms: data.response_time_ms || null,++ session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,++ country: privacy.countryCode(data.country),++ user_agent: ua ? privacy.uaClass(ua) : null,++ referer: privacy.refererOrigin(data.referer),++ is_bot: !!data.is_bot,++ bot_type: data.bot_type || null,++ device_type: data.device_type || parsed.device || null,++ browser: data.browser || parsed.browser || null,++ os: data.os || parsed.os || null,++ authenticated: !!data.authenticated,++ created_at: sqlTime(nowMs),++ }, null);++ }++++ /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */++ flush() {++ if (!this._buffer.length && !this._visitors.size) return;++ const events = this._buffer.splice(0);++ const visitors = [...this._visitors.values()];++ this._visitors.clear();++ try {++ this._write(events, visitors);++ } catch (err) {++ console.error(`[Analytics:${this.service}] Flush error:`, err.message);++ this._buffer = events.slice(-MAX_BUFFER);++ for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);++ }++ }++++ // ── Rollups ───────────────────────────────────────────────++++ /**++ * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has++ * visitor hashes (finalised here), after which those hashes are deleted.++ */++ aggregate() {++ try {++ this.flush();++ const nowMs = this._now();++ const today = dayOf(nowMs);++ const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);++ const days = new Set([today, dayOf(nowMs - 86400000)]);++ const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);++ for (const p of pending) { hours.add(p.hour); days.add(p.day); }++ for (const h of hours) this._aggregateHour(h);++ for (const d of days) this._aggregateDay(d);++ this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);++ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);++ } catch (err) {++ console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);++ }++ }++++ _aggregateHour(hourStart) {++ const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);++ const svc = this.service;++ const stats = this.db.prepare(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count++ FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ?++ `).get(svc, hourStart, nextHour);++ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;++++ // Uniques only grow within a period, so never lower a stored count (restarts, or rows++ // counted before this tracker when the old one still stored IPs).++ const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);++ const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};++++ const topPaths = this.db.prepare(`++ SELECT path, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0++ GROUP BY path ORDER BY cnt DESC LIMIT 10++ `).all(svc, hourStart, nextHour);++ const topReferers = this.db.prepare(`++ SELECT referer, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0++ GROUP BY referer ORDER BY cnt DESC LIMIT 10++ `).all(svc, hourStart, nextHour);++++ this.db.prepare(`++ INSERT INTO analytics_hourly++ (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)++ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ ON CONFLICT(service, hour) DO UPDATE SET++ pageviews = excluded.pageviews, api_calls = excluded.api_calls,++ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,++ bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,++ error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers++ `).run(++ svc, hourStart, stats.pageviews, stats.api_calls,++ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),++ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,++ JSON.stringify(topPaths), JSON.stringify(topReferers),++ );++ }++++ _aggregateDay(date) {++ const dayStart = date + ' 00:00:00';++ const dayEnd = date + ' 23:59:59';++ const svc = this.service;++ const q = (sql) => this.db.prepare(sql);++ const stats = q(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count++ FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at <= ?++ `).get(svc, dayStart, dayEnd);++ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;++++ const v = q(`++ SELECT COUNT(DISTINCT vhash) AS visitors,++ COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users++ FROM analytics_visitor_days WHERE service = ? AND day = ?++ `).get(svc, date);++ const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};++++ const range = [svc, dayStart, dayEnd];++ const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);++ const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);++ const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);++ const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);++ const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);++++ q(`++ INSERT INTO analytics_daily++ (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,++ avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)++ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ ON CONFLICT(service, date) DO UPDATE SET++ pageviews = excluded.pageviews, api_calls = excluded.api_calls,++ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,++ new_users = excluded.new_users, bot_hits = excluded.bot_hits,++ avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,++ top_paths = excluded.top_paths, top_referers = excluded.top_referers,++ top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,++ browser_breakdown = excluded.browser_breakdown++ `).run(++ svc, date, stats.pageviews, stats.api_calls,++ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),++ // New users needs a user id history, which analytics no longer has: keep what was++ // recorded before, NULL (not measured) from now on.++ prev.new_users != null ? prev.new_users : null,++ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,++ JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),++ JSON.stringify(devices), JSON.stringify(browsers),++ );++ }++++ // ── Reads (admin dashboards) ─────────────────────────────++ // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,++ // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based++ // numbers (daily/hourly unique_visitors) are true unique visitors per period.++++ getStats(options = {}) {++ const { days = 30, hours, service } = options;++ const svc = service || this.service;++ const nowMs = this._now();++ const rangeMs = hours ? hours * 3600000 : days * 86400000;++ const rangeCutoff = sqlTime(nowMs - rangeMs);++ const isSubDay = hours && hours < 24;++ const effectiveDays = hours ? hours / 24 : days;++ const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;++ const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;++ const q = (sql) => this.db.prepare(sql);++ const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };++++ let daily = [];++ if (!isSubDay) {++ daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));++ }++++ const hourlyLimit = hours ? Math.min(hours, 168) : 24;++ const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);++ const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);++++ let timeBuckets = [];++ if (isSubDay) {++ timeBuckets = q(`++ SELECT ${bucketExpr} as bucket,++ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms++ FROM analytics_events++ WHERE service = ? AND created_at >= ?++ GROUP BY bucket ORDER BY bucket ASC++ `).all(svc, rangeCutoff);++ }++++ let summary;++ if (hours && hours < 48) {++ summary = q(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,++ COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms++ FROM analytics_events++ WHERE service = ? AND created_at >= ?++ `).get(svc, rangeCutoff);++ } else {++ summary = q(`++ SELECT++ SUM(pageviews) as total_pageviews,++ SUM(api_calls) as total_api_calls,++ SUM(unique_visitors) as total_unique_visitors,++ SUM(unique_users) as total_unique_users,++ SUM(bot_hits) as total_bot_hits,++ SUM(error_count) as total_errors,++ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms++ FROM analytics_daily++ WHERE service = ? AND date >= ?++ `).get(svc, dayOf(nowMs - rangeMs));++ }++++ const realtime = q(`++ SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots++ FROM analytics_events WHERE service = ? AND created_at >= ?++ `).get(svc, sqlTime(nowMs - 300000));++++ const human = [svc, rangeCutoff];++ const topPages = q(`++ SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY path ORDER BY hits DESC LIMIT 20++ `).all(...human);++ const topReferers = q(`++ SELECT referer, COUNT(*) as hits FROM analytics_events++ WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0++ GROUP BY referer ORDER BY hits DESC LIMIT 15++ `).all(...human);++ const botBreakdown = q(`++ SELECT bot_type, COUNT(*) as hits FROM analytics_events++ WHERE service = ? AND created_at >= ? AND is_bot = 1++ GROUP BY bot_type ORDER BY hits DESC LIMIT 15++ `).all(...human);++ const statusCodes = q(`++ SELECT CASE++ WHEN status_code >= 200 AND status_code < 300 THEN '2xx'++ WHEN status_code >= 300 AND status_code < 400 THEN '3xx'++ WHEN status_code >= 400 AND status_code < 500 THEN '4xx'++ WHEN status_code >= 500 THEN '5xx'++ ELSE 'other' END as group_code,++ COUNT(*) as cnt++ FROM analytics_events WHERE service = ? AND created_at >= ?++ GROUP BY group_code ORDER BY cnt DESC++ `).all(...human);++ const breakdown = (col) => q(`++ SELECT ${col}, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY ${col} ORDER BY cnt DESC++ `).all(...human);++ const deviceBreakdown = breakdown('device_type');++ const browserBreakdown = breakdown('browser');++ const osBreakdown = breakdown('os');++ const countryBreakdown = q(`++ SELECT country, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0++ GROUP BY country ORDER BY cnt DESC LIMIT 20++ `).all(...human);++++ const responsePercentiles = safe(() => {++ const vals = q(`++ SELECT response_time_ms FROM analytics_events++ WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0++ ORDER BY response_time_ms ASC++ `).pluck().all(...human);++ if (!vals.length) return {};++ const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];++ return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };++ }, {});++++ const bandwidth = safe(() => {++ const bw = q(`++ SELECT COUNT(*) as total_requests,++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,++ SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ `).get(...human);++ return bw && {++ total_requests: bw.total_requests || 0,++ page_requests: bw.page_requests || 0,++ api_requests: bw.api_requests || 0,++ estimated_bytes: bw.estimated_bytes || 0,++ };++ }, {});++++ const authBreakdown = safe(() => q(`++ SELECT++ COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,++ COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,++ COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ `).get(...human), {});++++ const errorTrend = safe(() => (isSubDay++ ? q(`++ SELECT ${bucketExpr} as bucket, COUNT(*) AS total,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,++ COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY bucket ORDER BY bucket ASC++ `).all(...human)++ : q(`++ SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors++ FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC++ `).all(svc, hourlyCutoff)), []);++++ const peakHours = safe(() => q(`++ SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,++ COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY hour_of_day ORDER BY hour_of_day ASC++ `).all(...human), []);++++ const topApiEndpoints = safe(() => q(`++ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0++ GROUP BY path, method ORDER BY hits DESC LIMIT 20++ `).all(...human), []);++++ const sessionCount = safe(() => q(`++ SELECT COUNT(DISTINCT session_id) FROM analytics_events++ WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0++ `).pluck().get(...human), 0);++++ // New vs returning needs a visitor identity that outlives a day; analytics keeps none.++ const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };++++ const slowestEndpoints = safe(() => q(`++ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms++ FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0++ GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10++ `).all(...human), []);++++ const authTrend = safe(() => q(`++ SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,++ COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,++ COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY bucket ORDER BY bucket ASC++ `).all(...human), []);++++ return {++ service: svc,++ period_days: effectiveDays,++ period_hours: hours || null,++ summary: summary || {},++ realtime: realtime || {},++ daily,++ hourly,++ timeBuckets,++ topPages,++ topReferers,++ botBreakdown,++ statusCodes,++ deviceBreakdown,++ browserBreakdown,++ osBreakdown,++ countryBreakdown,++ responsePercentiles,++ bandwidth,++ authBreakdown,++ errorTrend,++ peakHours,++ topApiEndpoints,++ sessionCount,++ visitorTypes,++ slowestEndpoints,++ authTrend,++ };++ }++++ getOverview(days = 30) {++ const cutoff = dayOf(this._now() - days * 86400000);++ const services = this.db.prepare(`++ SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,++ SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,++ SUM(bot_hits) as bot_hits, SUM(error_count) as errors,++ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms++ FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC++ `).all(cutoff);++ const totals = this.db.prepare(`++ SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,++ SUM(bot_hits) as bot_hits, SUM(error_count) as errors++ FROM analytics_daily WHERE date >= ?++ `).get(cutoff);++ const dailyTrend = this.db.prepare(`++ SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,++ SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits++ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC++ `).all(cutoff);++ const realtime = this.db.prepare(`++ SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots++ FROM analytics_events WHERE created_at >= ? GROUP BY service++ `).all(sqlTime(this._now() - 300000));++ return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };++ }++++ /**++ * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes++ * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is++ * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.++ */++ getBotAnalysis(days = 30) {++ const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);++ const topBotIPs = this.db.prepare(`++ SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,++ COUNT(DISTINCT session_id) AS sessions,++ MIN(created_at) as first_seen, MAX(created_at) as last_seen++ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?++ GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25++ `).all(since);++ const botTrend = this.db.prepare(`++ SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits++ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC++ `).all(since.slice(0, 10));++ const botTypes = this.db.prepare(`++ SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions++ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?++ GROUP BY bot_type ORDER BY hits DESC++ `).all(since);++ const suspiciousIPs = this.db.prepare(`++ SELECT NULL AS ip, session_id, COUNT(*) as total_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,++ COUNT(DISTINCT path) as unique_paths,++ MIN(created_at) as first_seen, MAX(created_at) as last_seen++ FROM analytics_events++ WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL++ GROUP BY session_id HAVING total_hits > 500++ ORDER BY total_hits DESC LIMIT 20++ `).all(sqlTime(this._now() - 7 * 86400000));++ return { topBotIPs, botTrend, botTypes, suspiciousIPs };++ }++++ destroy() {++ this.flush();++ for (const t of this._timers) { clearInterval(t); clearTimeout(t); }++ this._timers = [];++ this._rates.clear();++ this._sessions.clear();++ }++}++++module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };@@@ -25,7 +25,7 @@ const { EmailService } = require('./notifications/email-service');const createNotificationRoutes = require('./notifications/routes');const createAdminRoutes = require('./admin/routes');const createSetupRoutes = require('./setup/routes');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const networkAnalytics = require('./analytics/network'); // ADR-021: no IP/user id, route templates, 30-day raw retentionconst { DiscordService } = require('./discord/discord-service');const createDiscordRoutes = require('./discord/routes');const createDeployRoutes = require('./deploy/routes');@@@ -310,8 +310,11 @@ app.locals.urlRegistry = resolvedRegistry;// Make registry accessible to signToken via config._registryconfig._registry = resolvedRegistry;--// ── Analytics Tracking ────────────────────────────────────────--const analytics = new AnalyticsTracker(db, 'openvibe-network');++// ── Analytics Tracking (ADR-021) ──────────────────────────────++// Same tables in network.db, on a connection of the tracker's own (server/analytics/network.js).++// Raw rows: route template, rotating session id, user-agent class, referer origin; never an IP or a++// user id. Pruned after 30 days by the analytics-prune job below; rollups are kept.++const analytics = networkAnalytics.openAnalytics(config.db.path);app.locals.analytics = analytics;app.use(analytics.middleware());@@@ -706,6 +709,10 @@ app.listen(config.port, config.host, () => {// Process email queue every 2 minutessetInterval(() => emailService.processQueue(notificationService), 2 * 60 * 1000);++ // Raw analytics retention (ADR-021), job analytics-prune: events older than 30 days go, in++ // bounded batches; hourly/daily rollups stay. First run 5 minutes after boot, then every 24 h.++ networkAnalytics.schedulePrune(analytics);++// Clean expired sessions dailysetInterval(() => {const cleaned = db.prepare("DELETE FROM user_sessions WHERE expires_at < datetime('now') OR is_active = 0").run().changes;nnew file mode 100644@@@ -0,0 +1,289 @@++'use strict';++// ADR-021 analytics bounds on OpenVibe.Network (server/analytics/, scripts/analytics-prune.js):++// - real requests through Network's own routers (auth, OAuth, avatars, admin analytics) leave no IP,++// user id, city, raw user agent, full referer, query value or username in any analytics table;++// the session id is a rotating id, paths are route templates;++// - the tracker runs on its own connection to network.db: the identity connection keeps busy_timeout 5000;++// - the admin dashboards keep their shapes (bot rows carry ua_class / session_id, never an IP);++// - the analytics-prune job deletes raw rows older than 30 days and leaves rollups alone;++// - the CLI's dry run changes nothing, --apply needs --backup <new file> or --no-backup, the backup is++// owner-only, and a scrub touches analytics rows only.++// node test/analytics-privacy.test.js++const assert = require('assert');++const crypto = require('crypto');++const fs = require('fs');++const os = require('os');++const path = require('path');++const http = require('http');++const express = require('express');++const cookieParser = require('cookie-parser');++const jwt = require('jsonwebtoken');++const Database = require('better-sqlite3');++const { initDb } = require('../server/db/database');++const { privacy, retention } = require('../server/analytics');++const { sqlTime } = require('../server/analytics/tracker');++const networkAnalytics = require('../server/analytics/network');++const cli = require('../scripts/analytics-prune');++++let failures = 0;++async function check(name, fn) {++ try { await fn(); console.log(' ✓', name); }++ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }++}++++const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-net-analytics-'));++const quietInit = (file) => { const log = console.log; console.log = () => {}; try { return initDb(file); } finally { console.log = log; } };++const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';++const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';++const USERNAME = 'alexsecretname';++++/** Every value in every analytics table (network.db also holds accounts, which are not analytics). */++function dumpAnalytics(db) {++ const tables = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'analytics_%'").pluck().all();++ assert.ok(tables.includes('analytics_events'));++ return tables.map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');++}++++(async () => {++ await check('Network paths: usernames, anon tokens and project slugs become parameters', () => {++ const t = (p) => privacy.normalisePath(networkAnalytics.preReducePath(p), networkAnalytics.PATH_OPTS);++ const cases = {++ '/avatar/alex?s=96': '/avatar/:param',++ '/api/auth/anon/k3yF00barBaz?x=1': '/api/auth/anon/:param',++ '/internal/users/by-username/alex': '/internal/users/:param/:username',++ '/api/v1/projects/my-app/apps': '/api/v1/projects/:param/apps',++ '/oauth/authorize?client_id=live&state=abc': '/oauth/authorize',++ '/reset-password?token=abc': '/reset-password',++ '/api/admin/users/42/role': '/api/admin/users/:param/role',++ };++ for (const [raw, want] of Object.entries(cases)) assert.strictEqual(t(raw), want, raw);++ // Without Network's rule the shared normaliser would keep the name.++ assert.strictEqual(privacy.normalisePath('/internal/users/by-username/alex'), '/internal/users/:param/alex');++ });++++ // ── Real requests through Network's routers ──────────────++ const dbFile = path.join(dir, 'network.db');++ const db = quietInit(dbFile);++ db.prepare("INSERT INTO users (id, username, password_hash, role) VALUES (42, ?, 'x', 'admin')").run(USERNAME);++ let clock = Date.parse('2026-09-23T10:15:00Z');++ const analytics = networkAnalytics.openAnalytics(dbFile, { timers: false, now: () => clock });++++ const keys = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });++ const ISSUER = 'https://openvibe.network';++ const closed = 'http://127.0.0.1:9'; // remote dashboards: refused at once, never a local dev server++ const config = {++ internalKey: 'k'.repeat(32), baseUrl: ISSUER, loginUrl: ISSUER, networkUrl: ISSUER,++ jwt: { issuer: ISSUER, accessTokenExpiry: '1h', refreshTokenExpiry: '30d' },++ services: { live: { internalUrl: closed }, tools: { internalUrl: closed }, games: { internalUrl: closed }, media: { internalUrl: closed } },++ };++ const requireAuth = require('../server/auth/session').makeRequireAuth(() => ({ db, publicKey: keys.publicKey, config }), require('../server/auth/routes').signToken);++ const avatarService = require('../server/profile/avatar').createAvatarService({ db, config, requireAuth, log: { log() {}, warn() {}, error() {} } });++++ const app = express();++ app.set('trust proxy', 2); // as server/index.js: Cloudflare → Nginx → Node++ app.use(cookieParser());++ app.use(express.json());++ app.use(analytics.middleware());++ app.locals.db = db;++ app.locals.config = config;++ app.locals.privateKey = keys.privateKey;++ app.locals.publicKey = keys.publicKey;++ app.use('/api/auth', require('../server/auth/routes'));++ app.use('/oauth', require('../server/auth/oauth-routes'));++ // Stands in for the /avatar rate limiter: a request refused before any route matched.++ app.use('/avatar', (req, res, next) => (req.query.block ? res.status(429).end() : next()), avatarService.pub);++ app.use('/internal', (req, res) => res.status(403).json({ error: 'internal only' })); // refused before routing++ app.use('/api/admin/analytics', require('../server/admin/analytics-routes')(analytics, requireAuth, config));++ const server = http.createServer(app);++ await new Promise((r) => server.listen(0, '127.0.0.1', r));++ const base = `http://127.0.0.1:${server.address().port}`;++ const token = jwt.sign({ sub: 42, id: 42, username: USERNAME, role: 'admin' }, keys.privateKey, { algorithm: 'RS256', issuer: ISSUER, expiresIn: '1h' });++ const hdr = (extra = {}) => ({++ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', 'cf-ipcountry': 'NL', 'cf-ipcity': 'Amsterdam',++ referer: 'https://www.google.com/search?q=secret-query', ...extra,++ });++ const get = async (p, headers) => { const r = await fetch(base + p, { headers, redirect: 'manual' }); await r.arrayBuffer(); return r.status; };++ const getJson = async (p, headers) => (await fetch(base + p, { headers })).json();++++ await check('real requests store no IP, user id, city, raw UA, full referer, query value or username', async () => {++ assert.strictEqual(await get('/api/auth/me?token=supersecret', hdr({ authorization: `Bearer ${token}` })), 200);++ assert.strictEqual(await get(`/avatar/${USERNAME}?s=96`, hdr()), 200);++ assert.strictEqual(await get('/avatar/bobsecretname?block=1', hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' })), 429);++ await get('/oauth/authorize?client_id=live&redirect_uri=https%3A%2F%2Fopenvibe.live%2Fcb&state=statesecret', hdr());++ assert.strictEqual(await get('/api/auth/anon/0123456789abcdefanon', hdr()), 404);++ await get('/no/such/page/424242?email=alex%40example.com', hdr());++ assert.strictEqual(await get('/internal/users/by-username/carolsecretname', hdr()), 403);++ await get('/api/auth/me', hdr({ 'user-agent': 'curl/8.5.0' }));++ await new Promise((r) => setTimeout(r, 50));++ analytics.flush();++++ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();++ assert.strictEqual(rows.length, 8);++ for (const r of rows) {++ assert.strictEqual(r.service, 'openvibe-network');++ assert.strictEqual(r.ip, null);++ assert.strictEqual(r.user_id, null);++ assert.strictEqual(r.city, null);++ assert.ok(!/[?#]/.test(r.path), r.path);++ assert.strictEqual(r.referer, 'https://www.google.com');++ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id), r.session_id);++ assert.strictEqual(r.country, 'NL');++ assert.ok(/^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/.test(r.user_agent), r.user_agent);++ }++ assert.deepStrictEqual(rows.map((r) => r.path), [++ '/api/auth/me', '/avatar/:username', '/avatar/:param', '/oauth/authorize',++ '/api/auth/anon/:token', '/no/such/page/:id', '/internal/users/:param/:username', '/api/auth/me',++ ]);++ assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0, 0, 0, 0, 0, 0, 0]);++ assert.strictEqual(rows[7].user_agent, 'bot:curl');++ assert.strictEqual(rows[0].session_id, rows[1].session_id, 'same visitor, same session');++ assert.notStrictEqual(rows[0].session_id, rows[2].session_id);++++ const everything = dumpAnalytics(db);++ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', 'Amsterdam', 'supersecret', 'secret-query',++ '/search', 'statesecret', 'openvibe.live%2Fcb', USERNAME, 'bobsecretname', 'carolsecretname', '0123456789abcdefanon', '424242',++ 'alex%40example.com', '[email protected]', 'Mozilla/5.0', 'curl/8.5.0', 'Bearer', token.slice(0, 20)]) {++ assert.ok(!everything.includes(needle), `found ${needle} in the analytics tables`);++ }++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);++ });++++ await check('the tracker has its own connection: the identity connection keeps busy_timeout 5000', () => {++ assert.notStrictEqual(analytics.db, db);++ assert.strictEqual(db.pragma('busy_timeout', { simple: true }), 5000);++ assert.strictEqual(analytics.db.pragma('busy_timeout', { simple: true }), 250);++ assert.strictEqual(db.pragma('secure_delete', { simple: true }), 0);++ });++++ await check('admin dashboards keep their shapes; bot rows carry ua_class / session_id, never an IP', async () => {++ const auth = { authorization: `Bearer ${token}`, 'user-agent': CHROME };++ analytics.aggregate();++ const warn = console.warn;++ console.warn = () => {}; // the remote services are unreachable here on purpose++ try { await dashboards(auth); } finally { console.warn = warn; }++ });++ async function dashboards(auth) {++ const svc = await getJson('/api/admin/analytics/service/openvibe-network?days=30', auth);++ assert.ok(svc.ok, JSON.stringify(svc));++ for (const k of ['summary', 'realtime', 'daily', 'hourly', 'topPages', 'authBreakdown', 'visitorTypes', 'authTrend']) assert.ok(k in svc.analytics, k);++ assert.strictEqual(svc.analytics.visitorTypes.new_visitors, null);++ const bots = (await getJson('/api/admin/analytics/bots?days=30', auth)).bots['openvibe-network'];++ for (const k of ['topBotIPs', 'botTrend', 'botTypes', 'suspiciousIPs']) assert.ok(Array.isArray(bots[k]), k);++ const curl = bots.topBotIPs.find((b) => b.ua_class === 'bot:curl');++ assert.ok(curl, JSON.stringify(bots.topBotIPs));++ assert.strictEqual(curl.ip, null);++ assert.ok(bots.botTypes.every((t) => t.unique_ips === null && typeof t.unique_sessions === 'number'));++ // /realtime reads the wall clock (last 5 minutes): one request stamped now.++ clock = Date.now();++ await get('/api/auth/me', hdr({ authorization: `Bearer ${token}` }));++ await new Promise((r) => setTimeout(r, 50));++ analytics.flush();++ const rt = await getJson('/api/admin/analytics/realtime', auth);++ assert.ok(rt.ok && typeof rt.realtime.visitors === 'number' && rt.realtime.visitors >= 1, JSON.stringify(rt));++ const ov = await getJson('/api/admin/analytics/overview?days=30', auth);++ assert.ok(ov.ok && ov.overview.services.some((s) => s.name === 'openvibe-network'));++ const hourly = db.prepare("SELECT * FROM analytics_hourly WHERE hour = '2026-09-23 10:00:00'").get();++ assert.ok(hourly && hourly.unique_visitors >= 2 && hourly.unique_users === 1, JSON.stringify(hourly));++ }++ server.close();++++ await check('analytics-prune job: raw rows older than 30 days go, rollups stay', async () => {++ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, ip, user_id, created_at) VALUES ('openvibe-network', 'pageview', '/@old', '198.51.100.1', 7, ?)`);++ for (let i = 0; i < 5; i++) ins.run(sqlTime(Date.now() - (40 + i) * 86400000));++ ins.run(sqlTime(Date.now() - 29 * 86400000));++ db.prepare("INSERT OR REPLACE INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors) VALUES ('openvibe-network', '2026-07-01', 9, 3, 4)").run();++ const totals = retention.rollupTotals(db);++ const before = db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get();++ const logs = [];++ const job = networkAnalytics.schedulePrune(analytics, { initialDelayMs: 1e9, intervalMs: 1e9, log: (l) => logs.push(l) });++ try {++ const out = await job.run();++ assert.strictEqual(out.deleted, 5);++ assert.ok(/pruned 5 raw events/.test(logs[0]), logs.join('\n'));++ } finally { job.stop(); }++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), before - 5);++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM users').pluck().get(), 1);++ });++ analytics.destroy();++ analytics.db.close();++ db.close();++++ // ── CLI against a network.db ─────────────────────────────++ const quiet = [];++ const log = (l) => quiet.push(l);++ const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');++ function legacyNetworkDb(name) {++ const file = path.join(dir, name);++ const d = quietInit(file);++ d.prepare("INSERT INTO users (id, username, password_hash) VALUES (7, 'keepme', 'x')").run();++ const t = networkAnalytics.openAnalytics(file, { timers: false });++ t.destroy();++ t.db.close();++ const ins = d.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)++ VALUES ('openvibe-network', 'pageview', ?, 'GET', 200, 7, ?, ?, 'Lyon', ?, 'https://t.co/abc?x=1', ?)`);++ const nowMs = Date.now() + 120000;++ for (let i = 0; i < 4; i++) ins.run('/avatar/keepme?s=' + i, 'legacy-session-' + i, '198.51.100.' + i, CHROME, sqlTime(nowMs - (45 + i) * 86400000));++ for (let i = 0; i < 3; i++) ins.run('/api/auth/anon/tok' + i + 'X9zz99', 'abcdef012345678' + i, '198.51.100.2' + i, FIREFOX, sqlTime(nowMs - (2 + i) * 86400000));++ ins.run('/internal/users/by-username/keepme', null, '127.0.0.1', 'node', sqlTime(nowMs - 3 * 86400000));++ d.prepare(`INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers)++ VALUES ('openvibe-network', '2026-07-01', 100, 40, 30, 10, 2, ?, ?)`)++ .run(JSON.stringify([{ path: '/@keepme', cnt: 3 }, { path: '/internal/users/by-username/keepme', cnt: 2 }, { path: '/login', cnt: 1 }]), JSON.stringify([{ referer: 'https://www.google.com/search?q=keepme', cnt: 4 }]));++ d.pragma('wal_checkpoint(TRUNCATE)');++ d.close();++ return file;++ }++ const snapshot = (file) => { const d = new Database(file, { readonly: true }); try { return dumpAnalytics(d); } finally { d.close(); } };++++ await check('CLI: default database is $DB_PATH, else data/network.db, from the repo root', () => {++ const root = path.join(__dirname, '..');++ assert.strictEqual(cli.defaultDbPath({}, {}), path.join(root, 'data', 'network.db'));++ assert.strictEqual(cli.defaultDbPath({}, { DB_PATH: './data/other.db' }), path.join(root, 'data', 'other.db'));++ assert.strictEqual(cli.defaultDbPath({ db: '/x/y.db' }, { DB_PATH: './data/other.db' }), '/x/y.db');++ });++++ await check('CLI dry run (with and without --scrub) changes nothing', async () => {++ const file = legacyNetworkDb('cli-dry.db');++ const before = { hash: sha(file), dump: snapshot(file) };++ assert.strictEqual(await cli.main(['--db', file], log), 0);++ assert.strictEqual(await cli.main(['--db', file, '--scrub', '--days', '7'], log), 0);++ assert.strictEqual(sha(file), before.hash);++ assert.strictEqual(snapshot(file), before.dump);++ assert.ok(quiet.some((l) => /prune\s+4 rows/.test(l)), quiet.join('\n'));++ });++++ await check('CLI --apply refuses without a backup choice, an existing target, or days > 30', async () => {++ const file = legacyNetworkDb('cli-refuse.db');++ const before = snapshot(file);++ assert.strictEqual(await cli.main(['--db', file, '--apply'], log), 2);++ const existing = path.join(dir, 'exists.bak');++ fs.writeFileSync(existing, 'x');++ assert.strictEqual(await cli.main(['--db', file, '--apply', '--backup', existing], log), 2);++ assert.strictEqual(await cli.main(['--db', file, '--apply', '--no-backup', '--days', '31'], log), 2);++ assert.strictEqual(snapshot(file), before);++ });++++ await check('CLI --apply --scrub --backup: owner-only verified backup, analytics reduced, accounts untouched', async () => {++ const file = legacyNetworkDb('cli-apply.db');++ const bak = path.join(dir, 'cli-apply.backup.db');++ assert.strictEqual(await cli.main(['--db', file, '--apply', '--scrub', '--backup', bak, '--batch', '2'], log), 0);++ assert.strictEqual(fs.statSync(bak).mode & 0o777, 0o600);++ const b = new Database(bak, { readonly: true });++ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 8, 'backup has every row');++ b.close();++ const d = new Database(file, { readonly: true });++ assert.strictEqual(d.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 4);++ assert.deepStrictEqual(d.prepare('SELECT DISTINCT path FROM analytics_events ORDER BY path').pluck().all(), ['/api/auth/anon/:param', '/internal/users/:param/:username']);++ assert.deepStrictEqual(JSON.parse(d.prepare('SELECT top_paths FROM analytics_daily').pluck().get()),++ [{ path: '/@:user', cnt: 3 }, { path: '/internal/users/:param/:username', cnt: 2 }, { path: '/login', cnt: 1 }]);++ assert.strictEqual(d.prepare("SELECT username FROM users WHERE id = 7").pluck().get(), 'keepme');++ d.close();++ const all = snapshot(file);++ for (const needle of ['198.51.100.', '127.0.0.1', 'Lyon', 'legacy-session', 'keepme', 'X9zz99', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);++ });++++ fs.rmSync(dir, { recursive: true, force: true });++ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }++ console.log('\nanalytics privacy: all passed');++ process.exit(0);++})();
@@ -285,6 +285,35 @@ Google-style account management supporting up to 5 accounts:
@@ -285,6 +285,35 @@ Google-style account management supporting up to 5 accounts:
- [docs/patches/live-observability.diff](docs/patches/live-observability.diff) is the same change for
- [docs/patches/live-observability.diff](docs/patches/live-observability.diff) is the same change for
OpenVibe.Live, to be applied there.
OpenVibe.Live, to be applied there.
+## Analytics (ADR-021)
+
+Network records one row per finished request in the `analytics_*` tables of `network.db` and rolls
+them up hourly and daily for the admin analytics pages (`/api/admin/analytics`, which also gathers
+Live, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (OpenVibe.Contracts
+`docs/adr/ADR-021-analytics.md`). `server/analytics/` holds the same module as OpenVibe.Live
+(`server/analytics/`) and OpenVibe.Tools (`apps/_shared/analytics/`); Network's wiring is
+`server/analytics/network.js`.
+
+- **Stored:** event type, service, **route template** (the matched Express route, else the path
+ without its query and with ids, usernames and tokens replaced by `:id` / `:param`), method, status,
+ response time, a **rotating session id**, country (CDN header), **user-agent class**
+ (`chrome/windows/desktop`, `bot:curl`), referer **origin** only, bot flags, a signed-in flag.
+- **Never stored:** IP address, user or subject id, city, the user-agent string, full referer URLs.
+ The `ip`, `user_id` and `city` columns stay for compatibility and are always NULL. Per-IP counters
+ for the bot rate check live in memory only. Unique visitors come from a daily-salted hash kept only
+ until that day's final rollup. "New vs returning visitors" is no longer measured.
+- **Admin panel:** the bot tables list user-agent classes and high-volume session ids, not IPs.
+- **Connection:** the tracker opens its own connection to `network.db`, so its settings
+ (`busy_timeout` 250, `secure_delete`) never apply to the identity connection.
+- **Retention:** raw events older than 30 days are deleted every night in batches of 5000 (the
+ `analytics-prune` job: first run 5 minutes after boot, then every 24 h). Rollups are kept.
+- **Operator CLI:** `scripts/analytics-prune.js` runs a dry run by default and changes nothing.
+ `--apply` needs `--backup <new file>` (a verified, owner-only copy of all of `network.db`) or an
+ explicit `--no-backup`. `--scrub` also rewrites the rows the old tracker wrote and the rollups'
+ top-path and referer lists. Rollup totals are compared before and after the run. VACUUM rewrites
+ all of `network.db`, so run it with the service stopped, or pass `--no-vacuum` while it is up. Run
+ the CLI as the service user.
+
## Shared Client Libraries
## Shared Client Libraries
The `openvibe-shared` package ([OpenVibers/OpenVibe.Shared](https://github.com/OpenVibers/OpenVibe.Shared), pinned in package.json) provides drop-in vanilla JS components served at `/shared/` (and `/shared/v1/`). Only the package's browser files are served; a `?v=` equal to a file's content hash is cached for a year, anything else for five minutes:
The `openvibe-shared` package ([OpenVibers/OpenVibe.Shared](https://github.com/OpenVibers/OpenVibe.Shared), pinned in package.json) provides drop-in vanilla JS components served at `/shared/` (and `/shared/v1/`). Only the package's browser files are served; a `?v=` equal to a file's content hash is cached for a year, anything else for five minutes:
@@ -363,6 +392,8 @@ Accessible to users with `role = 'admin'`. All endpoints under `/api/admin/`.
@@ -363,6 +392,8 @@ Accessible to users with `role = 'admin'`. All endpoints under `/api/admin/`.
| `dev_apps`, `dev_credentials`, `dev_auth_codes` | Developer apps, hashed client secrets, app authorization codes |
| `dev_apps`, `dev_credentials`, `dev_auth_codes` | Developer apps, hashed client secrets, app authorization codes |
| `dev_grants`, `dev_quotas` | App capability grants; per-project quotas (enforced by the owning service) |
| `dev_grants`, `dev_quotas` | App capability grants; per-project quotas (enforced by the owning service) |
| `dev_audit` | Append-only developer audit; rows that are platform events carry an event envelope |
| `dev_audit` | Append-only developer audit; rows that are platform events carry an event envelope |
+| `analytics_events`, `analytics_hourly`, `analytics_daily` | Request analytics (raw ≤ 30 days) and rollups, within ADR-021; see [Analytics](#analytics-adr-021) |
+| `analytics_visitor_days`, `analytics_day_salts` | The current day's salted visitor hashes and salt, deleted after the day's final rollup |
---
---
@@ -6,7 +6,7 @@
@@ -6,7 +6,7 @@
"scripts": {
"scripts": {
"start": "node server/index.js",
"start": "node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
- "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/security-session.test.js && node test/security-redirects.test.js",
+ "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/security-session.test.js && node test/security-redirects.test.js && node test/analytics-privacy.test.js",
"reset-db": "node server/reset-db.js",
"reset-db": "node server/reset-db.js",
"grant-admin": "node server/grant-admin.js"
"grant-admin": "node server/grant-admin.js"
},
},
@@ -3186,8 +3186,9 @@ async function loadServiceAnalytics(serviceName) {
@@ -3186,8 +3186,9 @@ async function loadServiceAnalytics(serviceName) {
{ icon: 'fa-circle-exclamation', label: 'Errors', value: fmtNum(s.total_errors) },
{ icon: 'fa-circle-exclamation', label: 'Errors', value: fmtNum(s.total_errors) },
{ icon: 'fa-gauge-high', label: 'Avg Response', value: (s.avg_response_ms || 0) + 'ms' },
{ icon: 'fa-gauge-high', label: 'Avg Response', value: (s.avg_response_ms || 0) + 'ms' },
{ icon: 'fa-arrow-up-from-bracket', label: 'Est. Bandwidth', value: fmtBytes(bw.estimated_bytes) },
{ icon: 'fa-arrow-up-from-bracket', label: 'Est. Bandwidth', value: fmtBytes(bw.estimated_bytes) },
- { icon: 'fa-user-plus', label: 'New Visitors', value: fmtNum(vis.new_visitors) },
- ].map(s => `<div class="stat-card"><div class="value">${s.value}</div><div class="label"><i class="fa-solid ${s.icon}"></i> ${s.label}</div></div>`).join('');
+ // New vs returning needs a visitor identity that outlives a day; ADR-021 analytics keeps none.
+ vis.new_visitors != null ? { icon: 'fa-user-plus', label: 'New Visitors', value: fmtNum(vis.new_visitors) } : null,
+ ].filter(Boolean).map(s => `<div class="stat-card"><div class="value">${s.value}</div><div class="label"><i class="fa-solid ${s.icon}"></i> ${s.label}</div></div>`).join('');
// For sub-day ranges, use timeBuckets for the "daily" chart
// For sub-day ranges, use timeBuckets for the "daily" chart
const isSubDay = period.hours && period.hours < 24;
const isSubDay = period.hours && period.hours < 24;
@@ -3322,7 +3323,9 @@ async function loadServiceAnalytics(serviceName) {
@@ -3322,7 +3323,9 @@ async function loadServiceAnalytics(serviceName) {
<div><strong style="color:var(--info)">${fmtNum(bw.page_requests)}</strong><br><span class="muted">Page Loads</span></div>
<div><strong style="color:var(--info)">${fmtNum(bw.page_requests)}</strong><br><span class="muted">Page Loads</span></div>
<div><strong style="color:var(--info)">${fmtNum(bw.api_requests)}</strong><br><span class="muted">API Requests</span></div>
<div><strong style="color:var(--info)">${fmtNum(bw.api_requests)}</strong><br><span class="muted">API Requests</span></div>
<div><strong style="color:var(--success)">${fmtNum(a.sessionCount || 0)}</strong><br><span class="muted">Sessions</span></div>
<div><strong style="color:var(--success)">${fmtNum(a.sessionCount || 0)}</strong><br><span class="muted">Sessions</span></div>
- <div><strong style="color:var(--success)">${fmtNum(vis.new_visitors || 0)}</strong> new / <strong>${fmtNum(vis.returning_visitors || 0)}</strong> returning<br><span class="muted">Visitors</span></div>
+ ${vis.new_visitors != null
+ ? `<div><strong style="color:var(--success)">${fmtNum(vis.new_visitors)}</strong> new / <strong>${fmtNum(vis.returning_visitors)}</strong> returning<br><span class="muted">Visitors</span></div>`
+ : '<div><strong class="muted">Not measured</strong><br><span class="muted" title="ADR-021: no visitor identity outlives a day">New / returning</span></div>'}
</div>`;
</div>`;
// Auth vs anonymous chart
// Auth vs anonymous chart
@@ -3438,30 +3441,31 @@ async function loadBotAnalysis() {
@@ -3438,30 +3441,31 @@ async function loadBotAnalysis() {
const types = svcBots.botTypes || [];
const types = svcBots.botTypes || [];
if (types.length) {
if (types.length) {
html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Bot Types</h4>
html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Bot Types</h4>
- <table class="admin-table"><thead><tr><th>Type</th><th>Hits</th><th>Unique IPs</th></tr></thead>
- <tbody>${types.map(t => `<tr><td><span class="badge badge-danger">${esc(t.bot_type || 'unknown')}</span></td><td>${fmtNum(t.hits)}</td><td>${fmtNum(t.unique_ips)}</td></tr>`).join('')}</tbody></table>`;
+ <table class="admin-table"><thead><tr><th>Type</th><th>Hits</th><th>Sessions</th></tr></thead>
+ <tbody>${types.map(t => `<tr><td><span class="badge badge-danger">${esc(t.bot_type || 'unknown')}</span></td><td>${fmtNum(t.hits)}</td><td>${t.unique_sessions != null ? fmtNum(t.unique_sessions) : '-'}</td></tr>`).join('')}</tbody></table>`;
}
}
- // Top bot IPs
- const topIPs = svcBots.topBotIPs || [];
- if (topIPs.length) {
- html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Top Bot IPs</h4>
- <table class="admin-table"><thead><tr><th>IP</th><th>Hits</th><th>Type</th><th>First Seen</th><th>Last Seen</th></tr></thead>
- <tbody>${topIPs.slice(0, 15).map(ip => `
- <tr><td style="font-family:monospace;font-size:11px">${esc(ip.ip)}</td><td>${fmtNum(ip.hits)}</td>
- <td><span class="badge badge-warning">${esc(ip.bot_type || '?')}</span></td>
- <td style="font-size:11px">${ip.first_seen ? timeAgo(ip.first_seen) : '-'}</td>
- <td style="font-size:11px">${ip.last_seen ? timeAgo(ip.last_seen) : '-'}</td></tr>
+ // Top bots by user-agent class (ADR-021: analytics stores no IP addresses)
+ const topBots = svcBots.topBotIPs || [];
+ if (topBots.length) {
+ html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Top Bots (user-agent class)</h4>
+ <table class="admin-table"><thead><tr><th>User-agent class</th><th>Hits</th><th>Sessions</th><th>Type</th><th>First Seen</th><th>Last Seen</th></tr></thead>
+ <tbody>${topBots.slice(0, 15).map(b => `
+ <tr><td style="font-family:monospace;font-size:11px">${esc(b.ua_class || '-')}</td><td>${fmtNum(b.hits)}</td>
+ <td>${b.sessions != null ? fmtNum(b.sessions) : '-'}</td>
+ <td><span class="badge badge-warning">${esc(b.bot_type || '?')}</span></td>
+ <td style="font-size:11px">${b.first_seen ? timeAgo(b.first_seen) : '-'}</td>
+ <td style="font-size:11px">${b.last_seen ? timeAgo(b.last_seen) : '-'}</td></tr>
`).join('')}</tbody></table>`;
`).join('')}</tbody></table>`;
}
}
- // Suspicious human IPs
+ // High-volume sessions classified as human (rotating session ids, never an IP or user id)
const suspicious = svcBots.suspiciousIPs || [];
const suspicious = svcBots.suspiciousIPs || [];
if (suspicious.length) {
if (suspicious.length) {
- html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Suspicious IPs (high volume, classified as human)</h4>
- <table class="admin-table"><thead><tr><th>IP</th><th>Total Hits</th><th>Errors</th><th>Unique Paths</th><th>Period</th></tr></thead>
+ html += `<h4 style="font-size:12px;color:var(--text-muted);margin:12px 0 8px">Suspicious Sessions (high volume, classified as human)</h4>
+ <table class="admin-table"><thead><tr><th>Session</th><th>Total Hits</th><th>Errors</th><th>Unique Paths</th><th>Period</th></tr></thead>
<tbody>${suspicious.map(s => `
<tbody>${suspicious.map(s => `
- <tr><td style="font-family:monospace;font-size:11px">${esc(s.ip)}</td><td>${fmtNum(s.total_hits)}</td>
+ <tr><td style="font-family:monospace;font-size:11px">${esc(s.session_id || '-')}</td><td>${fmtNum(s.total_hits)}</td>
<td style="color:var(--danger)">${fmtNum(s.error_hits)}</td><td>${fmtNum(s.unique_paths)}</td>
<td style="color:var(--danger)">${fmtNum(s.error_hits)}</td><td>${fmtNum(s.unique_paths)}</td>
<td style="font-size:11px">${s.first_seen ? timeAgo(s.first_seen) : '-'} — ${s.last_seen ? timeAgo(s.last_seen) : '-'}</td></tr>
<td style="font-size:11px">${s.first_seen ? timeAgo(s.first_seen) : '-'} — ${s.last_seen ? timeAgo(s.last_seen) : '-'}</td></tr>
`).join('')}</tbody></table>`;
`).join('')}</tbody></table>`;
new file mode 100644
new file mode 100644
@@ -0,0 +1,161 @@
@@ -0,0 +1,161 @@
+#!/usr/bin/env node
+/**
+ * Raw analytics retention and one-time scrub for Network's analytics tables (ADR-021). They live in
+ * network.db, Network's identity database, next to users and sessions.
+ *
+ * node scripts/analytics-prune.js # dry run: counts only, changes nothing
+ * node scripts/analytics-prune.js --scrub # dry run including what the scrub would rewrite
+ * node scripts/analytics-prune.js --apply --backup <file> # online backup to <file>, then prune
+ * node scripts/analytics-prune.js --apply --scrub --backup <file>
+ * node scripts/analytics-prune.js --apply --no-backup # prune without a backup (explicit)
+ *
+ * Options:
+ * --db <file> database (default: $DB_PATH, else data/network.db; relative paths from the repo root)
+ * --days <n> keep raw events newer than n days, 1..30 (default 30)
+ * --scrub after pruning, rewrite the remaining rows: ip/user_id/city → NULL, path → route
+ * template (with Network's path rules: /avatar/<name>, /users/by-username/<name>,
+ * /api/auth/anon/<token>), referer → origin, user_agent → class, legacy session
+ * ids → NULL; same path/referer reduction in the rollups' top lists (counts untouched)
+ * --no-vacuum skip the VACUUM that follows an --apply (VACUUM rewrites the file so pruned and
+ * scrubbed values do not survive in free pages; it needs ~2x the database size free).
+ * VACUUM rewrites all of network.db and holds its write lock meanwhile: Network's own
+ * writes (sign-ins, sessions) wait 5 s and then fail. Vacuum with the service stopped,
+ * or pass --no-vacuum while it is up.
+ * --batch <n> rows per write batch (default 5000)
+ *
+ * --apply refuses to run without --backup <file> (a new file, mode 0600 — it is a full copy of
+ * network.db; sqlite online backup, verified) or an explicit --no-backup. Rollup totals are compared
+ * before and after; any difference exits 1. Prune and scrub are safe while the server is up (short
+ * batches; the tracker retries a busy flush). Run as the service user so no file changes owner.
+ */
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const Database = require('better-sqlite3');
+const retention = require('../server/analytics/retention');
+const networkAnalytics = require('../server/analytics/network');
+
+const ROOT = path.join(__dirname, '..');
+const HELP_LINES = 29; // header lines 3..29
+const { paramPrefixes } = networkAnalytics.PATH_OPTS;
+
+function parseArgs(argv) {
+ const a = { apply: false, scrub: false, vacuum: true, backup: null, noBackup: false, days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, db: null };
+ for (let i = 0; i < argv.length; i++) {
+ const k = argv[i];
+ const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };
+ if (k === '--apply') a.apply = true;
+ else if (k === '--scrub') a.scrub = true;
+ else if (k === '--no-vacuum') a.vacuum = false;
+ else if (k === '--no-backup') a.noBackup = true;
+ else if (k === '--backup') a.backup = val();
+ else if (k === '--days') a.days = Number(val());
+ else if (k === '--batch') a.batch = Number(val());
+ else if (k === '--db') a.db = val();
+ else if (k === '-h' || k === '--help') a.help = true;
+ else throw new Error(`unknown option ${k}`);
+ }
+ retention.checkDays(a.days);
+ if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');
+ if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');
+ return a;
+}
+
+/** The database the server uses: --db, else $DB_PATH, else data/network.db (relative to the repo root). */
+function defaultDbPath(args, env = process.env) {
+ return path.resolve(ROOT, args.db || env.DB_PATH || path.join('data', 'network.db'));
+}
+
+const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;
+function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }
+function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }
+const sameTotals = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+async function main(argv, log = console.log) {
+ let args;
+ try { args = parseArgs(argv); } catch (e) { log(`error: ${e.message}`); return 2; }
+ if (args.help) { log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, HELP_LINES).join('\n')); return 0; }
+ const dbPath = defaultDbPath(args);
+ if (!fs.existsSync(dbPath)) { log(`error: ${dbPath} does not exist`); return 2; }
+
+ if (args.apply && !args.backup && !args.noBackup) {
+ log('refusing --apply without a backup: pass --backup <new file> (sqlite online backup) or --no-backup');
+ return 2;
+ }
+
+ const db = new Database(dbPath, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });
+ try {
+ db.pragma('busy_timeout = 5000');
+ const walBytes = fileBytes(dbPath + '-wal');
+ const before = retention.inspect(db, { days: args.days, paramPrefixes });
+ log(`database ${dbPath}`);
+ log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);
+ log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);
+ log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);
+ if (before.toScrub) {
+ const t = before.toScrub;
+ log(`scrub of the rows that stay: ${t.personal} with ip/user_id/city, ${t.paths} paths to template, ${t.referers} referers to origin, ${t.user_agents} user agents to class, ${t.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);
+ }
+ log(`rate rows ${before.rateRows || 0} (IP counters; the server keeps them in memory now)`);
+ for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);
+
+ if (!args.apply) {
+ log('\ndry run: nothing changed. Re-run with --apply --backup <file> (or --no-backup).');
+ return 0;
+ }
+
+ if (args.backup) {
+ const target = path.resolve(args.backup);
+ if (fs.existsSync(target)) { log(`error: backup target ${target} already exists; choose a new file`); return 2; }
+ const need = before.bytes + walBytes;
+ const free = freeBytes(path.dirname(target));
+ if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(target)}, the backup needs about ${mb(need)}`); return 2; }
+ // A full copy of network.db (accounts, sessions): owner-only from the first byte.
+ const umask = process.umask(0o077);
+ try { await db.backup(target); } finally { process.umask(umask); }
+ fs.chmodSync(target, 0o600);
+ const b = new Database(target, { readonly: true });
+ try {
+ const ok = b.pragma('quick_check', { simple: true });
+ const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()
+ ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;
+ if (ok !== 'ok' || n < before.events) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }
+ } finally { b.close(); }
+ log(`backup ${target} (${mb(fileBytes(target))}, mode 0600, quick_check ok)`);
+ }
+
+ db.pragma('secure_delete = ON');
+ const totalsBefore = retention.rollupTotals(db);
+ const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });
+ log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);
+ if (args.scrub) {
+ await networkAnalytics.prescrub(db, { batchSize: args.batch });
+ const s = await retention.scrubEvents(db, { batchSize: args.batch, paramPrefixes });
+ const r = retention.scrubRollups(db, { paramPrefixes });
+ log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);
+ }
+ const totalsAfter = retention.rollupTotals(db);
+ if (!sameTotals(totalsBefore, totalsAfter)) {
+ log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);
+ return 1;
+ }
+ log('rollups totals unchanged');
+ if (args.vacuum && (pruned.deleted || args.scrub)) {
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ db.exec('VACUUM');
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ log(`vacuumed ${mb(fileBytes(dbPath))}`);
+ }
+ const after = retention.inspect(db, { days: args.days, paramPrefixes });
+ log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);
+ return 0;
+ } finally {
+ db.close();
+ }
+}
+
+if (require.main === module) {
+ main(process.argv.slice(2)).then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });
+}
+
+module.exports = { main, parseArgs, defaultDbPath };
@@ -287,10 +287,11 @@ module.exports = function createAnalyticsRoutes(analytics, requireAuth, config)
@@ -287,10 +287,11 @@ module.exports = function createAnalyticsRoutes(analytics, requireAuth, config)
const fiveMinAgo = new Date(Date.now() - 300000).toISOString().slice(0, 19).replace('T', ' ');
const fiveMinAgo = new Date(Date.now() - 300000).toISOString().slice(0, 19).replace('T', ' ');
const db = analytics.db;
const db = analytics.db;
+ // ADR-021: raw events carry no IP; visitors are distinct rotating session ids.
const realtime = db.prepare(`
const realtime = db.prepare(`
SELECT
SELECT
COUNT(*) as requests,
COUNT(*) as requests,
- COUNT(DISTINCT ip) as visitors,
+ COUNT(DISTINCT session_id) as visitors,
COUNT(*) FILTER (WHERE is_bot = 1) as bots,
COUNT(*) FILTER (WHERE is_bot = 1) as bots,
COUNT(*) FILTER (WHERE event_type = 'pageview') as pageviews,
COUNT(*) FILTER (WHERE event_type = 'pageview') as pageviews,
COUNT(*) FILTER (WHERE event_type = 'api_call') as api_calls
COUNT(*) FILTER (WHERE event_type = 'api_call') as api_calls
new file mode 100644
new file mode 100644
@@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
+'use strict';
+// Request analytics within ADR-021 (see tracker.js, privacy.js, retention.js).
+const { AnalyticsTracker } = require('./tracker');
+const privacy = require('./privacy');
+const retention = require('./retention');
+const { ensureSchema } = require('./schema');
+
+module.exports = { AnalyticsTracker, privacy, retention, ensureSchema };
new file mode 100644
new file mode 100644
@@ -0,0 +1,114 @@
@@ -0,0 +1,114 @@
+'use strict';
+/**
+ * OpenVibe.Network's wiring of the ADR-021 analytics module. The other files in this directory are
+ * the module shared with OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);
+ * keep them identical to those copies and put anything Network-specific here.
+ *
+ * openAnalytics(dbPath) the tracker for service 'openvibe-network' (NetworkAnalyticsTracker).
+ * Network's analytics tables (analytics_events / _hourly / _daily …) live
+ * in network.db, as they always
+ * have; the tracker gets a better-sqlite3 connection of its own so its
+ * per-connection settings (busy_timeout 250, secure_delete) never apply to
+ * the identity connection (busy_timeout 5000).
+ * schedulePrune(tracker) the nightly `analytics-prune` job: raw events older than 30 days are
+ * deleted in bounded batches (retention.pruneRawEvents); rollups are kept.
+ * First run 5 minutes after boot, then every 24 hours; timers are unref'd.
+ * prescrub(db) Network's part of the one-time scrub (scripts/analytics-prune.js --scrub):
+ * the path rules below applied to legacy rows and rollup top lists, before
+ * the shared scrub templates them.
+ * PATH_OPTS the normaliser options (param prefixes) the CLI passes to the shared scrub.
+ */
+
+const path = require('path');
+const Database = require('better-sqlite3');
+const { AnalyticsTracker } = require('./tracker');
+const privacy = require('./privacy');
+const retention = require('./retention');
+
+const SERVICE = 'openvibe-network';
+
+/**
+ * Words whose next path segment is a parameter on Network (on top of privacy.DEFAULT_PARAM_PREFIXES).
+ * They matter when no Express route matched (a rate-limited or rejected request is templated from
+ * its raw path), and to the one-time scrub of rows the old tracker stored raw: /avatar/<username>,
+ * /api/auth/anon/<token>, /api/v1/projects/<slug>.
+ */
+const PARAM_PREFIXES = ['avatar', 'anon', 'projects'];
+const PATH_OPTS = { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...PARAM_PREFIXES]) };
+
+/**
+ * A parameter behind two fixed words, which the shared normaliser cannot see: it turns `users` into a
+ * prefix, so `/users/by-username/alex` would keep `alex`. Applied to the raw path before it.
+ */
+const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];
+function preReducePath(p) {
+ let s = String(p);
+ for (const [re, to] of PATH_RULES) s = s.replace(re, to);
+ return s;
+}
+
+class NetworkAnalyticsTracker extends AnalyticsTracker {
+ /** The raw path is only used when no Express route matched (a request refused before routing). */
+ record(req, res, entryPath, responseTimeMs) {
+ return super.record(req, res, preReducePath(entryPath || ''), responseTimeMs);
+ }
+}
+
+function openAnalytics(dbPath, opts = {}) {
+ const db = new Database(path.resolve(dbPath));
+ db.pragma('journal_mode = WAL');
+ return new NetworkAnalyticsTracker(db, SERVICE, { retention: false, paramPrefixes: PARAM_PREFIXES, ...opts });
+}
+
+const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
+const tick = () => new Promise((r) => setImmediate(r));
+
+/** PATH_RULES over legacy raw rows (id-range batches) and rollup top lists. Returns { rows, hourly, daily }. */
+async function prescrub(db, { batchSize = retention.DEFAULT_BATCH } = {}) {
+ const out = { rows: 0, hourly: 0, daily: 0 };
+ if (!tableExists(db, 'analytics_events')) return out;
+ db.function('ov_network_path', { deterministic: true }, (p) => (p == null ? null : preReducePath(p)));
+ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();
+ const upd = db.prepare("UPDATE analytics_events SET path = ov_network_path(path) WHERE id >= ? AND id < ? AND path LIKE '%/by-username/%' AND path IS NOT ov_network_path(path)");
+ for (let from = lo; lo != null && from <= hi; from += batchSize) {
+ out.rows += upd.run(from, from + batchSize).changes;
+ await tick();
+ }
+ for (const table of ['analytics_hourly', 'analytics_daily']) {
+ if (!tableExists(db, table)) continue;
+ const rows = db.prepare(`SELECT id, top_paths FROM ${table} WHERE top_paths LIKE '%/by-username/%'`).all();
+ const set = db.prepare(`UPDATE ${table} SET top_paths = ? WHERE id = ?`);
+ db.transaction(() => {
+ for (const r of rows) {
+ const next = retention.reduceTopList(r.top_paths, 'path', preReducePath);
+ if (next != null) { set.run(next, r.id); out[table === 'analytics_hourly' ? 'hourly' : 'daily']++; }
+ }
+ })();
+ }
+ return out;
+}
+
+function schedulePrune(tracker, { initialDelayMs = 5 * 60 * 1000, intervalMs = 24 * 60 * 60 * 1000, log = console.log } = {}) {
+ let running = false;
+ const run = async () => {
+ if (running) return null;
+ running = true;
+ try {
+ const out = await retention.pruneRawEvents(tracker.db, { days: retention.MAX_DAYS });
+ if (out.deleted) log(`[Analytics] pruned ${out.deleted} raw events older than ${out.cutoff}`);
+ return out;
+ } catch (err) {
+ console.error('[Analytics] analytics-prune error:', err.message);
+ return null;
+ } finally {
+ running = false;
+ }
+ };
+ const first = setTimeout(run, initialDelayMs);
+ const every = setInterval(run, intervalMs);
+ first.unref();
+ every.unref();
+ return { run, stop() { clearTimeout(first); clearInterval(every); } };
+}
+
+module.exports = { SERVICE, PARAM_PREFIXES, PATH_OPTS, NetworkAnalyticsTracker, preReducePath, openAnalytics, schedulePrune, prescrub };
new file mode 100644
new file mode 100644
@@ -0,0 +1,211 @@
@@ -0,0 +1,211 @@
+'use strict';
+/**
+ * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):
+ * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a
+ * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a
+ * precise location or a user/subject id.
+ *
+ * This file holds the pure reducers that turn a request into those fields. It has no dependencies,
+ * so the same file serves OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);
+ * keep the two copies identical.
+ */
+
+// ── Bots ──────────────────────────────────────────────────────
+
+const BOT_USER_AGENTS = [
+ /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,
+ /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,
+ /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,
+ /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,
+ /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,
+ /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,
+ /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,
+ /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,
+ /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,
+ /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,
+ /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,
+ /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,
+ /statuscake/i, /monitoring/i, /health.?check/i,
+];
+
+const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];
+
+/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */
+const HIGH_REQUEST_RATE = 60;
+
+/** The family name of the first bot pattern the user agent matches, or null. */
+function botName(ua) {
+ if (!ua) return null;
+ for (const re of BOT_USER_AGENTS) {
+ if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';
+ }
+ return null;
+}
+
+/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */
+function classifyRequest(req) {
+ const ua = (req.headers && req.headers['user-agent']) || '';
+ if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };
+ if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };
+ const path = String(req.path || req.url || '').toLowerCase();
+ if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };
+ return { isBot: false, botType: null, confidence: 0.1 };
+}
+
+// ── User agent → class ───────────────────────────────────────
+
+function parseUserAgent(ua) {
+ if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };
+ let device = 'desktop';
+ if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';
+ else if (/ipad|tablet/i.test(ua)) device = 'tablet';
+
+ let browser = 'other';
+ if (/edg\//i.test(ua)) browser = 'edge';
+ else if (/opr\/|opera/i.test(ua)) browser = 'opera';
+ else if (/firefox\//i.test(ua)) browser = 'firefox';
+ else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';
+ else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';
+ else if (/trident|msie/i.test(ua)) browser = 'ie';
+
+ let os = 'other';
+ if (/windows/i.test(ua)) os = 'windows';
+ else if (/macintosh|mac os/i.test(ua)) os = 'macos';
+ else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';
+ else if (/android/i.test(ua)) os = 'android';
+ else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';
+
+ return { device, browser, os };
+}
+
+const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;
+
+/**
+ * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,
+ * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps
+ * to itself), so a scrub can run over rows that are already reduced.
+ */
+function uaClass(ua) {
+ if (ua == null || ua === '') return 'none';
+ const s = String(ua);
+ if (UA_CLASS_RE.test(s)) return s;
+ const bot = botName(s);
+ if (bot) return `bot:${bot}`;
+ const p = parseUserAgent(s);
+ return `${p.browser}/${p.os}/${p.device}`;
+}
+
+// ── Referer → origin ─────────────────────────────────────────
+
+/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */
+function refererOrigin(ref) {
+ if (!ref) return null;
+ try {
+ const u = new URL(String(ref));
+ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
+ return u.origin;
+ } catch {
+ return null;
+ }
+}
+
+// ── Country ──────────────────────────────────────────────────
+
+/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */
+function countryCode(v) {
+ const s = String(v || '').trim().toUpperCase();
+ return /^[A-Z]{2}$/.test(s) ? s : null;
+}
+
+// ── Paths → route templates ──────────────────────────────────
+
+/**
+ * Words after which the next path segment is a parameter even when it looks like a word
+ * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.
+ */
+const DEFAULT_PARAM_PREFIXES = [
+ 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',
+ 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',
+ 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',
+ 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',
+ 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',
+];
+
+const MAX_SEGMENTS = 8;
+const MAX_TEMPLATE_LENGTH = 200;
+const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;
+const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;
+
+/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */
+function looksLikeId(s) {
+ if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…
+ if (/^\d+$/.test(s)) return true; // 123
+ if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid
+ if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes
+ if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…
+ if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid
+ if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers
+ if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names
+ if (s.length > 32) return true;
+ return false;
+}
+
+/**
+ * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by
+ * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated
+ * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.
+ */
+function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES } = {}) {
+ if (raw == null || raw === '') return '/';
+ let p = String(raw);
+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {
+ try { p = new URL(p).pathname; } catch { return '/'; }
+ }
+ p = p.split(/[?#]/)[0];
+ const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);
+ const segs = p.split('/').filter(Boolean);
+ const out = [];
+ let expectParam = false;
+ for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {
+ let s = segs[i];
+ try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }
+ if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }
+ if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }
+ if (expectParam) { out.push(':param'); expectParam = false; continue; }
+ if (looksLikeId(s)) { out.push(':id'); continue; }
+ out.push(s);
+ expectParam = prefixes.has(s.toLowerCase());
+ }
+ if (segs.length > MAX_SEGMENTS) out.push('*');
+ const t = '/' + out.join('/');
+ return t.length > MAX_TEMPLATE_LENGTH ? t.slice(0, MAX_TEMPLATE_LENGTH) : t;
+}
+
+/**
+ * The route template for a finished request: the matched Express route (mount path + route path)
+ * when there is one, else the normalised original path. Both go through normalisePath, because a
+ * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.
+ */
+function routeTemplate(req, fallbackPath, opts) {
+ const r = req && req.route;
+ if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {
+ return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);
+ }
+ return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);
+}
+
+module.exports = {
+ BOT_USER_AGENTS,
+ HONEYPOT_PATHS,
+ HIGH_REQUEST_RATE,
+ DEFAULT_PARAM_PREFIXES,
+ botName,
+ classifyRequest,
+ parseUserAgent,
+ uaClass,
+ refererOrigin,
+ countryCode,
+ looksLikeId,
+ normalisePath,
+ routeTemplate,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,209 @@
@@ -0,0 +1,209 @@
+'use strict';
+/**
+ * Raw analytics retention and the one-time scrub (ADR-021).
+ *
+ * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded
+ * batches, yielding to the event loop between batches. Rollups
+ * (analytics_hourly / analytics_daily) are never touched. Also clears
+ * leftovers: visitor hashes older than yesterday, salts of past days,
+ * and analytics_rate_tracking (IP counters live in memory now).
+ * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →
+ * NULL (authenticated = 1 where a user id was present), path → route
+ * template, referer → origin, user_agent → class, session ids that are
+ * not rotating ids → NULL.
+ * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /
+ * top_referers JSON (counts merged, never changed).
+ * inspect(db, { days }) read-only counts for a dry run.
+ * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.
+ *
+ * Used by the server (scheduled prune) and scripts/analytics-prune.js (CLI).
+ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).
+ */
+
+const { ensureSchema } = require('./schema');
+const privacy = require('./privacy');
+
+const MAX_DAYS = 30;
+const DEFAULT_BATCH = 5000;
+const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;
+
+const pad = (n) => String(n).padStart(2, '0');
+function sqlTime(ms) {
+ const d = new Date(ms);
+ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
+}
+
+function checkDays(days) {
+ const n = Number(days);
+ if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {
+ throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);
+ }
+ return n;
+}
+
+/** The created_at bound: rows strictly older than this are pruned. */
+function cutoffFor(days, nowMs = Date.now()) {
+ return sqlTime(nowMs - checkDays(days) * 86400000);
+}
+
+const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
+const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);
+const tick = () => new Promise((r) => setImmediate(r));
+
+/**
+ * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write
+ * transaction). Returns { cutoff, deleted, batches, complete }.
+ */
+async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {
+ const cutoff = cutoffFor(days, now());
+ const out = { cutoff, deleted: 0, batches: 0, complete: true };
+ if (!tableExists(db, 'analytics_events')) return out;
+ const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');
+ for (;;) {
+ if (out.batches >= maxBatches) { out.complete = false; break; }
+ const n = del.run(cutoff, batchSize).changes;
+ out.batches++;
+ out.deleted += n;
+ if (n < batchSize) break;
+ await tick();
+ }
+ const today = sqlTime(now()).slice(0, 10);
+ const yesterday = sqlTime(now() - 86400000).slice(0, 10);
+ if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);
+ if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);
+ if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();
+ return out;
+}
+
+/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */
+function registerFunctions(db, opts) {
+ db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));
+ db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));
+ db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));
+ db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));
+}
+
+/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */
+async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes } = {}) {
+ ensureSchema(db);
+ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);
+ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();
+ const out = { rows: 0, batches: 0 };
+ if (lo == null) return out;
+ const upd = db.prepare(`
+ UPDATE analytics_events SET
+ authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,
+ ip = NULL, user_id = NULL, city = NULL,
+ session_id = ov_rotating_session(session_id),
+ path = ov_route_template(path),
+ referer = ov_referer_origin(referer),
+ user_agent = ov_ua_class(user_agent)
+ WHERE id >= ? AND id < ?
+ `);
+ for (let from = lo; from <= hi; from += batchSize) {
+ out.rows += upd.run(from, from + batchSize).changes;
+ out.batches++;
+ await tick();
+ }
+ return out;
+}
+
+/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */
+function reduceTopList(json, key, fn) {
+ let list;
+ try { list = JSON.parse(json); } catch { return null; }
+ if (!Array.isArray(list)) return null;
+ const merged = new Map();
+ for (const item of list) {
+ if (!item || typeof item !== 'object') continue;
+ const k = fn(item[key]);
+ if (k == null) continue;
+ merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));
+ }
+ const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);
+ const text = JSON.stringify(next);
+ return text === json ? null : text;
+}
+
+/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */
+function scrubRollups(db, { paramPrefixes } = {}) {
+ const opts = paramPrefixes ? { paramPrefixes } : undefined;
+ const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));
+ const out = { hourly: 0, daily: 0 };
+ for (const table of ['analytics_hourly', 'analytics_daily']) {
+ if (!tableExists(db, table)) continue;
+ const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();
+ const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);
+ db.transaction(() => {
+ for (const r of rows) {
+ const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;
+ const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;
+ if (paths == null && refs == null) continue;
+ upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);
+ out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;
+ }
+ })();
+ }
+ return out;
+}
+
+/** Sums of every rollup counter; equal before and after a prune or scrub. */
+function rollupTotals(db) {
+ const out = {};
+ if (tableExists(db, 'analytics_hourly')) {
+ out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
+ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,
+ TOTAL(error_count) AS error_count FROM analytics_hourly`).get();
+ }
+ if (tableExists(db, 'analytics_daily')) {
+ out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
+ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,
+ TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();
+ }
+ return out;
+}
+
+/** Read-only counts for a dry run. */
+function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes } = {}) {
+ const cutoff = cutoffFor(days, now());
+ const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };
+ const pageSize = db.pragma('page_size', { simple: true });
+ out.bytes = db.pragma('page_count', { simple: true }) * pageSize;
+ out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;
+ out.rollups = rollupTotals(db);
+ if (!tableExists(db, 'analytics_events')) return out;
+ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);
+ const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();
+ out.events = e.n;
+ out.oldest = e.oldest;
+ out.newest = e.newest;
+ out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);
+ out.remaining = out.events - out.older;
+ const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))
+ .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';
+ const r = db.prepare(`
+ SELECT
+ COUNT(*) FILTER (WHERE ${personal}) AS personal,
+ COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,
+ COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,
+ COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,
+ COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions
+ FROM analytics_events WHERE created_at >= ?
+ `).get(cutoff);
+ out.toScrub = r;
+ out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;
+ return out;
+}
+
+module.exports = {
+ MAX_DAYS,
+ DEFAULT_BATCH,
+ checkDays,
+ cutoffFor,
+ pruneRawEvents,
+ scrubEvents,
+ scrubRollups,
+ rollupTotals,
+ inspect,
+ reduceTopList,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,123 @@
@@ -0,0 +1,123 @@
+'use strict';
+/**
+ * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the
+ * existing analytics.db files and the admin dashboards keep working; the columns that used to hold
+ * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and
+ * user_agent / referer / path hold the reduced forms (class, origin, route template).
+ *
+ * Added:
+ * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)
+ * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;
+ * rows are deleted once the day's rollup is final (≤ ~25 h)
+ * analytics_day_salts the random salt of the current UTC day (so a restart does not
+ * double-count); deleted together with that day's hashes
+ * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)
+ */
+
+const ANALYTICS_SCHEMA = `
+ CREATE TABLE IF NOT EXISTS analytics_events (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ event_type TEXT NOT NULL DEFAULT 'pageview',
+ path TEXT,
+ method TEXT DEFAULT 'GET',
+ status_code INTEGER,
+ response_time_ms INTEGER,
+ user_id INTEGER,
+ session_id TEXT,
+ ip TEXT,
+ country TEXT,
+ city TEXT,
+ user_agent TEXT,
+ referer TEXT,
+ is_bot INTEGER DEFAULT 0,
+ bot_type TEXT,
+ device_type TEXT,
+ browser TEXT,
+ os TEXT,
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_hourly (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ hour TEXT NOT NULL,
+ pageviews INTEGER DEFAULT 0,
+ api_calls INTEGER DEFAULT 0,
+ unique_visitors INTEGER DEFAULT 0,
+ unique_users INTEGER DEFAULT 0,
+ bot_hits INTEGER DEFAULT 0,
+ avg_response_ms INTEGER DEFAULT 0,
+ error_count INTEGER DEFAULT 0,
+ bandwidth_bytes INTEGER DEFAULT 0,
+ top_paths TEXT,
+ top_referers TEXT,
+ UNIQUE(service, hour)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_daily (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ date TEXT NOT NULL,
+ pageviews INTEGER DEFAULT 0,
+ api_calls INTEGER DEFAULT 0,
+ unique_visitors INTEGER DEFAULT 0,
+ unique_users INTEGER DEFAULT 0,
+ new_users INTEGER DEFAULT 0,
+ bot_hits INTEGER DEFAULT 0,
+ avg_response_ms INTEGER DEFAULT 0,
+ error_count INTEGER DEFAULT 0,
+ top_paths TEXT,
+ top_referers TEXT,
+ top_countries TEXT,
+ device_breakdown TEXT,
+ browser_breakdown TEXT,
+ UNIQUE(service, date)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_rate_tracking (
+ ip TEXT NOT NULL,
+ window_start INTEGER NOT NULL,
+ hit_count INTEGER DEFAULT 1,
+ PRIMARY KEY (ip, window_start)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_visitor_days (
+ service TEXT NOT NULL,
+ day TEXT NOT NULL,
+ hour TEXT NOT NULL,
+ vhash TEXT NOT NULL,
+ authenticated INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (service, hour, vhash)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_day_salts (
+ service TEXT NOT NULL,
+ day TEXT NOT NULL,
+ salt TEXT NOT NULL,
+ PRIMARY KEY (service, day)
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);
+ CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);
+ CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);
+
+ -- Indexes over the retired personal columns.
+ DROP INDEX IF EXISTS idx_analytics_events_ip;
+ DROP INDEX IF EXISTS idx_analytics_events_user;
+`;
+
+/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */
+function ensureSchema(db) {
+ db.exec(ANALYTICS_SCHEMA);
+ const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);
+ if (!cols.includes('authenticated')) {
+ db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');
+ }
+}
+
+module.exports = { ANALYTICS_SCHEMA, ensureSchema };
new file mode 100644
new file mode 100644
@@ -0,0 +1,728 @@
@@ -0,0 +1,728 @@
+'use strict';
+/**
+ * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the openvibe-shared
+ * AnalyticsTracker (same tables, same getStats/getOverview/getBotAnalysis shapes), minus the personal
+ * data:
+ *
+ * raw row service, event_type, route template, method, status, response time, rotating session
+ * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,
+ * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.
+ * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after
+ * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.
+ * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped
+ * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).
+ * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.
+ * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as
+ * the day is over); hashes go to analytics_visitor_days only, never to raw events,
+ * and are deleted by the first aggregation after their day ends, right after that day's
+ * final hourly/daily rollup — so no hash or salt outlives its day by more than about
+ * one aggregation interval (1 h). Rollups keep only the counts.
+ *
+ * Retention of raw events (30 days) is ./retention.js; this tracker only writes and rolls up.
+ *
+ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).
+ */
+
+const crypto = require('crypto');
+const { ensureSchema } = require('./schema');
+const privacy = require('./privacy');
+const retention = require('./retention');
+
+const SESSION_IDLE_MS = 30 * 60 * 1000;
+const MAX_TRACKED_CLIENTS = 200000;
+const MAX_BUFFER = 5000;
+const SESSION_ID_RE = /^[0-9a-f]{16}$/;
+
+const pad = (n) => String(n).padStart(2, '0');
+/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */
+function sqlTime(ms) {
+ const d = new Date(ms);
+ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
+}
+const dayOf = (ms) => sqlTime(ms).slice(0, 10);
+const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';
+const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');
+
+function isStaticOrHealth(path) {
+ return path === '/api/health' || path === '/health' ||
+ /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);
+}
+
+class AnalyticsTracker {
+ /**
+ * @param {object} db better-sqlite3 handle (its own analytics database)
+ * @param {string} service service name ('live', 'openvibe-yt', …)
+ * @param {object} [opts]
+ * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter
+ * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:
+ * run the raw-event prune on a timer (Tools). Live passes false and schedules
+ * retention.pruneRawEvents through server/utils/jobs.js instead.
+ * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)
+ * @param {function} [opts.now] clock (tests)
+ */
+ constructor(db, service, opts = {}) {
+ this.db = db;
+ this.service = service;
+ this._now = opts.now || Date.now;
+ this._pathOpts = opts.paramPrefixes
+ ? { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...opts.paramPrefixes]) }
+ : undefined;
+ this._buffer = [];
+ this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }
+ this._sessions = new Map(); // vhash -> { sid, last }
+ this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)
+ this._saltDay = null;
+ this._salt = null;
+ this._timers = [];
+
+ ensureSchema(db);
+ try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }
+ try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }
+ db.exec('DELETE FROM analytics_rate_tracking');
+
+ this._insertEvent = db.prepare(`
+ INSERT INTO analytics_events
+ (service, event_type, path, method, status_code, response_time_ms,
+ user_id, session_id, ip, country, city, user_agent, referer,
+ is_bot, bot_type, device_type, browser, os, authenticated, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `);
+ this._upsertVisitor = db.prepare(`
+ INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)
+ `);
+ this._write = db.transaction((events, visitors) => {
+ for (const e of events) {
+ this._insertEvent.run(
+ e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,
+ e.session_id, e.country, e.user_agent, e.referer,
+ e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,
+ );
+ }
+ for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);
+ });
+
+ if (opts.timers !== false) {
+ const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
+ const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
+ every(() => { this.flush(); this._sweep(); }, 5000);
+ every(() => this.aggregate(), 60 * 60 * 1000);
+ once(() => this.aggregate(), 10_000);
+ if (opts.retention) {
+ const r = opts.retention;
+ const run = () => retention.pruneRawEvents(db, { days: r.days || retention.MAX_DAYS })
+ .then((out) => { if (out.deleted) console.log(`[Analytics:${service}] pruned ${out.deleted} raw events older than ${out.cutoff}`); })
+ .catch((err) => console.error(`[Analytics:${service}] prune error:`, err.message));
+ once(run, r.initialDelayMs != null ? r.initialDelayMs : 5 * 60 * 1000);
+ every(run, r.intervalMs || 24 * 60 * 60 * 1000);
+ }
+ }
+ }
+
+ // ── Per-request reducers ──────────────────────────────────
+
+ _daySalt(day) {
+ if (this._saltDay === day) return this._salt;
+ let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
+ if (!row) {
+ this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')
+ .run(this.service, day, crypto.randomBytes(32).toString('hex'));
+ row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
+ }
+ // Earlier days' salts are never needed again: their hashes are already written.
+ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);
+ this._saltDay = day;
+ this._salt = row.salt;
+ this._sessions.clear();
+ return this._salt;
+ }
+
+ _visitorHash(ip, ua, day) {
+ return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);
+ }
+
+ _sessionId(vhash, nowMs) {
+ let s = this._sessions.get(vhash);
+ if (!s || nowMs - s.last > SESSION_IDLE_MS) {
+ s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };
+ this._sessions.set(vhash, s);
+ }
+ s.last = nowMs;
+ return s.sid;
+ }
+
+ /** Hits from this client in the current and previous minute (memory only). */
+ _rateHit(ip, nowMs) {
+ const minute = Math.floor(nowMs / 60000);
+ let r = this._rates.get(ip);
+ if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }
+ if (r.minute !== minute) {
+ r.prev = r.minute === minute - 1 ? r.count : 0;
+ r.count = 0;
+ r.minute = minute;
+ }
+ r.count++;
+ return r.count + r.prev;
+ }
+
+ _sweep() {
+ const nowMs = this._now();
+ const minute = Math.floor(nowMs / 60000);
+ for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);
+ for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);
+ if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();
+ if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();
+ }
+
+ _push(event, visitor) {
+ if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();
+ this._buffer.push(event);
+ if (visitor) {
+ const key = `${visitor.hour}|${visitor.vhash}`;
+ const prev = this._visitors.get(key);
+ if (!prev) this._visitors.set(key, visitor);
+ else if (visitor.authenticated) prev.authenticated = true;
+ }
+ if (this._buffer.length >= 100) this.flush();
+ }
+
+ /**
+ * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted
+ * router is relative by the time the response finishes).
+ */
+ record(req, res, entryPath, responseTimeMs) {
+ const nowMs = this._now();
+ const headers = req.headers || {};
+ const ua = String(headers['user-agent'] || '');
+ const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');
+ const bot = privacy.classifyRequest({ headers, path: entryPath });
+ if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {
+ bot.isBot = true;
+ bot.botType = 'rate_limit';
+ bot.confidence = 0.80;
+ }
+ const parsed = privacy.parseUserAgent(ua);
+ const day = dayOf(nowMs);
+ const vhash = this._visitorHash(ip, ua, day);
+ const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));
+ const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');
+ this._push({
+ service: this.service,
+ event_type: isApi ? 'api_call' : 'pageview',
+ path: privacy.routeTemplate(req, entryPath, this._pathOpts),
+ method: req.method,
+ status_code: res.statusCode,
+ response_time_ms: responseTimeMs,
+ session_id: this._sessionId(vhash, nowMs),
+ country: privacy.countryCode(headers['cf-ipcountry']),
+ user_agent: privacy.uaClass(ua),
+ referer: privacy.refererOrigin(headers.referer || headers.referrer),
+ is_bot: bot.isBot,
+ bot_type: bot.botType,
+ device_type: parsed.device,
+ browser: parsed.browser,
+ os: parsed.os,
+ authenticated,
+ created_at: sqlTime(nowMs),
+ }, { day, hour: hourOf(nowMs), vhash, authenticated });
+ }
+
+ /** Express middleware: one row per finished request (health checks and static assets skipped). */
+ middleware() {
+ return (req, res, next) => {
+ const entryPath = req.path || '';
+ if (isStaticOrHealth(entryPath)) return next();
+ const started = Date.now();
+ let done = false;
+ const finish = () => {
+ if (done) return;
+ done = true;
+ try { this.record(req, res, entryPath, Date.now() - started); }
+ catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }
+ };
+ res.once('finish', finish);
+ res.once('close', finish);
+ next();
+ };
+ }
+
+ /**
+ * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,
+ * referer and user_agent are reduced like a request's; session_id is kept only if it is one of
+ * this tracker's rotating ids.
+ */
+ trackEvent(eventType, data = {}) {
+ const nowMs = this._now();
+ const ua = data.user_agent ? String(data.user_agent) : '';
+ const parsed = ua ? privacy.parseUserAgent(ua) : {};
+ this._push({
+ service: this.service,
+ event_type: String(eventType || 'event').slice(0, 64),
+ path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,
+ method: data.method || null,
+ status_code: data.status_code || null,
+ response_time_ms: data.response_time_ms || null,
+ session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,
+ country: privacy.countryCode(data.country),
+ user_agent: ua ? privacy.uaClass(ua) : null,
+ referer: privacy.refererOrigin(data.referer),
+ is_bot: !!data.is_bot,
+ bot_type: data.bot_type || null,
+ device_type: data.device_type || parsed.device || null,
+ browser: data.browser || parsed.browser || null,
+ os: data.os || parsed.os || null,
+ authenticated: !!data.authenticated,
+ created_at: sqlTime(nowMs),
+ }, null);
+ }
+
+ /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */
+ flush() {
+ if (!this._buffer.length && !this._visitors.size) return;
+ const events = this._buffer.splice(0);
+ const visitors = [...this._visitors.values()];
+ this._visitors.clear();
+ try {
+ this._write(events, visitors);
+ } catch (err) {
+ console.error(`[Analytics:${this.service}] Flush error:`, err.message);
+ this._buffer = events.slice(-MAX_BUFFER);
+ for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);
+ }
+ }
+
+ // ── Rollups ───────────────────────────────────────────────
+
+ /**
+ * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has
+ * visitor hashes (finalised here), after which those hashes are deleted.
+ */
+ aggregate() {
+ try {
+ this.flush();
+ const nowMs = this._now();
+ const today = dayOf(nowMs);
+ const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);
+ const days = new Set([today, dayOf(nowMs - 86400000)]);
+ const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);
+ for (const p of pending) { hours.add(p.hour); days.add(p.day); }
+ for (const h of hours) this._aggregateHour(h);
+ for (const d of days) this._aggregateDay(d);
+ this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);
+ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);
+ } catch (err) {
+ console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);
+ }
+ }
+
+ _aggregateHour(hourStart) {
+ const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);
+ const svc = this.service;
+ const stats = this.db.prepare(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ?
+ `).get(svc, hourStart, nextHour);
+ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
+
+ // Uniques only grow within a period, so never lower a stored count (restarts, or rows
+ // counted before this tracker when the old one still stored IPs).
+ const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);
+ const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};
+
+ const topPaths = this.db.prepare(`
+ SELECT path, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0
+ GROUP BY path ORDER BY cnt DESC LIMIT 10
+ `).all(svc, hourStart, nextHour);
+ const topReferers = this.db.prepare(`
+ SELECT referer, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0
+ GROUP BY referer ORDER BY cnt DESC LIMIT 10
+ `).all(svc, hourStart, nextHour);
+
+ this.db.prepare(`
+ INSERT INTO analytics_hourly
+ (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(service, hour) DO UPDATE SET
+ pageviews = excluded.pageviews, api_calls = excluded.api_calls,
+ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
+ bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,
+ error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers
+ `).run(
+ svc, hourStart, stats.pageviews, stats.api_calls,
+ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
+ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
+ JSON.stringify(topPaths), JSON.stringify(topReferers),
+ );
+ }
+
+ _aggregateDay(date) {
+ const dayStart = date + ' 00:00:00';
+ const dayEnd = date + ' 23:59:59';
+ const svc = this.service;
+ const q = (sql) => this.db.prepare(sql);
+ const stats = q(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at <= ?
+ `).get(svc, dayStart, dayEnd);
+ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
+
+ const v = q(`
+ SELECT COUNT(DISTINCT vhash) AS visitors,
+ COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users
+ FROM analytics_visitor_days WHERE service = ? AND day = ?
+ `).get(svc, date);
+ const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};
+
+ const range = [svc, dayStart, dayEnd];
+ const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);
+ const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);
+ const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);
+ const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);
+ const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);
+
+ q(`
+ INSERT INTO analytics_daily
+ (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,
+ avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(service, date) DO UPDATE SET
+ pageviews = excluded.pageviews, api_calls = excluded.api_calls,
+ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
+ new_users = excluded.new_users, bot_hits = excluded.bot_hits,
+ avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,
+ top_paths = excluded.top_paths, top_referers = excluded.top_referers,
+ top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,
+ browser_breakdown = excluded.browser_breakdown
+ `).run(
+ svc, date, stats.pageviews, stats.api_calls,
+ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
+ // New users needs a user id history, which analytics no longer has: keep what was
+ // recorded before, NULL (not measured) from now on.
+ prev.new_users != null ? prev.new_users : null,
+ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
+ JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),
+ JSON.stringify(devices), JSON.stringify(browsers),
+ );
+ }
+
+ // ── Reads (admin dashboards) ─────────────────────────────
+ // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,
+ // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based
+ // numbers (daily/hourly unique_visitors) are true unique visitors per period.
+
+ getStats(options = {}) {
+ const { days = 30, hours, service } = options;
+ const svc = service || this.service;
+ const nowMs = this._now();
+ const rangeMs = hours ? hours * 3600000 : days * 86400000;
+ const rangeCutoff = sqlTime(nowMs - rangeMs);
+ const isSubDay = hours && hours < 24;
+ const effectiveDays = hours ? hours / 24 : days;
+ const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;
+ const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;
+ const q = (sql) => this.db.prepare(sql);
+ const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };
+
+ let daily = [];
+ if (!isSubDay) {
+ daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));
+ }
+
+ const hourlyLimit = hours ? Math.min(hours, 168) : 24;
+ const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);
+ const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);
+
+ let timeBuckets = [];
+ if (isSubDay) {
+ timeBuckets = q(`
+ SELECT ${bucketExpr} as bucket,
+ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ?
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(svc, rangeCutoff);
+ }
+
+ let summary;
+ if (hours && hours < 48) {
+ summary = q(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ?
+ `).get(svc, rangeCutoff);
+ } else {
+ summary = q(`
+ SELECT
+ SUM(pageviews) as total_pageviews,
+ SUM(api_calls) as total_api_calls,
+ SUM(unique_visitors) as total_unique_visitors,
+ SUM(unique_users) as total_unique_users,
+ SUM(bot_hits) as total_bot_hits,
+ SUM(error_count) as total_errors,
+ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
+ FROM analytics_daily
+ WHERE service = ? AND date >= ?
+ `).get(svc, dayOf(nowMs - rangeMs));
+ }
+
+ const realtime = q(`
+ SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
+ FROM analytics_events WHERE service = ? AND created_at >= ?
+ `).get(svc, sqlTime(nowMs - 300000));
+
+ const human = [svc, rangeCutoff];
+ const topPages = q(`
+ SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY path ORDER BY hits DESC LIMIT 20
+ `).all(...human);
+ const topReferers = q(`
+ SELECT referer, COUNT(*) as hits FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0
+ GROUP BY referer ORDER BY hits DESC LIMIT 15
+ `).all(...human);
+ const botBreakdown = q(`
+ SELECT bot_type, COUNT(*) as hits FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND is_bot = 1
+ GROUP BY bot_type ORDER BY hits DESC LIMIT 15
+ `).all(...human);
+ const statusCodes = q(`
+ SELECT CASE
+ WHEN status_code >= 200 AND status_code < 300 THEN '2xx'
+ WHEN status_code >= 300 AND status_code < 400 THEN '3xx'
+ WHEN status_code >= 400 AND status_code < 500 THEN '4xx'
+ WHEN status_code >= 500 THEN '5xx'
+ ELSE 'other' END as group_code,
+ COUNT(*) as cnt
+ FROM analytics_events WHERE service = ? AND created_at >= ?
+ GROUP BY group_code ORDER BY cnt DESC
+ `).all(...human);
+ const breakdown = (col) => q(`
+ SELECT ${col}, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY ${col} ORDER BY cnt DESC
+ `).all(...human);
+ const deviceBreakdown = breakdown('device_type');
+ const browserBreakdown = breakdown('browser');
+ const osBreakdown = breakdown('os');
+ const countryBreakdown = q(`
+ SELECT country, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0
+ GROUP BY country ORDER BY cnt DESC LIMIT 20
+ `).all(...human);
+
+ const responsePercentiles = safe(() => {
+ const vals = q(`
+ SELECT response_time_ms FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
+ ORDER BY response_time_ms ASC
+ `).pluck().all(...human);
+ if (!vals.length) return {};
+ const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];
+ return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };
+ }, {});
+
+ const bandwidth = safe(() => {
+ const bw = q(`
+ SELECT COUNT(*) as total_requests,
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,
+ SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ `).get(...human);
+ return bw && {
+ total_requests: bw.total_requests || 0,
+ page_requests: bw.page_requests || 0,
+ api_requests: bw.api_requests || 0,
+ estimated_bytes: bw.estimated_bytes || 0,
+ };
+ }, {});
+
+ const authBreakdown = safe(() => q(`
+ SELECT
+ COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,
+ COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,
+ COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ `).get(...human), {});
+
+ const errorTrend = safe(() => (isSubDay
+ ? q(`
+ SELECT ${bucketExpr} as bucket, COUNT(*) AS total,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
+ COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(...human)
+ : q(`
+ SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors
+ FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC
+ `).all(svc, hourlyCutoff)), []);
+
+ const peakHours = safe(() => q(`
+ SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,
+ COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY hour_of_day ORDER BY hour_of_day ASC
+ `).all(...human), []);
+
+ const topApiEndpoints = safe(() => q(`
+ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0
+ GROUP BY path, method ORDER BY hits DESC LIMIT 20
+ `).all(...human), []);
+
+ const sessionCount = safe(() => q(`
+ SELECT COUNT(DISTINCT session_id) FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0
+ `).pluck().get(...human), 0);
+
+ // New vs returning needs a visitor identity that outlives a day; analytics keeps none.
+ const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };
+
+ const slowestEndpoints = safe(() => q(`
+ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
+ GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10
+ `).all(...human), []);
+
+ const authTrend = safe(() => q(`
+ SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,
+ COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,
+ COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(...human), []);
+
+ return {
+ service: svc,
+ period_days: effectiveDays,
+ period_hours: hours || null,
+ summary: summary || {},
+ realtime: realtime || {},
+ daily,
+ hourly,
+ timeBuckets,
+ topPages,
+ topReferers,
+ botBreakdown,
+ statusCodes,
+ deviceBreakdown,
+ browserBreakdown,
+ osBreakdown,
+ countryBreakdown,
+ responsePercentiles,
+ bandwidth,
+ authBreakdown,
+ errorTrend,
+ peakHours,
+ topApiEndpoints,
+ sessionCount,
+ visitorTypes,
+ slowestEndpoints,
+ authTrend,
+ };
+ }
+
+ getOverview(days = 30) {
+ const cutoff = dayOf(this._now() - days * 86400000);
+ const services = this.db.prepare(`
+ SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
+ SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,
+ SUM(bot_hits) as bot_hits, SUM(error_count) as errors,
+ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
+ FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC
+ `).all(cutoff);
+ const totals = this.db.prepare(`
+ SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,
+ SUM(bot_hits) as bot_hits, SUM(error_count) as errors
+ FROM analytics_daily WHERE date >= ?
+ `).get(cutoff);
+ const dailyTrend = this.db.prepare(`
+ SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
+ SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits
+ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
+ `).all(cutoff);
+ const realtime = this.db.prepare(`
+ SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
+ FROM analytics_events WHERE created_at >= ? GROUP BY service
+ `).all(sqlTime(this._now() - 300000));
+ return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };
+ }
+
+ /**
+ * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes
+ * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is
+ * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.
+ */
+ getBotAnalysis(days = 30) {
+ const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);
+ const topBotIPs = this.db.prepare(`
+ SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,
+ COUNT(DISTINCT session_id) AS sessions,
+ MIN(created_at) as first_seen, MAX(created_at) as last_seen
+ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
+ GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25
+ `).all(since);
+ const botTrend = this.db.prepare(`
+ SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits
+ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
+ `).all(since.slice(0, 10));
+ const botTypes = this.db.prepare(`
+ SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions
+ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
+ GROUP BY bot_type ORDER BY hits DESC
+ `).all(since);
+ const suspiciousIPs = this.db.prepare(`
+ SELECT NULL AS ip, session_id, COUNT(*) as total_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,
+ COUNT(DISTINCT path) as unique_paths,
+ MIN(created_at) as first_seen, MAX(created_at) as last_seen
+ FROM analytics_events
+ WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL
+ GROUP BY session_id HAVING total_hits > 500
+ ORDER BY total_hits DESC LIMIT 20
+ `).all(sqlTime(this._now() - 7 * 86400000));
+ return { topBotIPs, botTrend, botTypes, suspiciousIPs };
+ }
+
+ destroy() {
+ this.flush();
+ for (const t of this._timers) { clearInterval(t); clearTimeout(t); }
+ this._timers = [];
+ this._rates.clear();
+ this._sessions.clear();
+ }
+}
+
+module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };
@@ -25,7 +25,7 @@ const { EmailService } = require('./notifications/email-service');
@@ -25,7 +25,7 @@ const { EmailService } = require('./notifications/email-service');
const createNotificationRoutes = require('./notifications/routes');
const createNotificationRoutes = require('./notifications/routes');
const createAdminRoutes = require('./admin/routes');
const createAdminRoutes = require('./admin/routes');
const createSetupRoutes = require('./setup/routes');
const createSetupRoutes = require('./setup/routes');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const networkAnalytics = require('./analytics/network'); // ADR-021: no IP/user id, route templates, 30-day raw retention
const { DiscordService } = require('./discord/discord-service');
const { DiscordService } = require('./discord/discord-service');
const createDiscordRoutes = require('./discord/routes');
const createDiscordRoutes = require('./discord/routes');
const createDeployRoutes = require('./deploy/routes');
const createDeployRoutes = require('./deploy/routes');
@@ -310,8 +310,11 @@ app.locals.urlRegistry = resolvedRegistry;
@@ -310,8 +310,11 @@ app.locals.urlRegistry = resolvedRegistry;
// Make registry accessible to signToken via config._registry
// Make registry accessible to signToken via config._registry
config._registry = resolvedRegistry;
config._registry = resolvedRegistry;
-// ── Analytics Tracking ────────────────────────────────────────
-const analytics = new AnalyticsTracker(db, 'openvibe-network');
+// ── Analytics Tracking (ADR-021) ──────────────────────────────
+// Same tables in network.db, on a connection of the tracker's own (server/analytics/network.js).
+// Raw rows: route template, rotating session id, user-agent class, referer origin; never an IP or a
+// user id. Pruned after 30 days by the analytics-prune job below; rollups are kept.
+const analytics = networkAnalytics.openAnalytics(config.db.path);
app.locals.analytics = analytics;
app.locals.analytics = analytics;
app.use(analytics.middleware());
app.use(analytics.middleware());
@@ -706,6 +709,10 @@ app.listen(config.port, config.host, () => {
@@ -706,6 +709,10 @@ app.listen(config.port, config.host, () => {
// Process email queue every 2 minutes
// Process email queue every 2 minutes
setInterval(() => emailService.processQueue(notificationService), 2 * 60 * 1000);
setInterval(() => emailService.processQueue(notificationService), 2 * 60 * 1000);
+ // Raw analytics retention (ADR-021), job analytics-prune: events older than 30 days go, in
+ // bounded batches; hourly/daily rollups stay. First run 5 minutes after boot, then every 24 h.
+ networkAnalytics.schedulePrune(analytics);
+
// Clean expired sessions daily
// Clean expired sessions daily
setInterval(() => {
setInterval(() => {
const cleaned = db.prepare("DELETE FROM user_sessions WHERE expires_at < datetime('now') OR is_active = 0").run().changes;
const cleaned = db.prepare("DELETE FROM user_sessions WHERE expires_at < datetime('now') OR is_active = 0").run().changes;
new file mode 100644
new file mode 100644
@@ -0,0 +1,289 @@
@@ -0,0 +1,289 @@
+'use strict';
+// ADR-021 analytics bounds on OpenVibe.Network (server/analytics/, scripts/analytics-prune.js):
+// - real requests through Network's own routers (auth, OAuth, avatars, admin analytics) leave no IP,
+// user id, city, raw user agent, full referer, query value or username in any analytics table;
+// the session id is a rotating id, paths are route templates;
+// - the tracker runs on its own connection to network.db: the identity connection keeps busy_timeout 5000;
+// - the admin dashboards keep their shapes (bot rows carry ua_class / session_id, never an IP);
+// - the analytics-prune job deletes raw rows older than 30 days and leaves rollups alone;
+// - the CLI's dry run changes nothing, --apply needs --backup <new file> or --no-backup, the backup is
+// owner-only, and a scrub touches analytics rows only.
+// node test/analytics-privacy.test.js
+const assert = require('assert');
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const http = require('http');
+const express = require('express');
+const cookieParser = require('cookie-parser');
+const jwt = require('jsonwebtoken');
+const Database = require('better-sqlite3');
+const { initDb } = require('../server/db/database');
+const { privacy, retention } = require('../server/analytics');
+const { sqlTime } = require('../server/analytics/tracker');
+const networkAnalytics = require('../server/analytics/network');
+const cli = require('../scripts/analytics-prune');
+
+let failures = 0;
+async function check(name, fn) {
+ try { await fn(); console.log(' ✓', name); }
+ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }
+}
+
+const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-net-analytics-'));
+const quietInit = (file) => { const log = console.log; console.log = () => {}; try { return initDb(file); } finally { console.log = log; } };
+const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
+const USERNAME = 'alexsecretname';
+
+/** Every value in every analytics table (network.db also holds accounts, which are not analytics). */
+function dumpAnalytics(db) {
+ const tables = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'analytics_%'").pluck().all();
+ assert.ok(tables.includes('analytics_events'));
+ return tables.map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');
+}
+
+(async () => {
+ await check('Network paths: usernames, anon tokens and project slugs become parameters', () => {
+ const t = (p) => privacy.normalisePath(networkAnalytics.preReducePath(p), networkAnalytics.PATH_OPTS);
+ const cases = {
+ '/avatar/alex?s=96': '/avatar/:param',
+ '/api/auth/anon/k3yF00barBaz?x=1': '/api/auth/anon/:param',
+ '/internal/users/by-username/alex': '/internal/users/:param/:username',
+ '/api/v1/projects/my-app/apps': '/api/v1/projects/:param/apps',
+ '/oauth/authorize?client_id=live&state=abc': '/oauth/authorize',
+ '/reset-password?token=abc': '/reset-password',
+ '/api/admin/users/42/role': '/api/admin/users/:param/role',
+ };
+ for (const [raw, want] of Object.entries(cases)) assert.strictEqual(t(raw), want, raw);
+ // Without Network's rule the shared normaliser would keep the name.
+ assert.strictEqual(privacy.normalisePath('/internal/users/by-username/alex'), '/internal/users/:param/alex');
+ });
+
+ // ── Real requests through Network's routers ──────────────
+ const dbFile = path.join(dir, 'network.db');
+ const db = quietInit(dbFile);
+ db.prepare("INSERT INTO users (id, username, password_hash, role) VALUES (42, ?, 'x', 'admin')").run(USERNAME);
+ let clock = Date.parse('2026-09-23T10:15:00Z');
+ const analytics = networkAnalytics.openAnalytics(dbFile, { timers: false, now: () => clock });
+
+ const keys = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
+ const ISSUER = 'https://openvibe.network';
+ const closed = 'http://127.0.0.1:9'; // remote dashboards: refused at once, never a local dev server
+ const config = {
+ internalKey: 'k'.repeat(32), baseUrl: ISSUER, loginUrl: ISSUER, networkUrl: ISSUER,
+ jwt: { issuer: ISSUER, accessTokenExpiry: '1h', refreshTokenExpiry: '30d' },
+ services: { live: { internalUrl: closed }, tools: { internalUrl: closed }, games: { internalUrl: closed }, media: { internalUrl: closed } },
+ };
+ const requireAuth = require('../server/auth/session').makeRequireAuth(() => ({ db, publicKey: keys.publicKey, config }), require('../server/auth/routes').signToken);
+ const avatarService = require('../server/profile/avatar').createAvatarService({ db, config, requireAuth, log: { log() {}, warn() {}, error() {} } });
+
+ const app = express();
+ app.set('trust proxy', 2); // as server/index.js: Cloudflare → Nginx → Node
+ app.use(cookieParser());
+ app.use(express.json());
+ app.use(analytics.middleware());
+ app.locals.db = db;
+ app.locals.config = config;
+ app.locals.privateKey = keys.privateKey;
+ app.locals.publicKey = keys.publicKey;
+ app.use('/api/auth', require('../server/auth/routes'));
+ app.use('/oauth', require('../server/auth/oauth-routes'));
+ // Stands in for the /avatar rate limiter: a request refused before any route matched.
+ app.use('/avatar', (req, res, next) => (req.query.block ? res.status(429).end() : next()), avatarService.pub);
+ app.use('/internal', (req, res) => res.status(403).json({ error: 'internal only' })); // refused before routing
+ app.use('/api/admin/analytics', require('../server/admin/analytics-routes')(analytics, requireAuth, config));
+ const server = http.createServer(app);
+ await new Promise((r) => server.listen(0, '127.0.0.1', r));
+ const base = `http://127.0.0.1:${server.address().port}`;
+ const token = jwt.sign({ sub: 42, id: 42, username: USERNAME, role: 'admin' }, keys.privateKey, { algorithm: 'RS256', issuer: ISSUER, expiresIn: '1h' });
+ const hdr = (extra = {}) => ({
+ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', 'cf-ipcountry': 'NL', 'cf-ipcity': 'Amsterdam',
+ referer: 'https://www.google.com/search?q=secret-query', ...extra,
+ });
+ const get = async (p, headers) => { const r = await fetch(base + p, { headers, redirect: 'manual' }); await r.arrayBuffer(); return r.status; };
+ const getJson = async (p, headers) => (await fetch(base + p, { headers })).json();
+
+ await check('real requests store no IP, user id, city, raw UA, full referer, query value or username', async () => {
+ assert.strictEqual(await get('/api/auth/me?token=supersecret', hdr({ authorization: `Bearer ${token}` })), 200);
+ assert.strictEqual(await get(`/avatar/${USERNAME}?s=96`, hdr()), 200);
+ assert.strictEqual(await get('/avatar/bobsecretname?block=1', hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' })), 429);
+ await get('/oauth/authorize?client_id=live&redirect_uri=https%3A%2F%2Fopenvibe.live%2Fcb&state=statesecret', hdr());
+ assert.strictEqual(await get('/api/auth/anon/0123456789abcdefanon', hdr()), 404);
+ await get('/no/such/page/424242?email=alex%40example.com', hdr());
+ assert.strictEqual(await get('/internal/users/by-username/carolsecretname', hdr()), 403);
+ await get('/api/auth/me', hdr({ 'user-agent': 'curl/8.5.0' }));
+ await new Promise((r) => setTimeout(r, 50));
+ analytics.flush();
+
+ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();
+ assert.strictEqual(rows.length, 8);
+ for (const r of rows) {
+ assert.strictEqual(r.service, 'openvibe-network');
+ assert.strictEqual(r.ip, null);
+ assert.strictEqual(r.user_id, null);
+ assert.strictEqual(r.city, null);
+ assert.ok(!/[?#]/.test(r.path), r.path);
+ assert.strictEqual(r.referer, 'https://www.google.com');
+ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id), r.session_id);
+ assert.strictEqual(r.country, 'NL');
+ assert.ok(/^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/.test(r.user_agent), r.user_agent);
+ }
+ assert.deepStrictEqual(rows.map((r) => r.path), [
+ '/api/auth/me', '/avatar/:username', '/avatar/:param', '/oauth/authorize',
+ '/api/auth/anon/:token', '/no/such/page/:id', '/internal/users/:param/:username', '/api/auth/me',
+ ]);
+ assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0, 0, 0, 0, 0, 0, 0]);
+ assert.strictEqual(rows[7].user_agent, 'bot:curl');
+ assert.strictEqual(rows[0].session_id, rows[1].session_id, 'same visitor, same session');
+ assert.notStrictEqual(rows[0].session_id, rows[2].session_id);
+
+ const everything = dumpAnalytics(db);
+ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', 'Amsterdam', 'supersecret', 'secret-query',
+ '/search', 'statesecret', 'openvibe.live%2Fcb', USERNAME, 'bobsecretname', 'carolsecretname', '0123456789abcdefanon', '424242',
+ 'alex%40example.com', '[email protected]', 'Mozilla/5.0', 'curl/8.5.0', 'Bearer', token.slice(0, 20)]) {
+ assert.ok(!everything.includes(needle), `found ${needle} in the analytics tables`);
+ }
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
+ });
+
+ await check('the tracker has its own connection: the identity connection keeps busy_timeout 5000', () => {
+ assert.notStrictEqual(analytics.db, db);
+ assert.strictEqual(db.pragma('busy_timeout', { simple: true }), 5000);
+ assert.strictEqual(analytics.db.pragma('busy_timeout', { simple: true }), 250);
+ assert.strictEqual(db.pragma('secure_delete', { simple: true }), 0);
+ });
+
+ await check('admin dashboards keep their shapes; bot rows carry ua_class / session_id, never an IP', async () => {
+ const auth = { authorization: `Bearer ${token}`, 'user-agent': CHROME };
+ analytics.aggregate();
+ const warn = console.warn;
+ console.warn = () => {}; // the remote services are unreachable here on purpose
+ try { await dashboards(auth); } finally { console.warn = warn; }
+ });
+ async function dashboards(auth) {
+ const svc = await getJson('/api/admin/analytics/service/openvibe-network?days=30', auth);
+ assert.ok(svc.ok, JSON.stringify(svc));
+ for (const k of ['summary', 'realtime', 'daily', 'hourly', 'topPages', 'authBreakdown', 'visitorTypes', 'authTrend']) assert.ok(k in svc.analytics, k);
+ assert.strictEqual(svc.analytics.visitorTypes.new_visitors, null);
+ const bots = (await getJson('/api/admin/analytics/bots?days=30', auth)).bots['openvibe-network'];
+ for (const k of ['topBotIPs', 'botTrend', 'botTypes', 'suspiciousIPs']) assert.ok(Array.isArray(bots[k]), k);
+ const curl = bots.topBotIPs.find((b) => b.ua_class === 'bot:curl');
+ assert.ok(curl, JSON.stringify(bots.topBotIPs));
+ assert.strictEqual(curl.ip, null);
+ assert.ok(bots.botTypes.every((t) => t.unique_ips === null && typeof t.unique_sessions === 'number'));
+ // /realtime reads the wall clock (last 5 minutes): one request stamped now.
+ clock = Date.now();
+ await get('/api/auth/me', hdr({ authorization: `Bearer ${token}` }));
+ await new Promise((r) => setTimeout(r, 50));
+ analytics.flush();
+ const rt = await getJson('/api/admin/analytics/realtime', auth);
+ assert.ok(rt.ok && typeof rt.realtime.visitors === 'number' && rt.realtime.visitors >= 1, JSON.stringify(rt));
+ const ov = await getJson('/api/admin/analytics/overview?days=30', auth);
+ assert.ok(ov.ok && ov.overview.services.some((s) => s.name === 'openvibe-network'));
+ const hourly = db.prepare("SELECT * FROM analytics_hourly WHERE hour = '2026-09-23 10:00:00'").get();
+ assert.ok(hourly && hourly.unique_visitors >= 2 && hourly.unique_users === 1, JSON.stringify(hourly));
+ }
+ server.close();
+
+ await check('analytics-prune job: raw rows older than 30 days go, rollups stay', async () => {
+ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, ip, user_id, created_at) VALUES ('openvibe-network', 'pageview', '/@old', '198.51.100.1', 7, ?)`);
+ for (let i = 0; i < 5; i++) ins.run(sqlTime(Date.now() - (40 + i) * 86400000));
+ ins.run(sqlTime(Date.now() - 29 * 86400000));
+ db.prepare("INSERT OR REPLACE INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors) VALUES ('openvibe-network', '2026-07-01', 9, 3, 4)").run();
+ const totals = retention.rollupTotals(db);
+ const before = db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get();
+ const logs = [];
+ const job = networkAnalytics.schedulePrune(analytics, { initialDelayMs: 1e9, intervalMs: 1e9, log: (l) => logs.push(l) });
+ try {
+ const out = await job.run();
+ assert.strictEqual(out.deleted, 5);
+ assert.ok(/pruned 5 raw events/.test(logs[0]), logs.join('\n'));
+ } finally { job.stop(); }
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), before - 5);
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM users').pluck().get(), 1);
+ });
+ analytics.destroy();
+ analytics.db.close();
+ db.close();
+
+ // ── CLI against a network.db ─────────────────────────────
+ const quiet = [];
+ const log = (l) => quiet.push(l);
+ const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
+ function legacyNetworkDb(name) {
+ const file = path.join(dir, name);
+ const d = quietInit(file);
+ d.prepare("INSERT INTO users (id, username, password_hash) VALUES (7, 'keepme', 'x')").run();
+ const t = networkAnalytics.openAnalytics(file, { timers: false });
+ t.destroy();
+ t.db.close();
+ const ins = d.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)
+ VALUES ('openvibe-network', 'pageview', ?, 'GET', 200, 7, ?, ?, 'Lyon', ?, 'https://t.co/abc?x=1', ?)`);
+ const nowMs = Date.now() + 120000;
+ for (let i = 0; i < 4; i++) ins.run('/avatar/keepme?s=' + i, 'legacy-session-' + i, '198.51.100.' + i, CHROME, sqlTime(nowMs - (45 + i) * 86400000));
+ for (let i = 0; i < 3; i++) ins.run('/api/auth/anon/tok' + i + 'X9zz99', 'abcdef012345678' + i, '198.51.100.2' + i, FIREFOX, sqlTime(nowMs - (2 + i) * 86400000));
+ ins.run('/internal/users/by-username/keepme', null, '127.0.0.1', 'node', sqlTime(nowMs - 3 * 86400000));
+ d.prepare(`INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers)
+ VALUES ('openvibe-network', '2026-07-01', 100, 40, 30, 10, 2, ?, ?)`)
+ .run(JSON.stringify([{ path: '/@keepme', cnt: 3 }, { path: '/internal/users/by-username/keepme', cnt: 2 }, { path: '/login', cnt: 1 }]), JSON.stringify([{ referer: 'https://www.google.com/search?q=keepme', cnt: 4 }]));
+ d.pragma('wal_checkpoint(TRUNCATE)');
+ d.close();
+ return file;
+ }
+ const snapshot = (file) => { const d = new Database(file, { readonly: true }); try { return dumpAnalytics(d); } finally { d.close(); } };
+
+ await check('CLI: default database is $DB_PATH, else data/network.db, from the repo root', () => {
+ const root = path.join(__dirname, '..');
+ assert.strictEqual(cli.defaultDbPath({}, {}), path.join(root, 'data', 'network.db'));
+ assert.strictEqual(cli.defaultDbPath({}, { DB_PATH: './data/other.db' }), path.join(root, 'data', 'other.db'));
+ assert.strictEqual(cli.defaultDbPath({ db: '/x/y.db' }, { DB_PATH: './data/other.db' }), '/x/y.db');
+ });
+
+ await check('CLI dry run (with and without --scrub) changes nothing', async () => {
+ const file = legacyNetworkDb('cli-dry.db');
+ const before = { hash: sha(file), dump: snapshot(file) };
+ assert.strictEqual(await cli.main(['--db', file], log), 0);
+ assert.strictEqual(await cli.main(['--db', file, '--scrub', '--days', '7'], log), 0);
+ assert.strictEqual(sha(file), before.hash);
+ assert.strictEqual(snapshot(file), before.dump);
+ assert.ok(quiet.some((l) => /prune\s+4 rows/.test(l)), quiet.join('\n'));
+ });
+
+ await check('CLI --apply refuses without a backup choice, an existing target, or days > 30', async () => {
+ const file = legacyNetworkDb('cli-refuse.db');
+ const before = snapshot(file);
+ assert.strictEqual(await cli.main(['--db', file, '--apply'], log), 2);
+ const existing = path.join(dir, 'exists.bak');
+ fs.writeFileSync(existing, 'x');
+ assert.strictEqual(await cli.main(['--db', file, '--apply', '--backup', existing], log), 2);
+ assert.strictEqual(await cli.main(['--db', file, '--apply', '--no-backup', '--days', '31'], log), 2);
+ assert.strictEqual(snapshot(file), before);
+ });
+
+ await check('CLI --apply --scrub --backup: owner-only verified backup, analytics reduced, accounts untouched', async () => {
+ const file = legacyNetworkDb('cli-apply.db');
+ const bak = path.join(dir, 'cli-apply.backup.db');
+ assert.strictEqual(await cli.main(['--db', file, '--apply', '--scrub', '--backup', bak, '--batch', '2'], log), 0);
+ assert.strictEqual(fs.statSync(bak).mode & 0o777, 0o600);
+ const b = new Database(bak, { readonly: true });
+ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 8, 'backup has every row');
+ b.close();
+ const d = new Database(file, { readonly: true });
+ assert.strictEqual(d.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 4);
+ assert.deepStrictEqual(d.prepare('SELECT DISTINCT path FROM analytics_events ORDER BY path').pluck().all(), ['/api/auth/anon/:param', '/internal/users/:param/:username']);
+ assert.deepStrictEqual(JSON.parse(d.prepare('SELECT top_paths FROM analytics_daily').pluck().get()),
+ [{ path: '/@:user', cnt: 3 }, { path: '/internal/users/:param/:username', cnt: 2 }, { path: '/login', cnt: 1 }]);
+ assert.strictEqual(d.prepare("SELECT username FROM users WHERE id = 7").pluck().get(), 'keepme');
+ d.close();
+ const all = snapshot(file);
+ for (const needle of ['198.51.100.', '127.0.0.1', 'Lyon', 'legacy-session', 'keepme', 'X9zz99', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);
+ });
+
+ fs.rmSync(dir, { recursive: true, force: true });
+ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }
+ console.log('\nanalytics privacy: all passed');
+ process.exit(0);
+})();