openvibers/openvibe.tools Commits
Analytics within ADR-021 on every satellite: apps/_shared/analytics replaces the openvibe-shared tracker (no IP, user id or city in raw events; route templates, referer origins, user-agent classes, rotating session ids); raw events pruned after 30 days on a 24 h timer, rollups kept
Changes
@@@ -22,7 +22,9 @@ apps/├── audio # Audio.OpenVibe — audio converter & effects├── text # Text.OpenVibe + Logo.OpenVibe — text generators & logo makers└── docs # Docs.OpenVibe — PDF & document tools--apps/_shared # code the apps require by relative path: host roles, internal auth, the job runtime (jobs/)++apps/_shared # code the apps require by relative path: host roles, internal auth, the job runtime (jobs/),++ # visit analytics within ADR-021 (analytics/)++scripts/analytics-prune.js # operator CLI: raw analytics retention + one-time scrub (dry run by default)(openvibe-shared is a pinned OpenVibe.Shared release in each app's package.json; no vendor/ copy)```@@@ -154,6 +156,31 @@ Media results with a stand-in Media), img/audio/docs as real processes killed wihosts on every satellite, and the registry-driven catalog. The audio test needs `ffmpeg` and skips without it.Install first with `npm run install:all`.++## Analytics (ADR-021)++++Every satellite except the gateway keeps visit analytics in its own `data/analytics.db` through++`apps/_shared/analytics` (the same module as OpenVibe.Live's `server/analytics/`). Bound by ADR-021++(OpenVibe.Contracts `docs/adr/ADR-021-analytics.md`):++++- **A raw row carries** event type, service, route template (matched Express route, else a normaliser: no++ query string, ids/hashes → `:id`, the segment after `watch`, `jobs`, `recipe`, `place`, … → `:param`,++ `/@x` → `/@:user`), method, status, response time, a rotating session id, country (CDN header),++ user-agent class (`chrome/windows/desktop`, `bot:googlebot`) + browser/os/device, referer origin, bot++ flags, a signed-in flag, timestamp. **Never** an IP, a user id, a city, the UA string or a full referer++ (`ip`/`user_id`/`city` stay as always-NULL columns for compatibility).++- **Session id:** random, in memory against the visitor hash, new after 30 idle minutes and at UTC midnight.++- **Bot rate check:** per-IP counters in memory only (current + previous minute); `analytics_rate_tracking`++ is emptied at boot and no longer written.++- **Unique visitors:** `HMAC-SHA256(day salt, ip + "\n" + ua)` (16 hex chars), random salt per UTC day. Hashes++ and the salt are kept only until the first hourly aggregation after their day ends (right after the++ day's final rollup), never in raw rows. Rollups keep counts; sub-48 h raw summaries count sessions.++- **Retention:** each satellite prunes raw rows older than 30 days in batches of 5000, 5 minutes after boot++ and every 24 h after; rollups stay.++- **CLI:** `node scripts/analytics-prune.js` (dry run over every `apps/*/data/analytics.db`; `--app`, `--db`++ to narrow). `--apply` needs `--backup <file|dir>` (verified online backup; a directory when several++ databases are targeted) or `--no-backup`; `--scrub` also rewrites rows written before ADR-021 and the++ rollups' top lists (counts unchanged). Ends with VACUUM unless `--no-vacuum`.++## YouTube downloader: when YouTube refuses the serverYouTube blocks some datacentre addresses outright ("Sign in to confirm you're not a bot"). The appnnew file mode 100644@@@ -0,0 +1,8 @@++'use strict';++// Request analytics within ADR-021 (see tracker.js, privacy.js, retention.js).++const { AnalyticsTracker } = require('./tracker');++const privacy = require('./privacy');++const retention = require('./retention');++const { ensureSchema } = require('./schema');++++module.exports = { AnalyticsTracker, privacy, retention, ensureSchema };nnew file mode 100644@@@ -0,0 +1,211 @@++'use strict';++/**++ * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):++ * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a++ * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a++ * precise location or a user/subject id.++ *++ * This file holds the pure reducers that turn a request into those fields. It has no dependencies,++ * so the same file serves OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);++ * keep the two copies identical.++ */++++// ── Bots ──────────────────────────────────────────────────────++++const BOT_USER_AGENTS = [++ /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,++ /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,++ /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,++ /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,++ /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,++ /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,++ /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,++ /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,++ /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,++ /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,++ /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,++ /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,++ /statuscake/i, /monitoring/i, /health.?check/i,++];++++const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];++++/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */++const HIGH_REQUEST_RATE = 60;++++/** The family name of the first bot pattern the user agent matches, or null. */++function botName(ua) {++ if (!ua) return null;++ for (const re of BOT_USER_AGENTS) {++ if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';++ }++ return null;++}++++/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */++function classifyRequest(req) {++ const ua = (req.headers && req.headers['user-agent']) || '';++ if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };++ if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };++ const path = String(req.path || req.url || '').toLowerCase();++ if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };++ return { isBot: false, botType: null, confidence: 0.1 };++}++++// ── User agent → class ───────────────────────────────────────++++function parseUserAgent(ua) {++ if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };++ let device = 'desktop';++ if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';++ else if (/ipad|tablet/i.test(ua)) device = 'tablet';++++ let browser = 'other';++ if (/edg\//i.test(ua)) browser = 'edge';++ else if (/opr\/|opera/i.test(ua)) browser = 'opera';++ else if (/firefox\//i.test(ua)) browser = 'firefox';++ else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';++ else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';++ else if (/trident|msie/i.test(ua)) browser = 'ie';++++ let os = 'other';++ if (/windows/i.test(ua)) os = 'windows';++ else if (/macintosh|mac os/i.test(ua)) os = 'macos';++ else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';++ else if (/android/i.test(ua)) os = 'android';++ else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';++++ return { device, browser, os };++}++++const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;++++/**++ * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,++ * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps++ * to itself), so a scrub can run over rows that are already reduced.++ */++function uaClass(ua) {++ if (ua == null || ua === '') return 'none';++ const s = String(ua);++ if (UA_CLASS_RE.test(s)) return s;++ const bot = botName(s);++ if (bot) return `bot:${bot}`;++ const p = parseUserAgent(s);++ return `${p.browser}/${p.os}/${p.device}`;++}++++// ── Referer → origin ─────────────────────────────────────────++++/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */++function refererOrigin(ref) {++ if (!ref) return null;++ try {++ const u = new URL(String(ref));++ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;++ return u.origin;++ } catch {++ return null;++ }++}++++// ── Country ──────────────────────────────────────────────────++++/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */++function countryCode(v) {++ const s = String(v || '').trim().toUpperCase();++ return /^[A-Z]{2}$/.test(s) ? s : null;++}++++// ── Paths → route templates ──────────────────────────────────++++/**++ * Words after which the next path segment is a parameter even when it looks like a word++ * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.++ */++const DEFAULT_PARAM_PREFIXES = [++ 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',++ 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',++ 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',++ 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',++ 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',++];++++const MAX_SEGMENTS = 8;++const MAX_TEMPLATE_LENGTH = 200;++const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;++const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;++++/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */++function looksLikeId(s) {++ if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…++ if (/^\d+$/.test(s)) return true; // 123++ if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid++ if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes++ if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…++ if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid++ if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers++ if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names++ if (s.length > 32) return true;++ return false;++}++++/**++ * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by++ * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated++ * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.++ */++function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES } = {}) {++ if (raw == null || raw === '') return '/';++ let p = String(raw);++ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {++ try { p = new URL(p).pathname; } catch { return '/'; }++ }++ p = p.split(/[?#]/)[0];++ const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);++ const segs = p.split('/').filter(Boolean);++ const out = [];++ let expectParam = false;++ for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {++ let s = segs[i];++ try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }++ if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }++ if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }++ if (expectParam) { out.push(':param'); expectParam = false; continue; }++ if (looksLikeId(s)) { out.push(':id'); continue; }++ out.push(s);++ expectParam = prefixes.has(s.toLowerCase());++ }++ if (segs.length > MAX_SEGMENTS) out.push('*');++ const t = '/' + out.join('/');++ return t.length > MAX_TEMPLATE_LENGTH ? t.slice(0, MAX_TEMPLATE_LENGTH) : t;++}++++/**++ * The route template for a finished request: the matched Express route (mount path + route path)++ * when there is one, else the normalised original path. Both go through normalisePath, because a++ * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.++ */++function routeTemplate(req, fallbackPath, opts) {++ const r = req && req.route;++ if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {++ return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);++ }++ return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);++}++++module.exports = {++ BOT_USER_AGENTS,++ HONEYPOT_PATHS,++ HIGH_REQUEST_RATE,++ DEFAULT_PARAM_PREFIXES,++ botName,++ classifyRequest,++ parseUserAgent,++ uaClass,++ refererOrigin,++ countryCode,++ looksLikeId,++ normalisePath,++ routeTemplate,++};nnew file mode 100644@@@ -0,0 +1,209 @@++'use strict';++/**++ * Raw analytics retention and the one-time scrub (ADR-021).++ *++ * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded++ * batches, yielding to the event loop between batches. Rollups++ * (analytics_hourly / analytics_daily) are never touched. Also clears++ * leftovers: visitor hashes older than yesterday, salts of past days,++ * and analytics_rate_tracking (IP counters live in memory now).++ * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →++ * NULL (authenticated = 1 where a user id was present), path → route++ * template, referer → origin, user_agent → class, session ids that are++ * not rotating ids → NULL.++ * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /++ * top_referers JSON (counts merged, never changed).++ * inspect(db, { days }) read-only counts for a dry run.++ * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.++ *++ * Used by the server (scheduled prune) and scripts/analytics-prune.js (CLI).++ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).++ */++++const { ensureSchema } = require('./schema');++const privacy = require('./privacy');++++const MAX_DAYS = 30;++const DEFAULT_BATCH = 5000;++const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;++++const pad = (n) => String(n).padStart(2, '0');++function sqlTime(ms) {++ const d = new Date(ms);++ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;++}++++function checkDays(days) {++ const n = Number(days);++ if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {++ throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);++ }++ return n;++}++++/** The created_at bound: rows strictly older than this are pruned. */++function cutoffFor(days, nowMs = Date.now()) {++ return sqlTime(nowMs - checkDays(days) * 86400000);++}++++const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);++const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);++const tick = () => new Promise((r) => setImmediate(r));++++/**++ * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write++ * transaction). Returns { cutoff, deleted, batches, complete }.++ */++async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {++ const cutoff = cutoffFor(days, now());++ const out = { cutoff, deleted: 0, batches: 0, complete: true };++ if (!tableExists(db, 'analytics_events')) return out;++ const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');++ for (;;) {++ if (out.batches >= maxBatches) { out.complete = false; break; }++ const n = del.run(cutoff, batchSize).changes;++ out.batches++;++ out.deleted += n;++ if (n < batchSize) break;++ await tick();++ }++ const today = sqlTime(now()).slice(0, 10);++ const yesterday = sqlTime(now() - 86400000).slice(0, 10);++ if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);++ if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);++ if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();++ return out;++}++++/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */++function registerFunctions(db, opts) {++ db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));++ db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));++ db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));++ db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));++}++++/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */++async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes } = {}) {++ ensureSchema(db);++ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);++ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();++ const out = { rows: 0, batches: 0 };++ if (lo == null) return out;++ const upd = db.prepare(`++ UPDATE analytics_events SET++ authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,++ ip = NULL, user_id = NULL, city = NULL,++ session_id = ov_rotating_session(session_id),++ path = ov_route_template(path),++ referer = ov_referer_origin(referer),++ user_agent = ov_ua_class(user_agent)++ WHERE id >= ? AND id < ?++ `);++ for (let from = lo; from <= hi; from += batchSize) {++ out.rows += upd.run(from, from + batchSize).changes;++ out.batches++;++ await tick();++ }++ return out;++}++++/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */++function reduceTopList(json, key, fn) {++ let list;++ try { list = JSON.parse(json); } catch { return null; }++ if (!Array.isArray(list)) return null;++ const merged = new Map();++ for (const item of list) {++ if (!item || typeof item !== 'object') continue;++ const k = fn(item[key]);++ if (k == null) continue;++ merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));++ }++ const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);++ const text = JSON.stringify(next);++ return text === json ? null : text;++}++++/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */++function scrubRollups(db, { paramPrefixes } = {}) {++ const opts = paramPrefixes ? { paramPrefixes } : undefined;++ const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));++ const out = { hourly: 0, daily: 0 };++ for (const table of ['analytics_hourly', 'analytics_daily']) {++ if (!tableExists(db, table)) continue;++ const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();++ const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);++ db.transaction(() => {++ for (const r of rows) {++ const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;++ const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;++ if (paths == null && refs == null) continue;++ upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);++ out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;++ }++ })();++ }++ return out;++}++++/** Sums of every rollup counter; equal before and after a prune or scrub. */++function rollupTotals(db) {++ const out = {};++ if (tableExists(db, 'analytics_hourly')) {++ out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,++ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,++ TOTAL(error_count) AS error_count FROM analytics_hourly`).get();++ }++ if (tableExists(db, 'analytics_daily')) {++ out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,++ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,++ TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();++ }++ return out;++}++++/** Read-only counts for a dry run. */++function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes } = {}) {++ const cutoff = cutoffFor(days, now());++ const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };++ const pageSize = db.pragma('page_size', { simple: true });++ out.bytes = db.pragma('page_count', { simple: true }) * pageSize;++ out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;++ out.rollups = rollupTotals(db);++ if (!tableExists(db, 'analytics_events')) return out;++ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);++ const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();++ out.events = e.n;++ out.oldest = e.oldest;++ out.newest = e.newest;++ out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);++ out.remaining = out.events - out.older;++ const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))++ .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';++ const r = db.prepare(`++ SELECT++ COUNT(*) FILTER (WHERE ${personal}) AS personal,++ COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,++ COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,++ COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,++ COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions++ FROM analytics_events WHERE created_at >= ?++ `).get(cutoff);++ out.toScrub = r;++ out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;++ return out;++}++++module.exports = {++ MAX_DAYS,++ DEFAULT_BATCH,++ checkDays,++ cutoffFor,++ pruneRawEvents,++ scrubEvents,++ scrubRollups,++ rollupTotals,++ inspect,++ reduceTopList,++};nnew file mode 100644@@@ -0,0 +1,123 @@++'use strict';++/**++ * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the++ * existing analytics.db files and the admin dashboards keep working; the columns that used to hold++ * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and++ * user_agent / referer / path hold the reduced forms (class, origin, route template).++ *++ * Added:++ * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)++ * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;++ * rows are deleted once the day's rollup is final (≤ ~25 h)++ * analytics_day_salts the random salt of the current UTC day (so a restart does not++ * double-count); deleted together with that day's hashes++ * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)++ */++++const ANALYTICS_SCHEMA = `++ CREATE TABLE IF NOT EXISTS analytics_events (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ event_type TEXT NOT NULL DEFAULT 'pageview',++ path TEXT,++ method TEXT DEFAULT 'GET',++ status_code INTEGER,++ response_time_ms INTEGER,++ user_id INTEGER,++ session_id TEXT,++ ip TEXT,++ country TEXT,++ city TEXT,++ user_agent TEXT,++ referer TEXT,++ is_bot INTEGER DEFAULT 0,++ bot_type TEXT,++ device_type TEXT,++ browser TEXT,++ os TEXT,++ created_at DATETIME DEFAULT CURRENT_TIMESTAMP++ );++++ CREATE TABLE IF NOT EXISTS analytics_hourly (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ hour TEXT NOT NULL,++ pageviews INTEGER DEFAULT 0,++ api_calls INTEGER DEFAULT 0,++ unique_visitors INTEGER DEFAULT 0,++ unique_users INTEGER DEFAULT 0,++ bot_hits INTEGER DEFAULT 0,++ avg_response_ms INTEGER DEFAULT 0,++ error_count INTEGER DEFAULT 0,++ bandwidth_bytes INTEGER DEFAULT 0,++ top_paths TEXT,++ top_referers TEXT,++ UNIQUE(service, hour)++ );++++ CREATE TABLE IF NOT EXISTS analytics_daily (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ date TEXT NOT NULL,++ pageviews INTEGER DEFAULT 0,++ api_calls INTEGER DEFAULT 0,++ unique_visitors INTEGER DEFAULT 0,++ unique_users INTEGER DEFAULT 0,++ new_users INTEGER DEFAULT 0,++ bot_hits INTEGER DEFAULT 0,++ avg_response_ms INTEGER DEFAULT 0,++ error_count INTEGER DEFAULT 0,++ top_paths TEXT,++ top_referers TEXT,++ top_countries TEXT,++ device_breakdown TEXT,++ browser_breakdown TEXT,++ UNIQUE(service, date)++ );++++ CREATE TABLE IF NOT EXISTS analytics_rate_tracking (++ ip TEXT NOT NULL,++ window_start INTEGER NOT NULL,++ hit_count INTEGER DEFAULT 1,++ PRIMARY KEY (ip, window_start)++ );++++ CREATE TABLE IF NOT EXISTS analytics_visitor_days (++ service TEXT NOT NULL,++ day TEXT NOT NULL,++ hour TEXT NOT NULL,++ vhash TEXT NOT NULL,++ authenticated INTEGER NOT NULL DEFAULT 0,++ PRIMARY KEY (service, hour, vhash)++ );++++ CREATE TABLE IF NOT EXISTS analytics_day_salts (++ service TEXT NOT NULL,++ day TEXT NOT NULL,++ salt TEXT NOT NULL,++ PRIMARY KEY (service, day)++ );++++ CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);++ CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);++ CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);++++ -- Indexes over the retired personal columns.++ DROP INDEX IF EXISTS idx_analytics_events_ip;++ DROP INDEX IF EXISTS idx_analytics_events_user;++`;++++/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */++function ensureSchema(db) {++ db.exec(ANALYTICS_SCHEMA);++ const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);++ if (!cols.includes('authenticated')) {++ db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');++ }++}++++module.exports = { ANALYTICS_SCHEMA, ensureSchema };nnew file mode 100644@@@ -0,0 +1,728 @@++'use strict';++/**++ * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the openvibe-shared++ * AnalyticsTracker (same tables, same getStats/getOverview/getBotAnalysis shapes), minus the personal++ * data:++ *++ * raw row service, event_type, route template, method, status, response time, rotating session++ * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,++ * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.++ * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after++ * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.++ * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped++ * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).++ * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.++ * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as++ * the day is over); hashes go to analytics_visitor_days only, never to raw events,++ * and are deleted by the first aggregation after their day ends, right after that day's++ * final hourly/daily rollup — so no hash or salt outlives its day by more than about++ * one aggregation interval (1 h). Rollups keep only the counts.++ *++ * Retention of raw events (30 days) is ./retention.js; this tracker only writes and rolls up.++ *++ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).++ */++++const crypto = require('crypto');++const { ensureSchema } = require('./schema');++const privacy = require('./privacy');++const retention = require('./retention');++++const SESSION_IDLE_MS = 30 * 60 * 1000;++const MAX_TRACKED_CLIENTS = 200000;++const MAX_BUFFER = 5000;++const SESSION_ID_RE = /^[0-9a-f]{16}$/;++++const pad = (n) => String(n).padStart(2, '0');++/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */++function sqlTime(ms) {++ const d = new Date(ms);++ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;++}++const dayOf = (ms) => sqlTime(ms).slice(0, 10);++const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';++const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');++++function isStaticOrHealth(path) {++ return path === '/api/health' || path === '/health' ||++ /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);++}++++class AnalyticsTracker {++ /**++ * @param {object} db better-sqlite3 handle (its own analytics database)++ * @param {string} service service name ('live', 'openvibe-yt', …)++ * @param {object} [opts]++ * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter++ * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:++ * run the raw-event prune on a timer (Tools). Live passes false and schedules++ * retention.pruneRawEvents through server/utils/jobs.js instead.++ * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)++ * @param {function} [opts.now] clock (tests)++ */++ constructor(db, service, opts = {}) {++ this.db = db;++ this.service = service;++ this._now = opts.now || Date.now;++ this._pathOpts = opts.paramPrefixes++ ? { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...opts.paramPrefixes]) }++ : undefined;++ this._buffer = [];++ this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }++ this._sessions = new Map(); // vhash -> { sid, last }++ this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)++ this._saltDay = null;++ this._salt = null;++ this._timers = [];++++ ensureSchema(db);++ try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }++ try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }++ db.exec('DELETE FROM analytics_rate_tracking');++++ this._insertEvent = db.prepare(`++ INSERT INTO analytics_events++ (service, event_type, path, method, status_code, response_time_ms,++ user_id, session_id, ip, country, city, user_agent, referer,++ is_bot, bot_type, device_type, browser, os, authenticated, created_at)++ VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ `);++ this._upsertVisitor = db.prepare(`++ INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)++ ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)++ `);++ this._write = db.transaction((events, visitors) => {++ for (const e of events) {++ this._insertEvent.run(++ e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,++ e.session_id, e.country, e.user_agent, e.referer,++ e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,++ );++ }++ for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);++ });++++ if (opts.timers !== false) {++ const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };++ const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };++ every(() => { this.flush(); this._sweep(); }, 5000);++ every(() => this.aggregate(), 60 * 60 * 1000);++ once(() => this.aggregate(), 10_000);++ if (opts.retention) {++ const r = opts.retention;++ const run = () => retention.pruneRawEvents(db, { days: r.days || retention.MAX_DAYS })++ .then((out) => { if (out.deleted) console.log(`[Analytics:${service}] pruned ${out.deleted} raw events older than ${out.cutoff}`); })++ .catch((err) => console.error(`[Analytics:${service}] prune error:`, err.message));++ once(run, r.initialDelayMs != null ? r.initialDelayMs : 5 * 60 * 1000);++ every(run, r.intervalMs || 24 * 60 * 60 * 1000);++ }++ }++ }++++ // ── Per-request reducers ──────────────────────────────────++++ _daySalt(day) {++ if (this._saltDay === day) return this._salt;++ let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);++ if (!row) {++ this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')++ .run(this.service, day, crypto.randomBytes(32).toString('hex'));++ row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);++ }++ // Earlier days' salts are never needed again: their hashes are already written.++ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);++ this._saltDay = day;++ this._salt = row.salt;++ this._sessions.clear();++ return this._salt;++ }++++ _visitorHash(ip, ua, day) {++ return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);++ }++++ _sessionId(vhash, nowMs) {++ let s = this._sessions.get(vhash);++ if (!s || nowMs - s.last > SESSION_IDLE_MS) {++ s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };++ this._sessions.set(vhash, s);++ }++ s.last = nowMs;++ return s.sid;++ }++++ /** Hits from this client in the current and previous minute (memory only). */++ _rateHit(ip, nowMs) {++ const minute = Math.floor(nowMs / 60000);++ let r = this._rates.get(ip);++ if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }++ if (r.minute !== minute) {++ r.prev = r.minute === minute - 1 ? r.count : 0;++ r.count = 0;++ r.minute = minute;++ }++ r.count++;++ return r.count + r.prev;++ }++++ _sweep() {++ const nowMs = this._now();++ const minute = Math.floor(nowMs / 60000);++ for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);++ for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);++ if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();++ if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();++ }++++ _push(event, visitor) {++ if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();++ this._buffer.push(event);++ if (visitor) {++ const key = `${visitor.hour}|${visitor.vhash}`;++ const prev = this._visitors.get(key);++ if (!prev) this._visitors.set(key, visitor);++ else if (visitor.authenticated) prev.authenticated = true;++ }++ if (this._buffer.length >= 100) this.flush();++ }++++ /**++ * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted++ * router is relative by the time the response finishes).++ */++ record(req, res, entryPath, responseTimeMs) {++ const nowMs = this._now();++ const headers = req.headers || {};++ const ua = String(headers['user-agent'] || '');++ const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');++ const bot = privacy.classifyRequest({ headers, path: entryPath });++ if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {++ bot.isBot = true;++ bot.botType = 'rate_limit';++ bot.confidence = 0.80;++ }++ const parsed = privacy.parseUserAgent(ua);++ const day = dayOf(nowMs);++ const vhash = this._visitorHash(ip, ua, day);++ const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));++ const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');++ this._push({++ service: this.service,++ event_type: isApi ? 'api_call' : 'pageview',++ path: privacy.routeTemplate(req, entryPath, this._pathOpts),++ method: req.method,++ status_code: res.statusCode,++ response_time_ms: responseTimeMs,++ session_id: this._sessionId(vhash, nowMs),++ country: privacy.countryCode(headers['cf-ipcountry']),++ user_agent: privacy.uaClass(ua),++ referer: privacy.refererOrigin(headers.referer || headers.referrer),++ is_bot: bot.isBot,++ bot_type: bot.botType,++ device_type: parsed.device,++ browser: parsed.browser,++ os: parsed.os,++ authenticated,++ created_at: sqlTime(nowMs),++ }, { day, hour: hourOf(nowMs), vhash, authenticated });++ }++++ /** Express middleware: one row per finished request (health checks and static assets skipped). */++ middleware() {++ return (req, res, next) => {++ const entryPath = req.path || '';++ if (isStaticOrHealth(entryPath)) return next();++ const started = Date.now();++ let done = false;++ const finish = () => {++ if (done) return;++ done = true;++ try { this.record(req, res, entryPath, Date.now() - started); }++ catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }++ };++ res.once('finish', finish);++ res.once('close', finish);++ next();++ };++ }++++ /**++ * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,++ * referer and user_agent are reduced like a request's; session_id is kept only if it is one of++ * this tracker's rotating ids.++ */++ trackEvent(eventType, data = {}) {++ const nowMs = this._now();++ const ua = data.user_agent ? String(data.user_agent) : '';++ const parsed = ua ? privacy.parseUserAgent(ua) : {};++ this._push({++ service: this.service,++ event_type: String(eventType || 'event').slice(0, 64),++ path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,++ method: data.method || null,++ status_code: data.status_code || null,++ response_time_ms: data.response_time_ms || null,++ session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,++ country: privacy.countryCode(data.country),++ user_agent: ua ? privacy.uaClass(ua) : null,++ referer: privacy.refererOrigin(data.referer),++ is_bot: !!data.is_bot,++ bot_type: data.bot_type || null,++ device_type: data.device_type || parsed.device || null,++ browser: data.browser || parsed.browser || null,++ os: data.os || parsed.os || null,++ authenticated: !!data.authenticated,++ created_at: sqlTime(nowMs),++ }, null);++ }++++ /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */++ flush() {++ if (!this._buffer.length && !this._visitors.size) return;++ const events = this._buffer.splice(0);++ const visitors = [...this._visitors.values()];++ this._visitors.clear();++ try {++ this._write(events, visitors);++ } catch (err) {++ console.error(`[Analytics:${this.service}] Flush error:`, err.message);++ this._buffer = events.slice(-MAX_BUFFER);++ for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);++ }++ }++++ // ── Rollups ───────────────────────────────────────────────++++ /**++ * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has++ * visitor hashes (finalised here), after which those hashes are deleted.++ */++ aggregate() {++ try {++ this.flush();++ const nowMs = this._now();++ const today = dayOf(nowMs);++ const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);++ const days = new Set([today, dayOf(nowMs - 86400000)]);++ const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);++ for (const p of pending) { hours.add(p.hour); days.add(p.day); }++ for (const h of hours) this._aggregateHour(h);++ for (const d of days) this._aggregateDay(d);++ this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);++ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);++ } catch (err) {++ console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);++ }++ }++++ _aggregateHour(hourStart) {++ const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);++ const svc = this.service;++ const stats = this.db.prepare(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count++ FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ?++ `).get(svc, hourStart, nextHour);++ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;++++ // Uniques only grow within a period, so never lower a stored count (restarts, or rows++ // counted before this tracker when the old one still stored IPs).++ const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);++ const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};++++ const topPaths = this.db.prepare(`++ SELECT path, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0++ GROUP BY path ORDER BY cnt DESC LIMIT 10++ `).all(svc, hourStart, nextHour);++ const topReferers = this.db.prepare(`++ SELECT referer, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0++ GROUP BY referer ORDER BY cnt DESC LIMIT 10++ `).all(svc, hourStart, nextHour);++++ this.db.prepare(`++ INSERT INTO analytics_hourly++ (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)++ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ ON CONFLICT(service, hour) DO UPDATE SET++ pageviews = excluded.pageviews, api_calls = excluded.api_calls,++ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,++ bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,++ error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers++ `).run(++ svc, hourStart, stats.pageviews, stats.api_calls,++ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),++ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,++ JSON.stringify(topPaths), JSON.stringify(topReferers),++ );++ }++++ _aggregateDay(date) {++ const dayStart = date + ' 00:00:00';++ const dayEnd = date + ' 23:59:59';++ const svc = this.service;++ const q = (sql) => this.db.prepare(sql);++ const stats = q(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count++ FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at <= ?++ `).get(svc, dayStart, dayEnd);++ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;++++ const v = q(`++ SELECT COUNT(DISTINCT vhash) AS visitors,++ COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users++ FROM analytics_visitor_days WHERE service = ? AND day = ?++ `).get(svc, date);++ const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};++++ const range = [svc, dayStart, dayEnd];++ const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);++ const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);++ const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);++ const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);++ const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);++++ q(`++ INSERT INTO analytics_daily++ (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,++ avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)++ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ ON CONFLICT(service, date) DO UPDATE SET++ pageviews = excluded.pageviews, api_calls = excluded.api_calls,++ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,++ new_users = excluded.new_users, bot_hits = excluded.bot_hits,++ avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,++ top_paths = excluded.top_paths, top_referers = excluded.top_referers,++ top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,++ browser_breakdown = excluded.browser_breakdown++ `).run(++ svc, date, stats.pageviews, stats.api_calls,++ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),++ // New users needs a user id history, which analytics no longer has: keep what was++ // recorded before, NULL (not measured) from now on.++ prev.new_users != null ? prev.new_users : null,++ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,++ JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),++ JSON.stringify(devices), JSON.stringify(browsers),++ );++ }++++ // ── Reads (admin dashboards) ─────────────────────────────++ // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,++ // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based++ // numbers (daily/hourly unique_visitors) are true unique visitors per period.++++ getStats(options = {}) {++ const { days = 30, hours, service } = options;++ const svc = service || this.service;++ const nowMs = this._now();++ const rangeMs = hours ? hours * 3600000 : days * 86400000;++ const rangeCutoff = sqlTime(nowMs - rangeMs);++ const isSubDay = hours && hours < 24;++ const effectiveDays = hours ? hours / 24 : days;++ const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;++ const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;++ const q = (sql) => this.db.prepare(sql);++ const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };++++ let daily = [];++ if (!isSubDay) {++ daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));++ }++++ const hourlyLimit = hours ? Math.min(hours, 168) : 24;++ const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);++ const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);++++ let timeBuckets = [];++ if (isSubDay) {++ timeBuckets = q(`++ SELECT ${bucketExpr} as bucket,++ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms++ FROM analytics_events++ WHERE service = ? AND created_at >= ?++ GROUP BY bucket ORDER BY bucket ASC++ `).all(svc, rangeCutoff);++ }++++ let summary;++ if (hours && hours < 48) {++ summary = q(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,++ COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms++ FROM analytics_events++ WHERE service = ? AND created_at >= ?++ `).get(svc, rangeCutoff);++ } else {++ summary = q(`++ SELECT++ SUM(pageviews) as total_pageviews,++ SUM(api_calls) as total_api_calls,++ SUM(unique_visitors) as total_unique_visitors,++ SUM(unique_users) as total_unique_users,++ SUM(bot_hits) as total_bot_hits,++ SUM(error_count) as total_errors,++ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms++ FROM analytics_daily++ WHERE service = ? AND date >= ?++ `).get(svc, dayOf(nowMs - rangeMs));++ }++++ const realtime = q(`++ SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots++ FROM analytics_events WHERE service = ? AND created_at >= ?++ `).get(svc, sqlTime(nowMs - 300000));++++ const human = [svc, rangeCutoff];++ const topPages = q(`++ SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY path ORDER BY hits DESC LIMIT 20++ `).all(...human);++ const topReferers = q(`++ SELECT referer, COUNT(*) as hits FROM analytics_events++ WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0++ GROUP BY referer ORDER BY hits DESC LIMIT 15++ `).all(...human);++ const botBreakdown = q(`++ SELECT bot_type, COUNT(*) as hits FROM analytics_events++ WHERE service = ? AND created_at >= ? AND is_bot = 1++ GROUP BY bot_type ORDER BY hits DESC LIMIT 15++ `).all(...human);++ const statusCodes = q(`++ SELECT CASE++ WHEN status_code >= 200 AND status_code < 300 THEN '2xx'++ WHEN status_code >= 300 AND status_code < 400 THEN '3xx'++ WHEN status_code >= 400 AND status_code < 500 THEN '4xx'++ WHEN status_code >= 500 THEN '5xx'++ ELSE 'other' END as group_code,++ COUNT(*) as cnt++ FROM analytics_events WHERE service = ? AND created_at >= ?++ GROUP BY group_code ORDER BY cnt DESC++ `).all(...human);++ const breakdown = (col) => q(`++ SELECT ${col}, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY ${col} ORDER BY cnt DESC++ `).all(...human);++ const deviceBreakdown = breakdown('device_type');++ const browserBreakdown = breakdown('browser');++ const osBreakdown = breakdown('os');++ const countryBreakdown = q(`++ SELECT country, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0++ GROUP BY country ORDER BY cnt DESC LIMIT 20++ `).all(...human);++++ const responsePercentiles = safe(() => {++ const vals = q(`++ SELECT response_time_ms FROM analytics_events++ WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0++ ORDER BY response_time_ms ASC++ `).pluck().all(...human);++ if (!vals.length) return {};++ const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];++ return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };++ }, {});++++ const bandwidth = safe(() => {++ const bw = q(`++ SELECT COUNT(*) as total_requests,++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,++ SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ `).get(...human);++ return bw && {++ total_requests: bw.total_requests || 0,++ page_requests: bw.page_requests || 0,++ api_requests: bw.api_requests || 0,++ estimated_bytes: bw.estimated_bytes || 0,++ };++ }, {});++++ const authBreakdown = safe(() => q(`++ SELECT++ COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,++ COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,++ COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ `).get(...human), {});++++ const errorTrend = safe(() => (isSubDay++ ? q(`++ SELECT ${bucketExpr} as bucket, COUNT(*) AS total,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,++ COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY bucket ORDER BY bucket ASC++ `).all(...human)++ : q(`++ SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors++ FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC++ `).all(svc, hourlyCutoff)), []);++++ const peakHours = safe(() => q(`++ SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,++ COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY hour_of_day ORDER BY hour_of_day ASC++ `).all(...human), []);++++ const topApiEndpoints = safe(() => q(`++ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0++ GROUP BY path, method ORDER BY hits DESC LIMIT 20++ `).all(...human), []);++++ const sessionCount = safe(() => q(`++ SELECT COUNT(DISTINCT session_id) FROM analytics_events++ WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0++ `).pluck().get(...human), 0);++++ // New vs returning needs a visitor identity that outlives a day; analytics keeps none.++ const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };++++ const slowestEndpoints = safe(() => q(`++ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms++ FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0++ GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10++ `).all(...human), []);++++ const authTrend = safe(() => q(`++ SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,++ COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,++ COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY bucket ORDER BY bucket ASC++ `).all(...human), []);++++ return {++ service: svc,++ period_days: effectiveDays,++ period_hours: hours || null,++ summary: summary || {},++ realtime: realtime || {},++ daily,++ hourly,++ timeBuckets,++ topPages,++ topReferers,++ botBreakdown,++ statusCodes,++ deviceBreakdown,++ browserBreakdown,++ osBreakdown,++ countryBreakdown,++ responsePercentiles,++ bandwidth,++ authBreakdown,++ errorTrend,++ peakHours,++ topApiEndpoints,++ sessionCount,++ visitorTypes,++ slowestEndpoints,++ authTrend,++ };++ }++++ getOverview(days = 30) {++ const cutoff = dayOf(this._now() - days * 86400000);++ const services = this.db.prepare(`++ SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,++ SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,++ SUM(bot_hits) as bot_hits, SUM(error_count) as errors,++ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms++ FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC++ `).all(cutoff);++ const totals = this.db.prepare(`++ SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,++ SUM(bot_hits) as bot_hits, SUM(error_count) as errors++ FROM analytics_daily WHERE date >= ?++ `).get(cutoff);++ const dailyTrend = this.db.prepare(`++ SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,++ SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits++ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC++ `).all(cutoff);++ const realtime = this.db.prepare(`++ SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots++ FROM analytics_events WHERE created_at >= ? GROUP BY service++ `).all(sqlTime(this._now() - 300000));++ return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };++ }++++ /**++ * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes++ * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is++ * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.++ */++ getBotAnalysis(days = 30) {++ const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);++ const topBotIPs = this.db.prepare(`++ SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,++ COUNT(DISTINCT session_id) AS sessions,++ MIN(created_at) as first_seen, MAX(created_at) as last_seen++ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?++ GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25++ `).all(since);++ const botTrend = this.db.prepare(`++ SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits++ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC++ `).all(since.slice(0, 10));++ const botTypes = this.db.prepare(`++ SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions++ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?++ GROUP BY bot_type ORDER BY hits DESC++ `).all(since);++ const suspiciousIPs = this.db.prepare(`++ SELECT NULL AS ip, session_id, COUNT(*) as total_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,++ COUNT(DISTINCT path) as unique_paths,++ MIN(created_at) as first_seen, MAX(created_at) as last_seen++ FROM analytics_events++ WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL++ GROUP BY session_id HAVING total_hits > 500++ ORDER BY total_hits DESC LIMIT 20++ `).all(sqlTime(this._now() - 7 * 86400000));++ return { topBotIPs, botTrend, botTypes, suspiciousIPs };++ }++++ destroy() {++ this.flush();++ for (const t of this._timers) { clearInterval(t); clearTimeout(t); }++ this._timers = [];++ this._rates.clear();++ this._sessions.clear();++ }++}++++module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };nnew file mode 100644@@@ -0,0 +1,181 @@++'use strict';++// ADR-021 analytics bounds for every satellite (apps/_shared/analytics, scripts/analytics-prune.js):++// a tracked request stores no IP, user id, city, raw user agent, raw referer or query string anywhere++// in analytics.db; paths are route templates; uniques come from day-scoped hashes deleted after the++// day's rollup; prune removes strictly-older raw rows in bounded batches and never touches rollups;++// scrub rewrites legacy rows without changing a rollup counter; the CLI dry run changes nothing and++// --apply needs --backup or --no-backup.++const assert = require('assert');++const crypto = require('crypto');++const fs = require('fs');++const os = require('os');++const path = require('path');++const { dep } = require('./deps');++const { AnalyticsTracker, privacy, retention } = require('../analytics');++const { sqlTime } = require('../analytics/tracker');++const cli = require('../../../scripts/analytics-prune');++++const express = dep('express');++const Database = dep('better-sqlite3');++++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-tools-analytics-'));++const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';++const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';++const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';++const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };++const dumpAll = (db) => db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all()++ .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');++++function seedLegacy(db, nowMs) {++ new AnalyticsTracker(db, 'openvibe-yt', { timers: false }).destroy();++ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)++ VALUES ('openvibe-yt', 'pageview', ?, 'GET', 200, ?, ?, ?, ?, ?, ?, ?)`);++ const cutoffMs = nowMs - 30 * 86400000;++ for (let i = 0; i < 10; i++) ins.run(`/watch/dQw4w9WgXc${i}?v=${i}`, 7, 'tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/x?y=1', sqlTime(nowMs - (40 + i) * 86400000));++ ins.run('/edge-older', null, null, '198.51.100.50', null, CHROME, '', sqlTime(cutoffMs - 1000));++ ins.run('/edge-exact', null, null, '198.51.100.51', null, CHROME, '', sqlTime(cutoffMs));++ ins.run('/api/info/abc123def?url=https://youtube.com/watch?v=x', 9, 'abcdef0123456789', '198.51.100.52', 'Oslo', FIREFOX, 'https://www.reddit.com/r/a', sqlTime(nowMs - 2 * 86400000));++ const tops = JSON.stringify([{ path: '/watch/dQw4w9WgXcQ', cnt: 3 }, { path: '/watch/9bZkp7q19f0', cnt: 2 }]);++ const refs = JSON.stringify([{ referer: 'https://www.google.com/search?q=x', cnt: 4 }]);++ db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')++ .run('openvibe-yt', sqlTime(nowMs - 60 * 86400000).slice(0, 10), 100, 40, 30, 10, 2, tops, refs);++ db.prepare('INSERT INTO analytics_hourly (service, hour, pageviews, api_calls, unique_visitors, unique_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')++ .run('openvibe-yt', sqlTime(nowMs - 3 * 86400000).slice(0, 13) + ':00:00', 10, 4, 3, 1, tops, refs);++ db.prepare("INSERT INTO analytics_rate_tracking (ip, window_start, hit_count) VALUES ('198.51.100.99', 1, 1)").run();++}++++(async () => {++ // ── Templates and reducers ──++ const n = privacy.normalisePath;++ assert.strictEqual(n('/watch/dQw4w9WgXcQ?v=1'), '/watch/:param');++ assert.strictEqual(n('/api/info?url=https://youtube.com/watch?v=abc'), '/api/info');++ assert.strictEqual(n('/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events'), '/api/jobs/:param/events');++ assert.strictEqual(n('/recipe/chicken-tikka'), '/recipe/:param');++ assert.strictEqual(n('/place/@52.37,4.89,14z'), '/place/@:user');++ assert.strictEqual(n('/@alex'), '/@:user');++ assert.strictEqual(n('/api/things/:id'), '/api/things/:id');++ assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=secret'), 'https://www.google.com');++ assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');++++ // ── A tracked request through express ──++ {++ const db = newDb('track.db');++ let clock = Date.parse('2026-09-23T10:15:00Z');++ const tracker = new AnalyticsTracker(db, 'openvibe-yt', { timers: false, now: () => clock });++ const app = express();++ app.set('trust proxy', true);++ app.use(tracker.middleware());++ app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT }; next(); });++ app.get('/api/jobs/:id', (req, res) => res.json({ ok: true }));++ app.get('*', (req, res) => res.send('ok'));++ const server = app.listen(0, '127.0.0.1');++ await new Promise((r) => server.once('listening', r));++ const base = `http://127.0.0.1:${server.address().port}`;++ const hdr = (x) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', ...x });++ try {++ await (await fetch(`${base}/api/jobs/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();++ await (await fetch(`${base}/watch/dQw4w9WgXcQ?v=1`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();++ await new Promise((r) => setTimeout(r, 50));++ } finally { server.close(); }++ tracker.flush();++ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();++ assert.deepStrictEqual(rows.map((r) => r.path), ['/api/jobs/:id', '/watch/:param']);++ assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0]);++ for (const r of rows) {++ assert.ok(r.ip === null && r.user_id === null && r.city === null);++ assert.strictEqual(r.referer, 'https://www.google.com');++ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id));++ }++ const all = dumpAll(db);++ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'supersecret', 'secret-query', '98765', 'dQw4w9WgXcQ', 'Mozilla/5.0']) {++ assert.ok(!all.includes(needle), `found ${needle}`);++ }++ tracker.aggregate();++ const d = db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();++ assert.strictEqual(d.unique_visitors, 2);++ assert.strictEqual(d.unique_users, 1);++ clock = Date.parse('2026-09-24T00:10:00Z');++ tracker.aggregate();++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 0);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 0);++ assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2);++ assert.ok(tracker.getStats({ days: 30 }).summary);++ assert.ok(Array.isArray(tracker.getBotAnalysis(7).topBotIPs));++ tracker.destroy();++ db.close();++ }++++ // ── Prune boundaries, rollups untouched ──++ {++ const db = newDb('prune.db');++ const nowMs = Date.parse('2026-09-23T12:00:00Z');++ seedLegacy(db, nowMs);++ const totals = retention.rollupTotals(db);++ const part = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, maxBatches: 1, now: () => nowMs });++ assert.deepStrictEqual([part.deleted, part.complete], [4, false]);++ const rest = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, now: () => nowMs });++ assert.strictEqual(rest.deleted, 7);++ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all().map((p) => p.split('?')[0]), ['/edge-exact', '/api/info/abc123def']);++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);++++ // Scrub the survivors and the rollup lists.++ await retention.scrubEvents(db, { batchSize: 1 });++ retention.scrubRollups(db);++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY created_at').all();++ assert.deepStrictEqual(rows.map((r) => r.path), ['/edge-exact', '/api/info/:id']);++ assert.deepStrictEqual(rows.map((r) => r.authenticated), [0, 1]);++ assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()), [{ path: '/watch/:param', cnt: 5 }]);++ const all = dumpAll(db);++ for (const needle of ['198.51.100.', 'Lyon', 'Oslo', 'q=x', 'dQw4w9WgXcQ', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);++ db.close();++ }++++ // ── CLI ──++ {++ const out = [];++ const log = (l) => out.push(l);++ const make = (name) => {++ const file = path.join(tmp, name, 'data', 'analytics.db');++ fs.mkdirSync(path.dirname(file), { recursive: true });++ const db = new Database(file);++ db.pragma('journal_mode = WAL');++ seedLegacy(db, Date.now() + 120000); // edge rows clear of the CLI's own cutoff: 10 older rows++ db.pragma('wal_checkpoint(TRUNCATE)');++ db.close();++ return file;++ };++ const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');++ const yt = make('yt');++ const food = make('food');++ const hashes = [sha(yt), sha(food)];++++ assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--scrub'], log), 0);++ assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'dry run changed a file');++ assert.ok(out.some((l) => /prune\s+10 rows/.test(l)), out.join('\n'));++ assert.ok(out.some((l) => /dry run: nothing changed/.test(l)));++++ assert.strictEqual(await cli.main(['--db', yt, '--apply'], log), 2);++ assert.strictEqual(await cli.main(['--db', yt, '--apply', '--no-backup', '--days', '45'], log), 2);++ assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'a refused run changed a file');++++ const bdir = path.join(tmp, 'backups');++ assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--scrub', '--backup', bdir], log), 0, out.join('\n'));++ for (const [name, file] of [['yt', yt], ['food', food]]) {++ const b = new Database(path.join(bdir, `${name}.analytics.db`), { readonly: true });++ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13);++ b.close();++ const db = new Database(file, { readonly: true });++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 3);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events WHERE ip IS NOT NULL OR user_id IS NOT NULL OR city IS NOT NULL').pluck().get(), 0);++ assert.strictEqual(db.prepare('SELECT SUM(pageviews) FROM analytics_daily').pluck().get(), 100);++ db.close();++ }++ // Existing backup targets are refused.++ assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--backup', bdir], log), 2);++ }++++ fs.rmSync(tmp, { recursive: true, force: true });++ console.log('analytics ADR-021: tracked request, templates, rollups, prune, scrub, CLI ok');++})().catch((e) => { console.error(e); process.exit(1); });@@@ -30,13 +30,13 @@ const contracts = require('openvibe-contracts');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 daysconst { internalOk } = require('../../_shared/internal-auth');const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');--const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-audio');++const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-audio', { retention: { days: 30 } });const app = express();// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.@@@ -28,13 +28,13 @@ const contracts = require('openvibe-contracts');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 daysconst { internalOk } = require('../../_shared/internal-auth');const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');--const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-docs');++const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-docs', { retention: { days: 30 } });const app = express();// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.@@@ -14,14 +14,14 @@ const cookieParser = require('cookie-parser');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 daysconst { internalOk } = require('../../_shared/internal-auth');const { hostGuard, stampedPage } = require('../../_shared/host-role');const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');--const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-food');++const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-food', { retention: { days: 30 } });const PORT = parseInt(process.env.PORT) || 4011;const MAPS_API = process.env.MAPS_API || 'http://127.0.0.1:4010';@@@ -28,13 +28,13 @@ const contracts = require('openvibe-contracts');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 daysconst { internalOk } = require('../../_shared/internal-auth');const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');--const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-img');++const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-img', { retention: { days: 30 } });const app = express();// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.@@@ -19,14 +19,14 @@ const config = require('./config');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 daysconst { internalOk } = require('../../_shared/internal-auth');const { hostGuard, stampedPage } = require('../../_shared/host-role');const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');--const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-maps');++const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-maps', { retention: { days: 30 } });const app = express();// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.@@@ -12,14 +12,14 @@ const config = require('./config');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 daysconst { internalOk } = require('../../_shared/internal-auth');const { hostGuard } = require('../../_shared/host-role');const analyticsDbPath = path.resolve(__dirname, '..', process.env.DATA_DIR || 'data', 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');--const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-text');++const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-text', { retention: { days: 30 } });const app = express();// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.@@@ -21,13 +21,13 @@ const { hostGuard } = require('../../_shared/host-role');// ── Analytics ────────────────────────────────────────────────const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('openvibe-shared/analytics');++const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 daysconst { internalOk } = require('../../_shared/internal-auth');const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });const analyticsDb = new Database(analyticsDbPath);analyticsDb.pragma('journal_mode = WAL');--const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-yt');++const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-yt', { retention: { days: 30 } });const app = express();// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.nnew file mode 100644@@@ -0,0 +1,191 @@++#!/usr/bin/env node++/**++ * Raw analytics retention and one-time scrub for the satellites' analytics.db files (ADR-021).++ *++ * node scripts/analytics-prune.js # dry run over every apps/<app>/data/analytics.db++ * node scripts/analytics-prune.js --app yt --scrub # dry run for one app, including the scrub++ * node scripts/analytics-prune.js --apply --scrub --backup <dir> # back up each db into <dir>, then prune + scrub++ * node scripts/analytics-prune.js --apply --no-backup # prune without a backup (explicit)++ *++ * Options:++ * --app <name> only this app (repeatable); default: every app that has data/analytics.db++ * (an app's DATA_DIR other than data/ needs --db)++ * --db <file> an analytics database by path (repeatable; replaces the app discovery)++ * --days <n> keep raw events newer than n days, 1..30 (default 30)++ * --scrub after pruning, rewrite the remaining rows: ip/user_id/city → NULL, path → route++ * template, referer → origin, user_agent → class, legacy session ids → NULL; same++ * path/referer reduction in the rollups' top lists (their counts are untouched)++ * --backup <path> one database: a new file; several: a directory, each backup `<app>.analytics.db`++ * in it (sqlite online backup, verified with quick_check and a row count)++ * --no-backup explicitly skip the backup++ * --no-vacuum skip the VACUUM after an --apply (it needs ~2x the database size free)++ * --batch <n> rows per write batch (default 5000)++ *++ * --apply refuses to run without --backup or --no-backup. Rollup totals are compared before and after++ * every database; any difference exits 1. Safe to run while the satellites are up.++ */++'use strict';++const fs = require('fs');++const path = require('path');++const retention = require('../apps/_shared/analytics/retention');++++const ROOT = path.join(__dirname, '..');++const APPS = path.join(ROOT, 'apps');++++/** better-sqlite3 from the first app that has it installed (the repo root has no node_modules). */++function loadSqlite() {++ for (const a of ['yt', 'maps', 'food', 'img', 'docs', 'audio', 'text', 'gateway']) {++ try { return require(require.resolve('better-sqlite3', { paths: [path.join(APPS, a)] })); } catch { /* next */ }++ }++ throw new Error('better-sqlite3 is not installed in any app (npm run install:all)');++}++++function parseArgs(argv) {++ const a = { apply: false, scrub: false, vacuum: true, backup: null, noBackup: false, days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, apps: [], dbs: [] };++ for (let i = 0; i < argv.length; i++) {++ const k = argv[i];++ const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };++ if (k === '--apply') a.apply = true;++ else if (k === '--scrub') a.scrub = true;++ else if (k === '--no-vacuum') a.vacuum = false;++ else if (k === '--no-backup') a.noBackup = true;++ else if (k === '--backup') a.backup = val();++ else if (k === '--days') a.days = Number(val());++ else if (k === '--batch') a.batch = Number(val());++ else if (k === '--app') a.apps.push(val());++ else if (k === '--db') a.dbs.push(val());++ else if (k === '-h' || k === '--help') a.help = true;++ else throw new Error(`unknown option ${k}`);++ }++ retention.checkDays(a.days);++ if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');++ if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');++ return a;++}++++/** [{ name, file }] to process. */++function targets(args) {++ if (args.dbs.length) {++ const seen = new Map();++ return args.dbs.map((f) => {++ const file = path.resolve(f);++ // apps/<app>/data/analytics.db → "<app>"; anything else → the file name without extension.++ let name = path.basename(path.dirname(file)) === 'data' ? path.basename(path.dirname(path.dirname(file))) : path.basename(file, path.extname(file));++ const n = (seen.get(name) || 0) + 1;++ seen.set(name, n);++ if (n > 1) name = `${name}-${n}`;++ return { name, file };++ });++ }++ const names = args.apps.length ? args.apps : fs.readdirSync(APPS).filter((a) => !a.startsWith('_')).sort();++ return names.map((name) => ({ name, file: path.join(APPS, name, 'data', 'analytics.db') }))++ .filter((t) => args.apps.length || fs.existsSync(t.file));++}++++const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;++function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }++function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }++++async function runOne(Database, t, args, backupTarget, log) {++ if (!fs.existsSync(t.file)) { log(`error: ${t.file} does not exist`); return 2; }++ const db = new Database(t.file, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });++ try {++ db.pragma('busy_timeout = 5000');++ const walBytes = fileBytes(t.file + '-wal');++ const before = retention.inspect(db, { days: args.days });++ log(`\n[${t.name}] ${t.file}`);++ log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);++ log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);++ log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);++ if (before.toScrub) {++ const s = before.toScrub;++ log(`scrub of the rows that stay: ${s.personal} with ip/user_id/city, ${s.paths} paths to template, ${s.referers} referers to origin, ${s.user_agents} user agents to class, ${s.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);++ }++ log(`rate rows ${before.rateRows || 0} (IP counters; the satellites keep them in memory now)`);++ for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);++ if (!args.apply) return 0;++++ if (backupTarget) {++ if (fs.existsSync(backupTarget)) { log(`error: backup target ${backupTarget} already exists; choose a new path`); return 2; }++ const need = before.bytes + walBytes;++ const free = freeBytes(path.dirname(backupTarget));++ if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(backupTarget)}, the backup needs about ${mb(need)}`); return 2; }++ await db.backup(backupTarget);++ const b = new Database(backupTarget, { readonly: true });++ try {++ const ok = b.pragma('quick_check', { simple: true });++ const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()++ ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;++ if (ok !== 'ok' || n < before.events) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }++ } finally { b.close(); }++ log(`backup ${backupTarget} (${mb(fileBytes(backupTarget))}, quick_check ok)`);++ }++++ db.pragma('secure_delete = ON');++ const totalsBefore = retention.rollupTotals(db);++ const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });++ log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);++ if (args.scrub) {++ const s = await retention.scrubEvents(db, { batchSize: args.batch });++ const r = retention.scrubRollups(db);++ log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);++ }++ const totalsAfter = retention.rollupTotals(db);++ if (JSON.stringify(totalsBefore) !== JSON.stringify(totalsAfter)) {++ log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);++ return 1;++ }++ log('rollups totals unchanged');++ if (args.vacuum && (pruned.deleted || args.scrub)) {++ db.pragma('wal_checkpoint(TRUNCATE)');++ db.exec('VACUUM');++ db.pragma('wal_checkpoint(TRUNCATE)');++ log(`vacuumed ${mb(fileBytes(t.file))}`);++ }++ const after = retention.inspect(db, { days: args.days });++ log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);++ return 0;++ } finally {++ db.close();++ }++}++++async function main(argv, log = console.log) {++ let args;++ try { args = parseArgs(argv); } catch (e) { log(`error: ${e.message}`); return 2; }++ if (args.help) { log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, 29).join('\n')); return 0; }++ const list = targets(args);++ if (!list.length) { log('no analytics databases found (use --app or --db)'); return 2; }++ if (args.apply && !args.backup && !args.noBackup) {++ log('refusing --apply without a backup: pass --backup <new file or directory> (sqlite online backup) or --no-backup');++ return 2;++ }++ let backupFor = () => null;++ if (args.apply && args.backup) {++ const target = path.resolve(args.backup);++ if (list.length === 1) {++ backupFor = () => target;++ } else {++ if (fs.existsSync(target) && !fs.statSync(target).isDirectory()) { log(`error: ${target} is a file; several databases need a directory`); return 2; }++ const clash = list.map((t) => path.join(target, `${t.name}.analytics.db`)).filter((f) => fs.existsSync(f));++ if (clash.length) { log(`error: backup targets already exist: ${clash.join(', ')}`); return 2; }++ fs.mkdirSync(target, { recursive: true });++ backupFor = (t) => path.join(target, `${t.name}.analytics.db`);++ }++ }++ const Database = loadSqlite();++ let worst = 0;++ for (const t of list) {++ const code = await runOne(Database, t, args, backupFor(t), log);++ worst = Math.max(worst, code);++ if (code) { log(`[${t.name}] stopped (exit ${code}); later databases not touched`); break; }++ }++ if (!args.apply && worst === 0) log('\ndry run: nothing changed. Re-run with --apply --backup <file|dir> (or --no-backup).');++ return worst;++}++++if (require.main === module) {++ main(process.argv.slice(2)).then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });++}++++module.exports = { main, parseArgs, targets };
@@ -22,7 +22,9 @@ apps/
@@ -22,7 +22,9 @@ apps/
├── audio # Audio.OpenVibe — audio converter & effects
├── audio # Audio.OpenVibe — audio converter & effects
├── text # Text.OpenVibe + Logo.OpenVibe — text generators & logo makers
├── text # Text.OpenVibe + Logo.OpenVibe — text generators & logo makers
└── docs # Docs.OpenVibe — PDF & document tools
└── docs # Docs.OpenVibe — PDF & document tools
-apps/_shared # code the apps require by relative path: host roles, internal auth, the job runtime (jobs/)
+apps/_shared # code the apps require by relative path: host roles, internal auth, the job runtime (jobs/),
+ # visit analytics within ADR-021 (analytics/)
+scripts/analytics-prune.js # operator CLI: raw analytics retention + one-time scrub (dry run by default)
(openvibe-shared is a pinned OpenVibe.Shared release in each app's package.json; no vendor/ copy)
(openvibe-shared is a pinned OpenVibe.Shared release in each app's package.json; no vendor/ copy)
```
```
@@ -154,6 +156,31 @@ Media results with a stand-in Media), img/audio/docs as real processes killed wi
@@ -154,6 +156,31 @@ Media results with a stand-in Media), img/audio/docs as real processes killed wi
hosts on every satellite, and the registry-driven catalog. The audio test needs `ffmpeg` and skips without it.
hosts on every satellite, and the registry-driven catalog. The audio test needs `ffmpeg` and skips without it.
Install first with `npm run install:all`.
Install first with `npm run install:all`.
+## Analytics (ADR-021)
+
+Every satellite except the gateway keeps visit analytics in its own `data/analytics.db` through
+`apps/_shared/analytics` (the same module as OpenVibe.Live's `server/analytics/`). Bound by ADR-021
+(OpenVibe.Contracts `docs/adr/ADR-021-analytics.md`):
+
+- **A raw row carries** event type, service, route template (matched Express route, else a normaliser: no
+ query string, ids/hashes → `:id`, the segment after `watch`, `jobs`, `recipe`, `place`, … → `:param`,
+ `/@x` → `/@:user`), method, status, response time, a rotating session id, country (CDN header),
+ user-agent class (`chrome/windows/desktop`, `bot:googlebot`) + browser/os/device, referer origin, bot
+ flags, a signed-in flag, timestamp. **Never** an IP, a user id, a city, the UA string or a full referer
+ (`ip`/`user_id`/`city` stay as always-NULL columns for compatibility).
+- **Session id:** random, in memory against the visitor hash, new after 30 idle minutes and at UTC midnight.
+- **Bot rate check:** per-IP counters in memory only (current + previous minute); `analytics_rate_tracking`
+ is emptied at boot and no longer written.
+- **Unique visitors:** `HMAC-SHA256(day salt, ip + "\n" + ua)` (16 hex chars), random salt per UTC day. Hashes
+ and the salt are kept only until the first hourly aggregation after their day ends (right after the
+ day's final rollup), never in raw rows. Rollups keep counts; sub-48 h raw summaries count sessions.
+- **Retention:** each satellite prunes raw rows older than 30 days in batches of 5000, 5 minutes after boot
+ and every 24 h after; rollups stay.
+- **CLI:** `node scripts/analytics-prune.js` (dry run over every `apps/*/data/analytics.db`; `--app`, `--db`
+ to narrow). `--apply` needs `--backup <file|dir>` (verified online backup; a directory when several
+ databases are targeted) or `--no-backup`; `--scrub` also rewrites rows written before ADR-021 and the
+ rollups' top lists (counts unchanged). Ends with VACUUM unless `--no-vacuum`.
+
## YouTube downloader: when YouTube refuses the server
## YouTube downloader: when YouTube refuses the server
YouTube blocks some datacentre addresses outright ("Sign in to confirm you're not a bot"). The app
YouTube blocks some datacentre addresses outright ("Sign in to confirm you're not a bot"). The app
new file mode 100644
new file mode 100644
@@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
+'use strict';
+// Request analytics within ADR-021 (see tracker.js, privacy.js, retention.js).
+const { AnalyticsTracker } = require('./tracker');
+const privacy = require('./privacy');
+const retention = require('./retention');
+const { ensureSchema } = require('./schema');
+
+module.exports = { AnalyticsTracker, privacy, retention, ensureSchema };
new file mode 100644
new file mode 100644
@@ -0,0 +1,211 @@
@@ -0,0 +1,211 @@
+'use strict';
+/**
+ * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):
+ * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a
+ * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a
+ * precise location or a user/subject id.
+ *
+ * This file holds the pure reducers that turn a request into those fields. It has no dependencies,
+ * so the same file serves OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);
+ * keep the two copies identical.
+ */
+
+// ── Bots ──────────────────────────────────────────────────────
+
+const BOT_USER_AGENTS = [
+ /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,
+ /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,
+ /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,
+ /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,
+ /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,
+ /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,
+ /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,
+ /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,
+ /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,
+ /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,
+ /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,
+ /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,
+ /statuscake/i, /monitoring/i, /health.?check/i,
+];
+
+const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];
+
+/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */
+const HIGH_REQUEST_RATE = 60;
+
+/** The family name of the first bot pattern the user agent matches, or null. */
+function botName(ua) {
+ if (!ua) return null;
+ for (const re of BOT_USER_AGENTS) {
+ if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';
+ }
+ return null;
+}
+
+/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */
+function classifyRequest(req) {
+ const ua = (req.headers && req.headers['user-agent']) || '';
+ if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };
+ if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };
+ const path = String(req.path || req.url || '').toLowerCase();
+ if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };
+ return { isBot: false, botType: null, confidence: 0.1 };
+}
+
+// ── User agent → class ───────────────────────────────────────
+
+function parseUserAgent(ua) {
+ if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };
+ let device = 'desktop';
+ if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';
+ else if (/ipad|tablet/i.test(ua)) device = 'tablet';
+
+ let browser = 'other';
+ if (/edg\//i.test(ua)) browser = 'edge';
+ else if (/opr\/|opera/i.test(ua)) browser = 'opera';
+ else if (/firefox\//i.test(ua)) browser = 'firefox';
+ else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';
+ else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';
+ else if (/trident|msie/i.test(ua)) browser = 'ie';
+
+ let os = 'other';
+ if (/windows/i.test(ua)) os = 'windows';
+ else if (/macintosh|mac os/i.test(ua)) os = 'macos';
+ else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';
+ else if (/android/i.test(ua)) os = 'android';
+ else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';
+
+ return { device, browser, os };
+}
+
+const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;
+
+/**
+ * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,
+ * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps
+ * to itself), so a scrub can run over rows that are already reduced.
+ */
+function uaClass(ua) {
+ if (ua == null || ua === '') return 'none';
+ const s = String(ua);
+ if (UA_CLASS_RE.test(s)) return s;
+ const bot = botName(s);
+ if (bot) return `bot:${bot}`;
+ const p = parseUserAgent(s);
+ return `${p.browser}/${p.os}/${p.device}`;
+}
+
+// ── Referer → origin ─────────────────────────────────────────
+
+/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */
+function refererOrigin(ref) {
+ if (!ref) return null;
+ try {
+ const u = new URL(String(ref));
+ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
+ return u.origin;
+ } catch {
+ return null;
+ }
+}
+
+// ── Country ──────────────────────────────────────────────────
+
+/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */
+function countryCode(v) {
+ const s = String(v || '').trim().toUpperCase();
+ return /^[A-Z]{2}$/.test(s) ? s : null;
+}
+
+// ── Paths → route templates ──────────────────────────────────
+
+/**
+ * Words after which the next path segment is a parameter even when it looks like a word
+ * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.
+ */
+const DEFAULT_PARAM_PREFIXES = [
+ 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',
+ 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',
+ 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',
+ 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',
+ 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',
+];
+
+const MAX_SEGMENTS = 8;
+const MAX_TEMPLATE_LENGTH = 200;
+const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;
+const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;
+
+/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */
+function looksLikeId(s) {
+ if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…
+ if (/^\d+$/.test(s)) return true; // 123
+ if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid
+ if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes
+ if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…
+ if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid
+ if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers
+ if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names
+ if (s.length > 32) return true;
+ return false;
+}
+
+/**
+ * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by
+ * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated
+ * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.
+ */
+function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES } = {}) {
+ if (raw == null || raw === '') return '/';
+ let p = String(raw);
+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {
+ try { p = new URL(p).pathname; } catch { return '/'; }
+ }
+ p = p.split(/[?#]/)[0];
+ const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);
+ const segs = p.split('/').filter(Boolean);
+ const out = [];
+ let expectParam = false;
+ for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {
+ let s = segs[i];
+ try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }
+ if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }
+ if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }
+ if (expectParam) { out.push(':param'); expectParam = false; continue; }
+ if (looksLikeId(s)) { out.push(':id'); continue; }
+ out.push(s);
+ expectParam = prefixes.has(s.toLowerCase());
+ }
+ if (segs.length > MAX_SEGMENTS) out.push('*');
+ const t = '/' + out.join('/');
+ return t.length > MAX_TEMPLATE_LENGTH ? t.slice(0, MAX_TEMPLATE_LENGTH) : t;
+}
+
+/**
+ * The route template for a finished request: the matched Express route (mount path + route path)
+ * when there is one, else the normalised original path. Both go through normalisePath, because a
+ * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.
+ */
+function routeTemplate(req, fallbackPath, opts) {
+ const r = req && req.route;
+ if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {
+ return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);
+ }
+ return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);
+}
+
+module.exports = {
+ BOT_USER_AGENTS,
+ HONEYPOT_PATHS,
+ HIGH_REQUEST_RATE,
+ DEFAULT_PARAM_PREFIXES,
+ botName,
+ classifyRequest,
+ parseUserAgent,
+ uaClass,
+ refererOrigin,
+ countryCode,
+ looksLikeId,
+ normalisePath,
+ routeTemplate,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,209 @@
@@ -0,0 +1,209 @@
+'use strict';
+/**
+ * Raw analytics retention and the one-time scrub (ADR-021).
+ *
+ * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded
+ * batches, yielding to the event loop between batches. Rollups
+ * (analytics_hourly / analytics_daily) are never touched. Also clears
+ * leftovers: visitor hashes older than yesterday, salts of past days,
+ * and analytics_rate_tracking (IP counters live in memory now).
+ * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →
+ * NULL (authenticated = 1 where a user id was present), path → route
+ * template, referer → origin, user_agent → class, session ids that are
+ * not rotating ids → NULL.
+ * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /
+ * top_referers JSON (counts merged, never changed).
+ * inspect(db, { days }) read-only counts for a dry run.
+ * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.
+ *
+ * Used by the server (scheduled prune) and scripts/analytics-prune.js (CLI).
+ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).
+ */
+
+const { ensureSchema } = require('./schema');
+const privacy = require('./privacy');
+
+const MAX_DAYS = 30;
+const DEFAULT_BATCH = 5000;
+const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;
+
+const pad = (n) => String(n).padStart(2, '0');
+function sqlTime(ms) {
+ const d = new Date(ms);
+ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
+}
+
+function checkDays(days) {
+ const n = Number(days);
+ if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {
+ throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);
+ }
+ return n;
+}
+
+/** The created_at bound: rows strictly older than this are pruned. */
+function cutoffFor(days, nowMs = Date.now()) {
+ return sqlTime(nowMs - checkDays(days) * 86400000);
+}
+
+const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
+const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);
+const tick = () => new Promise((r) => setImmediate(r));
+
+/**
+ * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write
+ * transaction). Returns { cutoff, deleted, batches, complete }.
+ */
+async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {
+ const cutoff = cutoffFor(days, now());
+ const out = { cutoff, deleted: 0, batches: 0, complete: true };
+ if (!tableExists(db, 'analytics_events')) return out;
+ const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');
+ for (;;) {
+ if (out.batches >= maxBatches) { out.complete = false; break; }
+ const n = del.run(cutoff, batchSize).changes;
+ out.batches++;
+ out.deleted += n;
+ if (n < batchSize) break;
+ await tick();
+ }
+ const today = sqlTime(now()).slice(0, 10);
+ const yesterday = sqlTime(now() - 86400000).slice(0, 10);
+ if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);
+ if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);
+ if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();
+ return out;
+}
+
+/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */
+function registerFunctions(db, opts) {
+ db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));
+ db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));
+ db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));
+ db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));
+}
+
+/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */
+async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes } = {}) {
+ ensureSchema(db);
+ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);
+ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();
+ const out = { rows: 0, batches: 0 };
+ if (lo == null) return out;
+ const upd = db.prepare(`
+ UPDATE analytics_events SET
+ authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,
+ ip = NULL, user_id = NULL, city = NULL,
+ session_id = ov_rotating_session(session_id),
+ path = ov_route_template(path),
+ referer = ov_referer_origin(referer),
+ user_agent = ov_ua_class(user_agent)
+ WHERE id >= ? AND id < ?
+ `);
+ for (let from = lo; from <= hi; from += batchSize) {
+ out.rows += upd.run(from, from + batchSize).changes;
+ out.batches++;
+ await tick();
+ }
+ return out;
+}
+
+/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */
+function reduceTopList(json, key, fn) {
+ let list;
+ try { list = JSON.parse(json); } catch { return null; }
+ if (!Array.isArray(list)) return null;
+ const merged = new Map();
+ for (const item of list) {
+ if (!item || typeof item !== 'object') continue;
+ const k = fn(item[key]);
+ if (k == null) continue;
+ merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));
+ }
+ const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);
+ const text = JSON.stringify(next);
+ return text === json ? null : text;
+}
+
+/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */
+function scrubRollups(db, { paramPrefixes } = {}) {
+ const opts = paramPrefixes ? { paramPrefixes } : undefined;
+ const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));
+ const out = { hourly: 0, daily: 0 };
+ for (const table of ['analytics_hourly', 'analytics_daily']) {
+ if (!tableExists(db, table)) continue;
+ const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();
+ const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);
+ db.transaction(() => {
+ for (const r of rows) {
+ const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;
+ const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;
+ if (paths == null && refs == null) continue;
+ upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);
+ out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;
+ }
+ })();
+ }
+ return out;
+}
+
+/** Sums of every rollup counter; equal before and after a prune or scrub. */
+function rollupTotals(db) {
+ const out = {};
+ if (tableExists(db, 'analytics_hourly')) {
+ out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
+ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,
+ TOTAL(error_count) AS error_count FROM analytics_hourly`).get();
+ }
+ if (tableExists(db, 'analytics_daily')) {
+ out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
+ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,
+ TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();
+ }
+ return out;
+}
+
+/** Read-only counts for a dry run. */
+function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes } = {}) {
+ const cutoff = cutoffFor(days, now());
+ const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };
+ const pageSize = db.pragma('page_size', { simple: true });
+ out.bytes = db.pragma('page_count', { simple: true }) * pageSize;
+ out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;
+ out.rollups = rollupTotals(db);
+ if (!tableExists(db, 'analytics_events')) return out;
+ registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);
+ const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();
+ out.events = e.n;
+ out.oldest = e.oldest;
+ out.newest = e.newest;
+ out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);
+ out.remaining = out.events - out.older;
+ const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))
+ .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';
+ const r = db.prepare(`
+ SELECT
+ COUNT(*) FILTER (WHERE ${personal}) AS personal,
+ COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,
+ COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,
+ COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,
+ COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions
+ FROM analytics_events WHERE created_at >= ?
+ `).get(cutoff);
+ out.toScrub = r;
+ out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;
+ return out;
+}
+
+module.exports = {
+ MAX_DAYS,
+ DEFAULT_BATCH,
+ checkDays,
+ cutoffFor,
+ pruneRawEvents,
+ scrubEvents,
+ scrubRollups,
+ rollupTotals,
+ inspect,
+ reduceTopList,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,123 @@
@@ -0,0 +1,123 @@
+'use strict';
+/**
+ * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the
+ * existing analytics.db files and the admin dashboards keep working; the columns that used to hold
+ * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and
+ * user_agent / referer / path hold the reduced forms (class, origin, route template).
+ *
+ * Added:
+ * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)
+ * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;
+ * rows are deleted once the day's rollup is final (≤ ~25 h)
+ * analytics_day_salts the random salt of the current UTC day (so a restart does not
+ * double-count); deleted together with that day's hashes
+ * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)
+ */
+
+const ANALYTICS_SCHEMA = `
+ CREATE TABLE IF NOT EXISTS analytics_events (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ event_type TEXT NOT NULL DEFAULT 'pageview',
+ path TEXT,
+ method TEXT DEFAULT 'GET',
+ status_code INTEGER,
+ response_time_ms INTEGER,
+ user_id INTEGER,
+ session_id TEXT,
+ ip TEXT,
+ country TEXT,
+ city TEXT,
+ user_agent TEXT,
+ referer TEXT,
+ is_bot INTEGER DEFAULT 0,
+ bot_type TEXT,
+ device_type TEXT,
+ browser TEXT,
+ os TEXT,
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_hourly (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ hour TEXT NOT NULL,
+ pageviews INTEGER DEFAULT 0,
+ api_calls INTEGER DEFAULT 0,
+ unique_visitors INTEGER DEFAULT 0,
+ unique_users INTEGER DEFAULT 0,
+ bot_hits INTEGER DEFAULT 0,
+ avg_response_ms INTEGER DEFAULT 0,
+ error_count INTEGER DEFAULT 0,
+ bandwidth_bytes INTEGER DEFAULT 0,
+ top_paths TEXT,
+ top_referers TEXT,
+ UNIQUE(service, hour)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_daily (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ date TEXT NOT NULL,
+ pageviews INTEGER DEFAULT 0,
+ api_calls INTEGER DEFAULT 0,
+ unique_visitors INTEGER DEFAULT 0,
+ unique_users INTEGER DEFAULT 0,
+ new_users INTEGER DEFAULT 0,
+ bot_hits INTEGER DEFAULT 0,
+ avg_response_ms INTEGER DEFAULT 0,
+ error_count INTEGER DEFAULT 0,
+ top_paths TEXT,
+ top_referers TEXT,
+ top_countries TEXT,
+ device_breakdown TEXT,
+ browser_breakdown TEXT,
+ UNIQUE(service, date)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_rate_tracking (
+ ip TEXT NOT NULL,
+ window_start INTEGER NOT NULL,
+ hit_count INTEGER DEFAULT 1,
+ PRIMARY KEY (ip, window_start)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_visitor_days (
+ service TEXT NOT NULL,
+ day TEXT NOT NULL,
+ hour TEXT NOT NULL,
+ vhash TEXT NOT NULL,
+ authenticated INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (service, hour, vhash)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_day_salts (
+ service TEXT NOT NULL,
+ day TEXT NOT NULL,
+ salt TEXT NOT NULL,
+ PRIMARY KEY (service, day)
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);
+ CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);
+ CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);
+
+ -- Indexes over the retired personal columns.
+ DROP INDEX IF EXISTS idx_analytics_events_ip;
+ DROP INDEX IF EXISTS idx_analytics_events_user;
+`;
+
+/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */
+function ensureSchema(db) {
+ db.exec(ANALYTICS_SCHEMA);
+ const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);
+ if (!cols.includes('authenticated')) {
+ db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');
+ }
+}
+
+module.exports = { ANALYTICS_SCHEMA, ensureSchema };
new file mode 100644
new file mode 100644
@@ -0,0 +1,728 @@
@@ -0,0 +1,728 @@
+'use strict';
+/**
+ * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the openvibe-shared
+ * AnalyticsTracker (same tables, same getStats/getOverview/getBotAnalysis shapes), minus the personal
+ * data:
+ *
+ * raw row service, event_type, route template, method, status, response time, rotating session
+ * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,
+ * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.
+ * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after
+ * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.
+ * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped
+ * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).
+ * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.
+ * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as
+ * the day is over); hashes go to analytics_visitor_days only, never to raw events,
+ * and are deleted by the first aggregation after their day ends, right after that day's
+ * final hourly/daily rollup — so no hash or salt outlives its day by more than about
+ * one aggregation interval (1 h). Rollups keep only the counts.
+ *
+ * Retention of raw events (30 days) is ./retention.js; this tracker only writes and rolls up.
+ *
+ * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).
+ */
+
+const crypto = require('crypto');
+const { ensureSchema } = require('./schema');
+const privacy = require('./privacy');
+const retention = require('./retention');
+
+const SESSION_IDLE_MS = 30 * 60 * 1000;
+const MAX_TRACKED_CLIENTS = 200000;
+const MAX_BUFFER = 5000;
+const SESSION_ID_RE = /^[0-9a-f]{16}$/;
+
+const pad = (n) => String(n).padStart(2, '0');
+/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */
+function sqlTime(ms) {
+ const d = new Date(ms);
+ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
+}
+const dayOf = (ms) => sqlTime(ms).slice(0, 10);
+const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';
+const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');
+
+function isStaticOrHealth(path) {
+ return path === '/api/health' || path === '/health' ||
+ /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);
+}
+
+class AnalyticsTracker {
+ /**
+ * @param {object} db better-sqlite3 handle (its own analytics database)
+ * @param {string} service service name ('live', 'openvibe-yt', …)
+ * @param {object} [opts]
+ * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter
+ * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:
+ * run the raw-event prune on a timer (Tools). Live passes false and schedules
+ * retention.pruneRawEvents through server/utils/jobs.js instead.
+ * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)
+ * @param {function} [opts.now] clock (tests)
+ */
+ constructor(db, service, opts = {}) {
+ this.db = db;
+ this.service = service;
+ this._now = opts.now || Date.now;
+ this._pathOpts = opts.paramPrefixes
+ ? { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...opts.paramPrefixes]) }
+ : undefined;
+ this._buffer = [];
+ this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }
+ this._sessions = new Map(); // vhash -> { sid, last }
+ this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)
+ this._saltDay = null;
+ this._salt = null;
+ this._timers = [];
+
+ ensureSchema(db);
+ try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }
+ try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }
+ db.exec('DELETE FROM analytics_rate_tracking');
+
+ this._insertEvent = db.prepare(`
+ INSERT INTO analytics_events
+ (service, event_type, path, method, status_code, response_time_ms,
+ user_id, session_id, ip, country, city, user_agent, referer,
+ is_bot, bot_type, device_type, browser, os, authenticated, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `);
+ this._upsertVisitor = db.prepare(`
+ INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)
+ `);
+ this._write = db.transaction((events, visitors) => {
+ for (const e of events) {
+ this._insertEvent.run(
+ e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,
+ e.session_id, e.country, e.user_agent, e.referer,
+ e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,
+ );
+ }
+ for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);
+ });
+
+ if (opts.timers !== false) {
+ const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
+ const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
+ every(() => { this.flush(); this._sweep(); }, 5000);
+ every(() => this.aggregate(), 60 * 60 * 1000);
+ once(() => this.aggregate(), 10_000);
+ if (opts.retention) {
+ const r = opts.retention;
+ const run = () => retention.pruneRawEvents(db, { days: r.days || retention.MAX_DAYS })
+ .then((out) => { if (out.deleted) console.log(`[Analytics:${service}] pruned ${out.deleted} raw events older than ${out.cutoff}`); })
+ .catch((err) => console.error(`[Analytics:${service}] prune error:`, err.message));
+ once(run, r.initialDelayMs != null ? r.initialDelayMs : 5 * 60 * 1000);
+ every(run, r.intervalMs || 24 * 60 * 60 * 1000);
+ }
+ }
+ }
+
+ // ── Per-request reducers ──────────────────────────────────
+
+ _daySalt(day) {
+ if (this._saltDay === day) return this._salt;
+ let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
+ if (!row) {
+ this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')
+ .run(this.service, day, crypto.randomBytes(32).toString('hex'));
+ row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
+ }
+ // Earlier days' salts are never needed again: their hashes are already written.
+ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);
+ this._saltDay = day;
+ this._salt = row.salt;
+ this._sessions.clear();
+ return this._salt;
+ }
+
+ _visitorHash(ip, ua, day) {
+ return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);
+ }
+
+ _sessionId(vhash, nowMs) {
+ let s = this._sessions.get(vhash);
+ if (!s || nowMs - s.last > SESSION_IDLE_MS) {
+ s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };
+ this._sessions.set(vhash, s);
+ }
+ s.last = nowMs;
+ return s.sid;
+ }
+
+ /** Hits from this client in the current and previous minute (memory only). */
+ _rateHit(ip, nowMs) {
+ const minute = Math.floor(nowMs / 60000);
+ let r = this._rates.get(ip);
+ if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }
+ if (r.minute !== minute) {
+ r.prev = r.minute === minute - 1 ? r.count : 0;
+ r.count = 0;
+ r.minute = minute;
+ }
+ r.count++;
+ return r.count + r.prev;
+ }
+
+ _sweep() {
+ const nowMs = this._now();
+ const minute = Math.floor(nowMs / 60000);
+ for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);
+ for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);
+ if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();
+ if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();
+ }
+
+ _push(event, visitor) {
+ if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();
+ this._buffer.push(event);
+ if (visitor) {
+ const key = `${visitor.hour}|${visitor.vhash}`;
+ const prev = this._visitors.get(key);
+ if (!prev) this._visitors.set(key, visitor);
+ else if (visitor.authenticated) prev.authenticated = true;
+ }
+ if (this._buffer.length >= 100) this.flush();
+ }
+
+ /**
+ * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted
+ * router is relative by the time the response finishes).
+ */
+ record(req, res, entryPath, responseTimeMs) {
+ const nowMs = this._now();
+ const headers = req.headers || {};
+ const ua = String(headers['user-agent'] || '');
+ const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');
+ const bot = privacy.classifyRequest({ headers, path: entryPath });
+ if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {
+ bot.isBot = true;
+ bot.botType = 'rate_limit';
+ bot.confidence = 0.80;
+ }
+ const parsed = privacy.parseUserAgent(ua);
+ const day = dayOf(nowMs);
+ const vhash = this._visitorHash(ip, ua, day);
+ const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));
+ const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');
+ this._push({
+ service: this.service,
+ event_type: isApi ? 'api_call' : 'pageview',
+ path: privacy.routeTemplate(req, entryPath, this._pathOpts),
+ method: req.method,
+ status_code: res.statusCode,
+ response_time_ms: responseTimeMs,
+ session_id: this._sessionId(vhash, nowMs),
+ country: privacy.countryCode(headers['cf-ipcountry']),
+ user_agent: privacy.uaClass(ua),
+ referer: privacy.refererOrigin(headers.referer || headers.referrer),
+ is_bot: bot.isBot,
+ bot_type: bot.botType,
+ device_type: parsed.device,
+ browser: parsed.browser,
+ os: parsed.os,
+ authenticated,
+ created_at: sqlTime(nowMs),
+ }, { day, hour: hourOf(nowMs), vhash, authenticated });
+ }
+
+ /** Express middleware: one row per finished request (health checks and static assets skipped). */
+ middleware() {
+ return (req, res, next) => {
+ const entryPath = req.path || '';
+ if (isStaticOrHealth(entryPath)) return next();
+ const started = Date.now();
+ let done = false;
+ const finish = () => {
+ if (done) return;
+ done = true;
+ try { this.record(req, res, entryPath, Date.now() - started); }
+ catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }
+ };
+ res.once('finish', finish);
+ res.once('close', finish);
+ next();
+ };
+ }
+
+ /**
+ * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,
+ * referer and user_agent are reduced like a request's; session_id is kept only if it is one of
+ * this tracker's rotating ids.
+ */
+ trackEvent(eventType, data = {}) {
+ const nowMs = this._now();
+ const ua = data.user_agent ? String(data.user_agent) : '';
+ const parsed = ua ? privacy.parseUserAgent(ua) : {};
+ this._push({
+ service: this.service,
+ event_type: String(eventType || 'event').slice(0, 64),
+ path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,
+ method: data.method || null,
+ status_code: data.status_code || null,
+ response_time_ms: data.response_time_ms || null,
+ session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,
+ country: privacy.countryCode(data.country),
+ user_agent: ua ? privacy.uaClass(ua) : null,
+ referer: privacy.refererOrigin(data.referer),
+ is_bot: !!data.is_bot,
+ bot_type: data.bot_type || null,
+ device_type: data.device_type || parsed.device || null,
+ browser: data.browser || parsed.browser || null,
+ os: data.os || parsed.os || null,
+ authenticated: !!data.authenticated,
+ created_at: sqlTime(nowMs),
+ }, null);
+ }
+
+ /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */
+ flush() {
+ if (!this._buffer.length && !this._visitors.size) return;
+ const events = this._buffer.splice(0);
+ const visitors = [...this._visitors.values()];
+ this._visitors.clear();
+ try {
+ this._write(events, visitors);
+ } catch (err) {
+ console.error(`[Analytics:${this.service}] Flush error:`, err.message);
+ this._buffer = events.slice(-MAX_BUFFER);
+ for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);
+ }
+ }
+
+ // ── Rollups ───────────────────────────────────────────────
+
+ /**
+ * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has
+ * visitor hashes (finalised here), after which those hashes are deleted.
+ */
+ aggregate() {
+ try {
+ this.flush();
+ const nowMs = this._now();
+ const today = dayOf(nowMs);
+ const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);
+ const days = new Set([today, dayOf(nowMs - 86400000)]);
+ const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);
+ for (const p of pending) { hours.add(p.hour); days.add(p.day); }
+ for (const h of hours) this._aggregateHour(h);
+ for (const d of days) this._aggregateDay(d);
+ this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);
+ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);
+ } catch (err) {
+ console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);
+ }
+ }
+
+ _aggregateHour(hourStart) {
+ const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);
+ const svc = this.service;
+ const stats = this.db.prepare(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ?
+ `).get(svc, hourStart, nextHour);
+ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
+
+ // Uniques only grow within a period, so never lower a stored count (restarts, or rows
+ // counted before this tracker when the old one still stored IPs).
+ const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);
+ const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};
+
+ const topPaths = this.db.prepare(`
+ SELECT path, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0
+ GROUP BY path ORDER BY cnt DESC LIMIT 10
+ `).all(svc, hourStart, nextHour);
+ const topReferers = this.db.prepare(`
+ SELECT referer, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0
+ GROUP BY referer ORDER BY cnt DESC LIMIT 10
+ `).all(svc, hourStart, nextHour);
+
+ this.db.prepare(`
+ INSERT INTO analytics_hourly
+ (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(service, hour) DO UPDATE SET
+ pageviews = excluded.pageviews, api_calls = excluded.api_calls,
+ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
+ bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,
+ error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers
+ `).run(
+ svc, hourStart, stats.pageviews, stats.api_calls,
+ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
+ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
+ JSON.stringify(topPaths), JSON.stringify(topReferers),
+ );
+ }
+
+ _aggregateDay(date) {
+ const dayStart = date + ' 00:00:00';
+ const dayEnd = date + ' 23:59:59';
+ const svc = this.service;
+ const q = (sql) => this.db.prepare(sql);
+ const stats = q(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at <= ?
+ `).get(svc, dayStart, dayEnd);
+ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
+
+ const v = q(`
+ SELECT COUNT(DISTINCT vhash) AS visitors,
+ COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users
+ FROM analytics_visitor_days WHERE service = ? AND day = ?
+ `).get(svc, date);
+ const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};
+
+ const range = [svc, dayStart, dayEnd];
+ const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);
+ const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);
+ const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);
+ const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);
+ const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);
+
+ q(`
+ INSERT INTO analytics_daily
+ (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,
+ avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(service, date) DO UPDATE SET
+ pageviews = excluded.pageviews, api_calls = excluded.api_calls,
+ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
+ new_users = excluded.new_users, bot_hits = excluded.bot_hits,
+ avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,
+ top_paths = excluded.top_paths, top_referers = excluded.top_referers,
+ top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,
+ browser_breakdown = excluded.browser_breakdown
+ `).run(
+ svc, date, stats.pageviews, stats.api_calls,
+ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
+ // New users needs a user id history, which analytics no longer has: keep what was
+ // recorded before, NULL (not measured) from now on.
+ prev.new_users != null ? prev.new_users : null,
+ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
+ JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),
+ JSON.stringify(devices), JSON.stringify(browsers),
+ );
+ }
+
+ // ── Reads (admin dashboards) ─────────────────────────────
+ // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,
+ // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based
+ // numbers (daily/hourly unique_visitors) are true unique visitors per period.
+
+ getStats(options = {}) {
+ const { days = 30, hours, service } = options;
+ const svc = service || this.service;
+ const nowMs = this._now();
+ const rangeMs = hours ? hours * 3600000 : days * 86400000;
+ const rangeCutoff = sqlTime(nowMs - rangeMs);
+ const isSubDay = hours && hours < 24;
+ const effectiveDays = hours ? hours / 24 : days;
+ const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;
+ const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;
+ const q = (sql) => this.db.prepare(sql);
+ const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };
+
+ let daily = [];
+ if (!isSubDay) {
+ daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));
+ }
+
+ const hourlyLimit = hours ? Math.min(hours, 168) : 24;
+ const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);
+ const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);
+
+ let timeBuckets = [];
+ if (isSubDay) {
+ timeBuckets = q(`
+ SELECT ${bucketExpr} as bucket,
+ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ?
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(svc, rangeCutoff);
+ }
+
+ let summary;
+ if (hours && hours < 48) {
+ summary = q(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ?
+ `).get(svc, rangeCutoff);
+ } else {
+ summary = q(`
+ SELECT
+ SUM(pageviews) as total_pageviews,
+ SUM(api_calls) as total_api_calls,
+ SUM(unique_visitors) as total_unique_visitors,
+ SUM(unique_users) as total_unique_users,
+ SUM(bot_hits) as total_bot_hits,
+ SUM(error_count) as total_errors,
+ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
+ FROM analytics_daily
+ WHERE service = ? AND date >= ?
+ `).get(svc, dayOf(nowMs - rangeMs));
+ }
+
+ const realtime = q(`
+ SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
+ FROM analytics_events WHERE service = ? AND created_at >= ?
+ `).get(svc, sqlTime(nowMs - 300000));
+
+ const human = [svc, rangeCutoff];
+ const topPages = q(`
+ SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY path ORDER BY hits DESC LIMIT 20
+ `).all(...human);
+ const topReferers = q(`
+ SELECT referer, COUNT(*) as hits FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0
+ GROUP BY referer ORDER BY hits DESC LIMIT 15
+ `).all(...human);
+ const botBreakdown = q(`
+ SELECT bot_type, COUNT(*) as hits FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND is_bot = 1
+ GROUP BY bot_type ORDER BY hits DESC LIMIT 15
+ `).all(...human);
+ const statusCodes = q(`
+ SELECT CASE
+ WHEN status_code >= 200 AND status_code < 300 THEN '2xx'
+ WHEN status_code >= 300 AND status_code < 400 THEN '3xx'
+ WHEN status_code >= 400 AND status_code < 500 THEN '4xx'
+ WHEN status_code >= 500 THEN '5xx'
+ ELSE 'other' END as group_code,
+ COUNT(*) as cnt
+ FROM analytics_events WHERE service = ? AND created_at >= ?
+ GROUP BY group_code ORDER BY cnt DESC
+ `).all(...human);
+ const breakdown = (col) => q(`
+ SELECT ${col}, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY ${col} ORDER BY cnt DESC
+ `).all(...human);
+ const deviceBreakdown = breakdown('device_type');
+ const browserBreakdown = breakdown('browser');
+ const osBreakdown = breakdown('os');
+ const countryBreakdown = q(`
+ SELECT country, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0
+ GROUP BY country ORDER BY cnt DESC LIMIT 20
+ `).all(...human);
+
+ const responsePercentiles = safe(() => {
+ const vals = q(`
+ SELECT response_time_ms FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
+ ORDER BY response_time_ms ASC
+ `).pluck().all(...human);
+ if (!vals.length) return {};
+ const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];
+ return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };
+ }, {});
+
+ const bandwidth = safe(() => {
+ const bw = q(`
+ SELECT COUNT(*) as total_requests,
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,
+ SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ `).get(...human);
+ return bw && {
+ total_requests: bw.total_requests || 0,
+ page_requests: bw.page_requests || 0,
+ api_requests: bw.api_requests || 0,
+ estimated_bytes: bw.estimated_bytes || 0,
+ };
+ }, {});
+
+ const authBreakdown = safe(() => q(`
+ SELECT
+ COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,
+ COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,
+ COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ `).get(...human), {});
+
+ const errorTrend = safe(() => (isSubDay
+ ? q(`
+ SELECT ${bucketExpr} as bucket, COUNT(*) AS total,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
+ COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(...human)
+ : q(`
+ SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors
+ FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC
+ `).all(svc, hourlyCutoff)), []);
+
+ const peakHours = safe(() => q(`
+ SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,
+ COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY hour_of_day ORDER BY hour_of_day ASC
+ `).all(...human), []);
+
+ const topApiEndpoints = safe(() => q(`
+ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0
+ GROUP BY path, method ORDER BY hits DESC LIMIT 20
+ `).all(...human), []);
+
+ const sessionCount = safe(() => q(`
+ SELECT COUNT(DISTINCT session_id) FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0
+ `).pluck().get(...human), 0);
+
+ // New vs returning needs a visitor identity that outlives a day; analytics keeps none.
+ const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };
+
+ const slowestEndpoints = safe(() => q(`
+ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
+ GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10
+ `).all(...human), []);
+
+ const authTrend = safe(() => q(`
+ SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,
+ COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,
+ COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(...human), []);
+
+ return {
+ service: svc,
+ period_days: effectiveDays,
+ period_hours: hours || null,
+ summary: summary || {},
+ realtime: realtime || {},
+ daily,
+ hourly,
+ timeBuckets,
+ topPages,
+ topReferers,
+ botBreakdown,
+ statusCodes,
+ deviceBreakdown,
+ browserBreakdown,
+ osBreakdown,
+ countryBreakdown,
+ responsePercentiles,
+ bandwidth,
+ authBreakdown,
+ errorTrend,
+ peakHours,
+ topApiEndpoints,
+ sessionCount,
+ visitorTypes,
+ slowestEndpoints,
+ authTrend,
+ };
+ }
+
+ getOverview(days = 30) {
+ const cutoff = dayOf(this._now() - days * 86400000);
+ const services = this.db.prepare(`
+ SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
+ SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,
+ SUM(bot_hits) as bot_hits, SUM(error_count) as errors,
+ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
+ FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC
+ `).all(cutoff);
+ const totals = this.db.prepare(`
+ SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,
+ SUM(bot_hits) as bot_hits, SUM(error_count) as errors
+ FROM analytics_daily WHERE date >= ?
+ `).get(cutoff);
+ const dailyTrend = this.db.prepare(`
+ SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
+ SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits
+ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
+ `).all(cutoff);
+ const realtime = this.db.prepare(`
+ SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
+ FROM analytics_events WHERE created_at >= ? GROUP BY service
+ `).all(sqlTime(this._now() - 300000));
+ return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };
+ }
+
+ /**
+ * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes
+ * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is
+ * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.
+ */
+ getBotAnalysis(days = 30) {
+ const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);
+ const topBotIPs = this.db.prepare(`
+ SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,
+ COUNT(DISTINCT session_id) AS sessions,
+ MIN(created_at) as first_seen, MAX(created_at) as last_seen
+ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
+ GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25
+ `).all(since);
+ const botTrend = this.db.prepare(`
+ SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits
+ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
+ `).all(since.slice(0, 10));
+ const botTypes = this.db.prepare(`
+ SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions
+ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
+ GROUP BY bot_type ORDER BY hits DESC
+ `).all(since);
+ const suspiciousIPs = this.db.prepare(`
+ SELECT NULL AS ip, session_id, COUNT(*) as total_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,
+ COUNT(DISTINCT path) as unique_paths,
+ MIN(created_at) as first_seen, MAX(created_at) as last_seen
+ FROM analytics_events
+ WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL
+ GROUP BY session_id HAVING total_hits > 500
+ ORDER BY total_hits DESC LIMIT 20
+ `).all(sqlTime(this._now() - 7 * 86400000));
+ return { topBotIPs, botTrend, botTypes, suspiciousIPs };
+ }
+
+ destroy() {
+ this.flush();
+ for (const t of this._timers) { clearInterval(t); clearTimeout(t); }
+ this._timers = [];
+ this._rates.clear();
+ this._sessions.clear();
+ }
+}
+
+module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,181 @@
@@ -0,0 +1,181 @@
+'use strict';
+// ADR-021 analytics bounds for every satellite (apps/_shared/analytics, scripts/analytics-prune.js):
+// a tracked request stores no IP, user id, city, raw user agent, raw referer or query string anywhere
+// in analytics.db; paths are route templates; uniques come from day-scoped hashes deleted after the
+// day's rollup; prune removes strictly-older raw rows in bounded batches and never touches rollups;
+// scrub rewrites legacy rows without changing a rollup counter; the CLI dry run changes nothing and
+// --apply needs --backup or --no-backup.
+const assert = require('assert');
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { dep } = require('./deps');
+const { AnalyticsTracker, privacy, retention } = require('../analytics');
+const { sqlTime } = require('../analytics/tracker');
+const cli = require('../../../scripts/analytics-prune');
+
+const express = dep('express');
+const Database = dep('better-sqlite3');
+
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-tools-analytics-'));
+const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
+const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';
+const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };
+const dumpAll = (db) => db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all()
+ .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');
+
+function seedLegacy(db, nowMs) {
+ new AnalyticsTracker(db, 'openvibe-yt', { timers: false }).destroy();
+ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)
+ VALUES ('openvibe-yt', 'pageview', ?, 'GET', 200, ?, ?, ?, ?, ?, ?, ?)`);
+ const cutoffMs = nowMs - 30 * 86400000;
+ for (let i = 0; i < 10; i++) ins.run(`/watch/dQw4w9WgXc${i}?v=${i}`, 7, 'tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/x?y=1', sqlTime(nowMs - (40 + i) * 86400000));
+ ins.run('/edge-older', null, null, '198.51.100.50', null, CHROME, '', sqlTime(cutoffMs - 1000));
+ ins.run('/edge-exact', null, null, '198.51.100.51', null, CHROME, '', sqlTime(cutoffMs));
+ ins.run('/api/info/abc123def?url=https://youtube.com/watch?v=x', 9, 'abcdef0123456789', '198.51.100.52', 'Oslo', FIREFOX, 'https://www.reddit.com/r/a', sqlTime(nowMs - 2 * 86400000));
+ const tops = JSON.stringify([{ path: '/watch/dQw4w9WgXcQ', cnt: 3 }, { path: '/watch/9bZkp7q19f0', cnt: 2 }]);
+ const refs = JSON.stringify([{ referer: 'https://www.google.com/search?q=x', cnt: 4 }]);
+ db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
+ .run('openvibe-yt', sqlTime(nowMs - 60 * 86400000).slice(0, 10), 100, 40, 30, 10, 2, tops, refs);
+ db.prepare('INSERT INTO analytics_hourly (service, hour, pageviews, api_calls, unique_visitors, unique_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
+ .run('openvibe-yt', sqlTime(nowMs - 3 * 86400000).slice(0, 13) + ':00:00', 10, 4, 3, 1, tops, refs);
+ db.prepare("INSERT INTO analytics_rate_tracking (ip, window_start, hit_count) VALUES ('198.51.100.99', 1, 1)").run();
+}
+
+(async () => {
+ // ── Templates and reducers ──
+ const n = privacy.normalisePath;
+ assert.strictEqual(n('/watch/dQw4w9WgXcQ?v=1'), '/watch/:param');
+ assert.strictEqual(n('/api/info?url=https://youtube.com/watch?v=abc'), '/api/info');
+ assert.strictEqual(n('/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events'), '/api/jobs/:param/events');
+ assert.strictEqual(n('/recipe/chicken-tikka'), '/recipe/:param');
+ assert.strictEqual(n('/place/@52.37,4.89,14z'), '/place/@:user');
+ assert.strictEqual(n('/@alex'), '/@:user');
+ assert.strictEqual(n('/api/things/:id'), '/api/things/:id');
+ assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=secret'), 'https://www.google.com');
+ assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');
+
+ // ── A tracked request through express ──
+ {
+ const db = newDb('track.db');
+ let clock = Date.parse('2026-09-23T10:15:00Z');
+ const tracker = new AnalyticsTracker(db, 'openvibe-yt', { timers: false, now: () => clock });
+ const app = express();
+ app.set('trust proxy', true);
+ app.use(tracker.middleware());
+ app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT }; next(); });
+ app.get('/api/jobs/:id', (req, res) => res.json({ ok: true }));
+ app.get('*', (req, res) => res.send('ok'));
+ const server = app.listen(0, '127.0.0.1');
+ await new Promise((r) => server.once('listening', r));
+ const base = `http://127.0.0.1:${server.address().port}`;
+ const hdr = (x) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', ...x });
+ try {
+ await (await fetch(`${base}/api/jobs/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();
+ await (await fetch(`${base}/watch/dQw4w9WgXcQ?v=1`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();
+ await new Promise((r) => setTimeout(r, 50));
+ } finally { server.close(); }
+ tracker.flush();
+ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();
+ assert.deepStrictEqual(rows.map((r) => r.path), ['/api/jobs/:id', '/watch/:param']);
+ assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0]);
+ for (const r of rows) {
+ assert.ok(r.ip === null && r.user_id === null && r.city === null);
+ assert.strictEqual(r.referer, 'https://www.google.com');
+ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id));
+ }
+ const all = dumpAll(db);
+ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'supersecret', 'secret-query', '98765', 'dQw4w9WgXcQ', 'Mozilla/5.0']) {
+ assert.ok(!all.includes(needle), `found ${needle}`);
+ }
+ tracker.aggregate();
+ const d = db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();
+ assert.strictEqual(d.unique_visitors, 2);
+ assert.strictEqual(d.unique_users, 1);
+ clock = Date.parse('2026-09-24T00:10:00Z');
+ tracker.aggregate();
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 0);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 0);
+ assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2);
+ assert.ok(tracker.getStats({ days: 30 }).summary);
+ assert.ok(Array.isArray(tracker.getBotAnalysis(7).topBotIPs));
+ tracker.destroy();
+ db.close();
+ }
+
+ // ── Prune boundaries, rollups untouched ──
+ {
+ const db = newDb('prune.db');
+ const nowMs = Date.parse('2026-09-23T12:00:00Z');
+ seedLegacy(db, nowMs);
+ const totals = retention.rollupTotals(db);
+ const part = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, maxBatches: 1, now: () => nowMs });
+ assert.deepStrictEqual([part.deleted, part.complete], [4, false]);
+ const rest = await retention.pruneRawEvents(db, { days: 30, batchSize: 4, now: () => nowMs });
+ assert.strictEqual(rest.deleted, 7);
+ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all().map((p) => p.split('?')[0]), ['/edge-exact', '/api/info/abc123def']);
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
+
+ // Scrub the survivors and the rollup lists.
+ await retention.scrubEvents(db, { batchSize: 1 });
+ retention.scrubRollups(db);
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY created_at').all();
+ assert.deepStrictEqual(rows.map((r) => r.path), ['/edge-exact', '/api/info/:id']);
+ assert.deepStrictEqual(rows.map((r) => r.authenticated), [0, 1]);
+ assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()), [{ path: '/watch/:param', cnt: 5 }]);
+ const all = dumpAll(db);
+ for (const needle of ['198.51.100.', 'Lyon', 'Oslo', 'q=x', 'dQw4w9WgXcQ', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);
+ db.close();
+ }
+
+ // ── CLI ──
+ {
+ const out = [];
+ const log = (l) => out.push(l);
+ const make = (name) => {
+ const file = path.join(tmp, name, 'data', 'analytics.db');
+ fs.mkdirSync(path.dirname(file), { recursive: true });
+ const db = new Database(file);
+ db.pragma('journal_mode = WAL');
+ seedLegacy(db, Date.now() + 120000); // edge rows clear of the CLI's own cutoff: 10 older rows
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ db.close();
+ return file;
+ };
+ const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
+ const yt = make('yt');
+ const food = make('food');
+ const hashes = [sha(yt), sha(food)];
+
+ assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--scrub'], log), 0);
+ assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'dry run changed a file');
+ assert.ok(out.some((l) => /prune\s+10 rows/.test(l)), out.join('\n'));
+ assert.ok(out.some((l) => /dry run: nothing changed/.test(l)));
+
+ assert.strictEqual(await cli.main(['--db', yt, '--apply'], log), 2);
+ assert.strictEqual(await cli.main(['--db', yt, '--apply', '--no-backup', '--days', '45'], log), 2);
+ assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'a refused run changed a file');
+
+ const bdir = path.join(tmp, 'backups');
+ assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--scrub', '--backup', bdir], log), 0, out.join('\n'));
+ for (const [name, file] of [['yt', yt], ['food', food]]) {
+ const b = new Database(path.join(bdir, `${name}.analytics.db`), { readonly: true });
+ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13);
+ b.close();
+ const db = new Database(file, { readonly: true });
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 3);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events WHERE ip IS NOT NULL OR user_id IS NOT NULL OR city IS NOT NULL').pluck().get(), 0);
+ assert.strictEqual(db.prepare('SELECT SUM(pageviews) FROM analytics_daily').pluck().get(), 100);
+ db.close();
+ }
+ // Existing backup targets are refused.
+ assert.strictEqual(await cli.main(['--db', yt, '--db', food, '--apply', '--backup', bdir], log), 2);
+ }
+
+ fs.rmSync(tmp, { recursive: true, force: true });
+ console.log('analytics ADR-021: tracked request, templates, rollups, prune, scrub, CLI ok');
+})().catch((e) => { console.error(e); process.exit(1); });
@@ -30,13 +30,13 @@ const contracts = require('openvibe-contracts');
@@ -30,13 +30,13 @@ const contracts = require('openvibe-contracts');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
-const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-audio');
+const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-audio', { retention: { days: 30 } });
const app = express();
const app = express();
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
@@ -28,13 +28,13 @@ const contracts = require('openvibe-contracts');
@@ -28,13 +28,13 @@ const contracts = require('openvibe-contracts');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
-const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-docs');
+const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-docs', { retention: { days: 30 } });
const app = express();
const app = express();
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
@@ -14,14 +14,14 @@ const cookieParser = require('cookie-parser');
@@ -14,14 +14,14 @@ const cookieParser = require('cookie-parser');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
const { hostGuard, stampedPage } = require('../../_shared/host-role');
const { hostGuard, stampedPage } = require('../../_shared/host-role');
const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
-const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-food');
+const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-food', { retention: { days: 30 } });
const PORT = parseInt(process.env.PORT) || 4011;
const PORT = parseInt(process.env.PORT) || 4011;
const MAPS_API = process.env.MAPS_API || 'http://127.0.0.1:4010';
const MAPS_API = process.env.MAPS_API || 'http://127.0.0.1:4010';
@@ -28,13 +28,13 @@ const contracts = require('openvibe-contracts');
@@ -28,13 +28,13 @@ const contracts = require('openvibe-contracts');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
const analyticsDbPath = path.resolve(__dirname, '..', config.dataDir, 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
-const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-img');
+const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-img', { retention: { days: 30 } });
const app = express();
const app = express();
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
@@ -19,14 +19,14 @@ const config = require('./config');
@@ -19,14 +19,14 @@ const config = require('./config');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
const { hostGuard, stampedPage } = require('../../_shared/host-role');
const { hostGuard, stampedPage } = require('../../_shared/host-role');
const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
-const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-maps');
+const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-maps', { retention: { days: 30 } });
const app = express();
const app = express();
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
@@ -12,14 +12,14 @@ const config = require('./config');
@@ -12,14 +12,14 @@ const config = require('./config');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
const { hostGuard } = require('../../_shared/host-role');
const { hostGuard } = require('../../_shared/host-role');
const analyticsDbPath = path.resolve(__dirname, '..', process.env.DATA_DIR || 'data', 'analytics.db');
const analyticsDbPath = path.resolve(__dirname, '..', process.env.DATA_DIR || 'data', 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
-const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-text');
+const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-text', { retention: { days: 30 } });
const app = express();
const app = express();
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
@@ -21,13 +21,13 @@ const { hostGuard } = require('../../_shared/host-role');
@@ -21,13 +21,13 @@ const { hostGuard } = require('../../_shared/host-role');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const { AnalyticsTracker } = require('../../_shared/analytics'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days
const { internalOk } = require('../../_shared/internal-auth');
const { internalOk } = require('../../_shared/internal-auth');
const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
const analyticsDbPath = path.join(__dirname, '..', 'data', 'analytics.db');
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
fs.mkdirSync(path.dirname(analyticsDbPath), { recursive: true });
const analyticsDb = new Database(analyticsDbPath);
const analyticsDb = new Database(analyticsDbPath);
analyticsDb.pragma('journal_mode = WAL');
analyticsDb.pragma('journal_mode = WAL');
-const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-yt');
+const analytics = new AnalyticsTracker(analyticsDb, 'openvibe-yt', { retention: { days: 30 } });
const app = express();
const app = express();
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
// What this deploy runs (ADR-016); the shared navbar's release-watch polls it on every tool host.
new file mode 100644
new file mode 100644
@@ -0,0 +1,191 @@
@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+/**
+ * Raw analytics retention and one-time scrub for the satellites' analytics.db files (ADR-021).
+ *
+ * node scripts/analytics-prune.js # dry run over every apps/<app>/data/analytics.db
+ * node scripts/analytics-prune.js --app yt --scrub # dry run for one app, including the scrub
+ * node scripts/analytics-prune.js --apply --scrub --backup <dir> # back up each db into <dir>, then prune + scrub
+ * node scripts/analytics-prune.js --apply --no-backup # prune without a backup (explicit)
+ *
+ * Options:
+ * --app <name> only this app (repeatable); default: every app that has data/analytics.db
+ * (an app's DATA_DIR other than data/ needs --db)
+ * --db <file> an analytics database by path (repeatable; replaces the app discovery)
+ * --days <n> keep raw events newer than n days, 1..30 (default 30)
+ * --scrub after pruning, rewrite the remaining rows: ip/user_id/city → NULL, path → route
+ * template, referer → origin, user_agent → class, legacy session ids → NULL; same
+ * path/referer reduction in the rollups' top lists (their counts are untouched)
+ * --backup <path> one database: a new file; several: a directory, each backup `<app>.analytics.db`
+ * in it (sqlite online backup, verified with quick_check and a row count)
+ * --no-backup explicitly skip the backup
+ * --no-vacuum skip the VACUUM after an --apply (it needs ~2x the database size free)
+ * --batch <n> rows per write batch (default 5000)
+ *
+ * --apply refuses to run without --backup or --no-backup. Rollup totals are compared before and after
+ * every database; any difference exits 1. Safe to run while the satellites are up.
+ */
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const retention = require('../apps/_shared/analytics/retention');
+
+const ROOT = path.join(__dirname, '..');
+const APPS = path.join(ROOT, 'apps');
+
+/** better-sqlite3 from the first app that has it installed (the repo root has no node_modules). */
+function loadSqlite() {
+ for (const a of ['yt', 'maps', 'food', 'img', 'docs', 'audio', 'text', 'gateway']) {
+ try { return require(require.resolve('better-sqlite3', { paths: [path.join(APPS, a)] })); } catch { /* next */ }
+ }
+ throw new Error('better-sqlite3 is not installed in any app (npm run install:all)');
+}
+
+function parseArgs(argv) {
+ const a = { apply: false, scrub: false, vacuum: true, backup: null, noBackup: false, days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, apps: [], dbs: [] };
+ for (let i = 0; i < argv.length; i++) {
+ const k = argv[i];
+ const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };
+ if (k === '--apply') a.apply = true;
+ else if (k === '--scrub') a.scrub = true;
+ else if (k === '--no-vacuum') a.vacuum = false;
+ else if (k === '--no-backup') a.noBackup = true;
+ else if (k === '--backup') a.backup = val();
+ else if (k === '--days') a.days = Number(val());
+ else if (k === '--batch') a.batch = Number(val());
+ else if (k === '--app') a.apps.push(val());
+ else if (k === '--db') a.dbs.push(val());
+ else if (k === '-h' || k === '--help') a.help = true;
+ else throw new Error(`unknown option ${k}`);
+ }
+ retention.checkDays(a.days);
+ if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');
+ if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');
+ return a;
+}
+
+/** [{ name, file }] to process. */
+function targets(args) {
+ if (args.dbs.length) {
+ const seen = new Map();
+ return args.dbs.map((f) => {
+ const file = path.resolve(f);
+ // apps/<app>/data/analytics.db → "<app>"; anything else → the file name without extension.
+ let name = path.basename(path.dirname(file)) === 'data' ? path.basename(path.dirname(path.dirname(file))) : path.basename(file, path.extname(file));
+ const n = (seen.get(name) || 0) + 1;
+ seen.set(name, n);
+ if (n > 1) name = `${name}-${n}`;
+ return { name, file };
+ });
+ }
+ const names = args.apps.length ? args.apps : fs.readdirSync(APPS).filter((a) => !a.startsWith('_')).sort();
+ return names.map((name) => ({ name, file: path.join(APPS, name, 'data', 'analytics.db') }))
+ .filter((t) => args.apps.length || fs.existsSync(t.file));
+}
+
+const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;
+function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }
+function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }
+
+async function runOne(Database, t, args, backupTarget, log) {
+ if (!fs.existsSync(t.file)) { log(`error: ${t.file} does not exist`); return 2; }
+ const db = new Database(t.file, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });
+ try {
+ db.pragma('busy_timeout = 5000');
+ const walBytes = fileBytes(t.file + '-wal');
+ const before = retention.inspect(db, { days: args.days });
+ log(`\n[${t.name}] ${t.file}`);
+ log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);
+ log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);
+ log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);
+ if (before.toScrub) {
+ const s = before.toScrub;
+ log(`scrub of the rows that stay: ${s.personal} with ip/user_id/city, ${s.paths} paths to template, ${s.referers} referers to origin, ${s.user_agents} user agents to class, ${s.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);
+ }
+ log(`rate rows ${before.rateRows || 0} (IP counters; the satellites keep them in memory now)`);
+ for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);
+ if (!args.apply) return 0;
+
+ if (backupTarget) {
+ if (fs.existsSync(backupTarget)) { log(`error: backup target ${backupTarget} already exists; choose a new path`); return 2; }
+ const need = before.bytes + walBytes;
+ const free = freeBytes(path.dirname(backupTarget));
+ if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(backupTarget)}, the backup needs about ${mb(need)}`); return 2; }
+ await db.backup(backupTarget);
+ const b = new Database(backupTarget, { readonly: true });
+ try {
+ const ok = b.pragma('quick_check', { simple: true });
+ const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()
+ ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;
+ if (ok !== 'ok' || n < before.events) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }
+ } finally { b.close(); }
+ log(`backup ${backupTarget} (${mb(fileBytes(backupTarget))}, quick_check ok)`);
+ }
+
+ db.pragma('secure_delete = ON');
+ const totalsBefore = retention.rollupTotals(db);
+ const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });
+ log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);
+ if (args.scrub) {
+ const s = await retention.scrubEvents(db, { batchSize: args.batch });
+ const r = retention.scrubRollups(db);
+ log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);
+ }
+ const totalsAfter = retention.rollupTotals(db);
+ if (JSON.stringify(totalsBefore) !== JSON.stringify(totalsAfter)) {
+ log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);
+ return 1;
+ }
+ log('rollups totals unchanged');
+ if (args.vacuum && (pruned.deleted || args.scrub)) {
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ db.exec('VACUUM');
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ log(`vacuumed ${mb(fileBytes(t.file))}`);
+ }
+ const after = retention.inspect(db, { days: args.days });
+ log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);
+ return 0;
+ } finally {
+ db.close();
+ }
+}
+
+async function main(argv, log = console.log) {
+ let args;
+ try { args = parseArgs(argv); } catch (e) { log(`error: ${e.message}`); return 2; }
+ if (args.help) { log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, 29).join('\n')); return 0; }
+ const list = targets(args);
+ if (!list.length) { log('no analytics databases found (use --app or --db)'); return 2; }
+ if (args.apply && !args.backup && !args.noBackup) {
+ log('refusing --apply without a backup: pass --backup <new file or directory> (sqlite online backup) or --no-backup');
+ return 2;
+ }
+ let backupFor = () => null;
+ if (args.apply && args.backup) {
+ const target = path.resolve(args.backup);
+ if (list.length === 1) {
+ backupFor = () => target;
+ } else {
+ if (fs.existsSync(target) && !fs.statSync(target).isDirectory()) { log(`error: ${target} is a file; several databases need a directory`); return 2; }
+ const clash = list.map((t) => path.join(target, `${t.name}.analytics.db`)).filter((f) => fs.existsSync(f));
+ if (clash.length) { log(`error: backup targets already exist: ${clash.join(', ')}`); return 2; }
+ fs.mkdirSync(target, { recursive: true });
+ backupFor = (t) => path.join(target, `${t.name}.analytics.db`);
+ }
+ }
+ const Database = loadSqlite();
+ let worst = 0;
+ for (const t of list) {
+ const code = await runOne(Database, t, args, backupFor(t), log);
+ worst = Math.max(worst, code);
+ if (code) { log(`[${t.name}] stopped (exit ${code}); later databases not touched`); break; }
+ }
+ if (!args.apply && worst === 0) log('\ndry run: nothing changed. Re-run with --apply --backup <file|dir> (or --no-backup).');
+ return worst;
+}
+
+if (require.main === module) {
+ main(process.argv.slice(2)).then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });
+}
+
+module.exports = { main, parseArgs, targets };