openvibers/openvibe.network Commits
Analytics comes from openvibe-shared/analytics (v1.4.0; retires C-80): server/analytics/ keeps only network.js, the service specifics (SERVICE, PARAM_PREFIXES, PATH_RULES, and PATH_OPTS = { paramPrefixes, pathRules }); openAnalytics keeps its own connection to network.db and returns the package's AnalyticsTracker with retention: false and those path options, so the NetworkAnalyticsTracker subclass, preReducePath and prescrub are gone (the package applies pathRules in the tracker and in the scrub); schedulePrune is retention.schedulePrune. The hand-copied index/privacy/retention/schema/tracker files are deleted. scripts/analytics-prune.js is a wrapper over openvibe-shared/analytics/prune-cli with Network's path options, repo-root-relative paths, the defaultDbPath export and the VACUUM warning in its help. test/analytics-privacy.test.js tests the package with Network's options, checks every written row against analytics/event.v1, and sends a Sec-GPC and a DNT request through Network's routers that leave nothing. New with v1.4.0: a request with Sec-GPC: 1 or DNT: 1 is not recorded at all (no raw row, visitor hash, session id or rate counter); README notes it
Changes
@@@ -366,9 +366,9 @@ Google-style account management supporting up to 5 accounts:Network records one row per finished request in the `analytics_*` tables of `network.db` and rollsthem up hourly and daily for the admin analytics pages (`/api/admin/analytics`, which also gathersLive, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (OpenVibe.Contracts--`docs/adr/ADR-021-analytics.md`). `server/analytics/` holds the same module as OpenVibe.Live--(`server/analytics/`) and OpenVibe.Tools (`apps/_shared/analytics/`); Network's wiring is--`server/analytics/network.js`.++`docs/adr/ADR-021-analytics.md`). The module is `openvibe-shared/analytics` (openvibe-shared v1.4.0;++the same one Live and Tools use). `server/analytics/network.js` holds only Network's wiring: its own++connection, its path options (`paramPrefixes`, `pathRules`) and the prune job.- **Stored:** event type, service, **route template** (the matched Express route, else the pathwithout its query and with ids, usernames and tokens replaced by `:id` / `:param`), method, status,@@@ -378,12 +378,15 @@ Live, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (OpeThe `ip`, `user_id` and `city` columns stay for compatibility and are always NULL. Per-IP countersfor the bot rate check live in memory only. Unique visitors come from a daily-salted hash kept onlyuntil that day's final rollup. "New vs returning visitors" is no longer measured.++- **Opt-out:** a request with `Sec-GPC: 1` or `DNT: 1` is not recorded at all (no raw row, visitor++ hash, session id or rate counter), so it is also missing from the rollups.- **Admin panel:** the bot tables list user-agent classes and high-volume session ids, not IPs.- **Connection:** the tracker opens its own connection to `network.db`, so its settings(`busy_timeout` 250, `secure_delete`) never apply to the identity connection.- **Retention:** raw events older than 30 days are deleted every night in batches of 5000 (the`analytics-prune` job: first run 5 minutes after boot, then every 24 h). Rollups are kept.--- **Operator CLI:** `scripts/analytics-prune.js` runs a dry run by default and changes nothing.++- **Operator CLI:** `scripts/analytics-prune.js` (a wrapper over `openvibe-shared/analytics/prune-cli`++ with Network's database and path rules) runs a dry run by default and changes nothing.`--apply` needs `--backup <new file>` (a verified, owner-only copy of all of `network.db`) or anexplicit `--no-backup`. `--scrub` also rewrites the rows the old tracker wrote and the rollups'top-path and referer lists. Rollup totals are compared before and after the run. VACUUM rewrites@@@ -22,7 +22,7 @@"multer": "^1.4.5-lts.1","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.30.1","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.4.0","uuid": "^11.1.0","web-push": "^3.6.7"}@@@ -1490,8 +1490,9 @@}},"node_modules/openvibe-shared": {-- "version": "1.3.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",++ "version": "1.4.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.4.0",++ "integrity": "sha512-H7jAsUklS1p57d9y6ozME++HmsX18iPYZ/U3P6bLoL1E2oSl6DbSPzLsVUkA7RYoYmAtcI3AwJ4hTNlAr66Vdw==","license": "MIT","dependencies": {"jsonwebtoken": "^9.0.0"@@@ -25,7 +25,7 @@"multer": "^1.4.5-lts.1","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.30.1","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.4.0","uuid": "^11.1.0","web-push": "^3.6.7"}@@@ -1,161 +1,54 @@#!/usr/bin/env node/*** Raw analytics retention and one-time scrub for Network's analytics tables (ADR-021). They live in-- * network.db, Network's identity database, next to users and sessions.++ * network.db, Network's identity database, next to users and sessions. The command itself is++ * openvibe-shared/analytics/prune-cli; this wrapper passes Network's better-sqlite3, database and path++ * rules (/avatar/<name>, /users/by-username/<name>, /api/auth/anon/<token>; server/analytics/network.js).** node scripts/analytics-prune.js # dry run: counts only, changes nothing* node scripts/analytics-prune.js --scrub # dry run including what the scrub would rewrite* node scripts/analytics-prune.js --apply --backup <file> # online backup to <file>, then prune* node scripts/analytics-prune.js --apply --scrub --backup <file>* node scripts/analytics-prune.js --apply --no-backup # prune without a backup (explicit)++ * node scripts/analytics-prune.js --help # every option*-- * Options:-- * --db <file> database (default: $DB_PATH, else data/network.db; relative paths from the repo root)-- * --days <n> keep raw events newer than n days, 1..30 (default 30)-- * --scrub after pruning, rewrite the remaining rows: ip/user_id/city → NULL, path → route-- * template (with Network's path rules: /avatar/<name>, /users/by-username/<name>,-- * /api/auth/anon/<token>), referer → origin, user_agent → class, legacy session-- * ids → NULL; same path/referer reduction in the rollups' top lists (counts untouched)-- * --no-vacuum skip the VACUUM that follows an --apply (VACUUM rewrites the file so pruned and-- * scrubbed values do not survive in free pages; it needs ~2x the database size free).-- * VACUUM rewrites all of network.db and holds its write lock meanwhile: Network's own-- * writes (sign-ins, sessions) wait 5 s and then fail. Vacuum with the service stopped,-- * or pass --no-vacuum while it is up.-- * --batch <n> rows per write batch (default 5000)++ * Default database: $DB_PATH, else data/network.db. Relative --db and --backup paths are from the repo++ * root. The backup is a full copy of network.db (accounts, sessions): mode 0600.*-- * --apply refuses to run without --backup <file> (a new file, mode 0600 — it is a full copy of-- * network.db; sqlite online backup, verified) or an explicit --no-backup. Rollup totals are compared-- * before and after; any difference exits 1. Prune and scrub are safe while the server is up (short-- * batches; the tracker retries a busy flush). Run as the service user so no file changes owner.++ * VACUUM (after an --apply, unless --no-vacuum) rewrites all of network.db and holds its write lock++ * meanwhile: Network's own writes (sign-ins, sessions) wait 5 s and then fail. Vacuum with the service++ * stopped, or pass --no-vacuum while it is up. Run as the service user so no file changes owner.*/'use strict';--const fs = require('fs');const path = require('path');const Database = require('better-sqlite3');--const retention = require('../server/analytics/retention');++const cli = require('openvibe-shared/analytics/prune-cli');const networkAnalytics = require('../server/analytics/network');const ROOT = path.join(__dirname, '..');--const HELP_LINES = 29; // header lines 3..29--const { paramPrefixes } = networkAnalytics.PATH_OPTS;--function parseArgs(argv) {-- const a = { apply: false, scrub: false, vacuum: true, backup: null, noBackup: false, days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, db: null };-- for (let i = 0; i < argv.length; i++) {-- const k = argv[i];-- const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };-- if (k === '--apply') a.apply = true;-- else if (k === '--scrub') a.scrub = true;-- else if (k === '--no-vacuum') a.vacuum = false;-- else if (k === '--no-backup') a.noBackup = true;-- else if (k === '--backup') a.backup = val();-- else if (k === '--days') a.days = Number(val());-- else if (k === '--batch') a.batch = Number(val());-- else if (k === '--db') a.db = val();-- else if (k === '-h' || k === '--help') a.help = true;-- else throw new Error(`unknown option ${k}`);-- }-- retention.checkDays(a.days);-- if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');-- if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');-- return a;--}++const VACUUM_WARNING = `++On Network, VACUUM rewrites all of network.db and holds its write lock meanwhile: Network's own writes++(sign-ins, sessions) wait 5 s and then fail. Vacuum with the service stopped, or pass --no-vacuum while++it is up. Default database: $DB_PATH, else data/network.db; relative paths are from the repo root. The++backup is a full copy of network.db (mode 0600). Run as the service user so no file changes owner.`;/** The database the server uses: --db, else $DB_PATH, else data/network.db (relative to the repo root). */function defaultDbPath(args, env = process.env) {return path.resolve(ROOT, args.db || env.DB_PATH || path.join('data', 'network.db'));}--const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;--function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }--function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }--const sameTotals = (a, b) => JSON.stringify(a) === JSON.stringify(b);----async function main(argv, log = console.log) {-- let args;-- try { args = parseArgs(argv); } catch (e) { log(`error: ${e.message}`); return 2; }-- if (args.help) { log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, HELP_LINES).join('\n')); return 0; }-- const dbPath = defaultDbPath(args);-- if (!fs.existsSync(dbPath)) { log(`error: ${dbPath} does not exist`); return 2; }---- if (args.apply && !args.backup && !args.noBackup) {-- log('refusing --apply without a backup: pass --backup <new file> (sqlite online backup) or --no-backup');-- return 2;-- }---- const db = new Database(dbPath, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });-- try {-- db.pragma('busy_timeout = 5000');-- const walBytes = fileBytes(dbPath + '-wal');-- const before = retention.inspect(db, { days: args.days, paramPrefixes });-- log(`database ${dbPath}`);-- log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);-- log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);-- log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);-- if (before.toScrub) {-- const t = before.toScrub;-- log(`scrub of the rows that stay: ${t.personal} with ip/user_id/city, ${t.paths} paths to template, ${t.referers} referers to origin, ${t.user_agents} user agents to class, ${t.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);-- }-- log(`rate rows ${before.rateRows || 0} (IP counters; the server keeps them in memory now)`);-- for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);---- if (!args.apply) {-- log('\ndry run: nothing changed. Re-run with --apply --backup <file> (or --no-backup).');-- return 0;-- }---- if (args.backup) {-- const target = path.resolve(args.backup);-- if (fs.existsSync(target)) { log(`error: backup target ${target} already exists; choose a new file`); return 2; }-- const need = before.bytes + walBytes;-- const free = freeBytes(path.dirname(target));-- if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(target)}, the backup needs about ${mb(need)}`); return 2; }-- // A full copy of network.db (accounts, sessions): owner-only from the first byte.-- const umask = process.umask(0o077);-- try { await db.backup(target); } finally { process.umask(umask); }-- fs.chmodSync(target, 0o600);-- const b = new Database(target, { readonly: true });-- try {-- const ok = b.pragma('quick_check', { simple: true });-- const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()-- ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;-- if (ok !== 'ok' || n < before.events) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }-- } finally { b.close(); }-- log(`backup ${target} (${mb(fileBytes(target))}, mode 0600, quick_check ok)`);-- }---- db.pragma('secure_delete = ON');-- const totalsBefore = retention.rollupTotals(db);-- const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });-- log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);-- if (args.scrub) {-- await networkAnalytics.prescrub(db, { batchSize: args.batch });-- const s = await retention.scrubEvents(db, { batchSize: args.batch, paramPrefixes });-- const r = retention.scrubRollups(db, { paramPrefixes });-- log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);-- }-- const totalsAfter = retention.rollupTotals(db);-- if (!sameTotals(totalsBefore, totalsAfter)) {-- log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);-- return 1;-- }-- log('rollups totals unchanged');-- if (args.vacuum && (pruned.deleted || args.scrub)) {-- db.pragma('wal_checkpoint(TRUNCATE)');-- db.exec('VACUUM');-- db.pragma('wal_checkpoint(TRUNCATE)');-- log(`vacuumed ${mb(fileBytes(dbPath))}`);-- }-- const after = retention.inspect(db, { days: args.days, paramPrefixes });-- log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);-- return 0;-- } finally {-- db.close();-- }++function main(argv, log) {++ return cli.main(argv, {++ Database,++ log,++ root: ROOT,++ defaultDb: defaultDbPath,++ ...networkAnalytics.PATH_OPTS,++ usage: cli.USAGE + '\n' + VACUUM_WARNING,++ });}--if (require.main === module) {-- main(process.argv.slice(2)).then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });--}++if (require.main === module) cli.run(main);--module.exports = { main, parseArgs, defaultDbPath };++module.exports = { main, defaultDbPath };ddeleted file mode 100644@@@ -1,8 +0,0 @@--'use strict';--// Request analytics within ADR-021 (see tracker.js, privacy.js, retention.js).--const { AnalyticsTracker } = require('./tracker');--const privacy = require('./privacy');--const retention = require('./retention');--const { ensureSchema } = require('./schema');----module.exports = { AnalyticsTracker, privacy, retention, ensureSchema };@@@ -1,114 +1,52 @@'use strict';/**-- * OpenVibe.Network's wiring of the ADR-021 analytics module. The other files in this directory are-- * the module shared with OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);-- * keep them identical to those copies and put anything Network-specific here.++ * OpenVibe.Network's wiring of the ADR-021 analytics module, openvibe-shared/analytics (the one module++ * Live, Tools and Network use). Only what is Network-specific lives here:*-- * openAnalytics(dbPath) the tracker for service 'openvibe-network' (NetworkAnalyticsTracker).-- * Network's analytics tables (analytics_events / _hourly / _daily …) live-- * in network.db, as they always++ * openAnalytics(dbPath) the tracker for service 'openvibe-network'. Network's analytics tables++ * (analytics_events / _hourly / _daily …) live in network.db, as they always* have; the tracker gets a better-sqlite3 connection of its own so its* per-connection settings (busy_timeout 250, secure_delete) never apply to* the identity connection (busy_timeout 5000).-- * schedulePrune(tracker) the nightly `analytics-prune` job: raw events older than 30 days are-- * deleted in bounded batches (retention.pruneRawEvents); rollups are kept.++ * schedulePrune(tracker) the nightly `analytics-prune` job (retention.schedulePrune): raw events++ * older than 30 days are deleted in bounded batches; rollups are kept.* First run 5 minutes after boot, then every 24 hours; timers are unref'd.-- * prescrub(db) Network's part of the one-time scrub (scripts/analytics-prune.js --scrub):-- * the path rules below applied to legacy rows and rollup top lists, before-- * the shared scrub templates them.-- * PATH_OPTS the normaliser options (param prefixes) the CLI passes to the shared scrub.++ * PATH_OPTS { paramPrefixes, pathRules }: the path options the tracker and++ * scripts/analytics-prune.js (prune-cli, for the one-time scrub) both use.++ *++ * A request with `Sec-GPC: 1` or `DNT: 1` is not recorded (openvibe-shared v1.4.0).*/const path = require('path');const Database = require('better-sqlite3');--const { AnalyticsTracker } = require('./tracker');--const privacy = require('./privacy');--const retention = require('./retention');++const { AnalyticsTracker, retention } = require('openvibe-shared/analytics');const SERVICE = 'openvibe-network';/**-- * Words whose next path segment is a parameter on Network (on top of privacy.DEFAULT_PARAM_PREFIXES).++ * Words whose next path segment is a parameter on Network (on top of the package's default prefixes).* They matter when no Express route matched (a rate-limited or rejected request is templated from* its raw path), and to the one-time scrub of rows the old tracker stored raw: /avatar/<username>,* /api/auth/anon/<token>, /api/v1/projects/<slug>.*/const PARAM_PREFIXES = ['avatar', 'anon', 'projects'];--const PATH_OPTS = { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...PARAM_PREFIXES]) };/*** A parameter behind two fixed words, which the shared normaliser cannot see: it turns `users` into a* prefix, so `/users/by-username/alex` would keep `alex`. Applied to the raw path before it.*/const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];--function preReducePath(p) {-- let s = String(p);-- for (const [re, to] of PATH_RULES) s = s.replace(re, to);-- return s;--}--class NetworkAnalyticsTracker extends AnalyticsTracker {-- /** The raw path is only used when no Express route matched (a request refused before routing). */-- record(req, res, entryPath, responseTimeMs) {-- return super.record(req, res, preReducePath(entryPath || ''), responseTimeMs);-- }--}++const PATH_OPTS = { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES };function openAnalytics(dbPath, opts = {}) {const db = new Database(path.resolve(dbPath));db.pragma('journal_mode = WAL');-- return new NetworkAnalyticsTracker(db, SERVICE, { retention: false, paramPrefixes: PARAM_PREFIXES, ...opts });--}----const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);--const tick = () => new Promise((r) => setImmediate(r));----/** PATH_RULES over legacy raw rows (id-range batches) and rollup top lists. Returns { rows, hourly, daily }. */--async function prescrub(db, { batchSize = retention.DEFAULT_BATCH } = {}) {-- const out = { rows: 0, hourly: 0, daily: 0 };-- if (!tableExists(db, 'analytics_events')) return out;-- db.function('ov_network_path', { deterministic: true }, (p) => (p == null ? null : preReducePath(p)));-- const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();-- const upd = db.prepare("UPDATE analytics_events SET path = ov_network_path(path) WHERE id >= ? AND id < ? AND path LIKE '%/by-username/%' AND path IS NOT ov_network_path(path)");-- for (let from = lo; lo != null && from <= hi; from += batchSize) {-- out.rows += upd.run(from, from + batchSize).changes;-- await tick();-- }-- for (const table of ['analytics_hourly', 'analytics_daily']) {-- if (!tableExists(db, table)) continue;-- const rows = db.prepare(`SELECT id, top_paths FROM ${table} WHERE top_paths LIKE '%/by-username/%'`).all();-- const set = db.prepare(`UPDATE ${table} SET top_paths = ? WHERE id = ?`);-- db.transaction(() => {-- for (const r of rows) {-- const next = retention.reduceTopList(r.top_paths, 'path', preReducePath);-- if (next != null) { set.run(next, r.id); out[table === 'analytics_hourly' ? 'hourly' : 'daily']++; }-- }-- })();-- }-- return out;++ return new AnalyticsTracker(db, SERVICE, { retention: false, paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES, ...opts });}--function schedulePrune(tracker, { initialDelayMs = 5 * 60 * 1000, intervalMs = 24 * 60 * 60 * 1000, log = console.log } = {}) {-- let running = false;-- const run = async () => {-- if (running) return null;-- running = true;-- try {-- const out = await retention.pruneRawEvents(tracker.db, { days: retention.MAX_DAYS });-- if (out.deleted) log(`[Analytics] pruned ${out.deleted} raw events older than ${out.cutoff}`);-- return out;-- } catch (err) {-- console.error('[Analytics] analytics-prune error:', err.message);-- return null;-- } finally {-- running = false;-- }-- };-- const first = setTimeout(run, initialDelayMs);-- const every = setInterval(run, intervalMs);-- first.unref();-- every.unref();-- return { run, stop() { clearTimeout(first); clearInterval(every); } };++function schedulePrune(tracker, opts = {}) {++ return retention.schedulePrune(tracker.db, { label: 'Analytics', ...opts });}--module.exports = { SERVICE, PARAM_PREFIXES, PATH_OPTS, NetworkAnalyticsTracker, preReducePath, openAnalytics, schedulePrune, prescrub };++module.exports = { SERVICE, PARAM_PREFIXES, PATH_RULES, PATH_OPTS, openAnalytics, schedulePrune };ddeleted file mode 100644@@@ -1,211 +0,0 @@--'use strict';--/**-- * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):-- * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a-- * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a-- * precise location or a user/subject id.-- *-- * This file holds the pure reducers that turn a request into those fields. It has no dependencies,-- * so the same file serves OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);-- * keep the two copies identical.-- */----// ── Bots ──────────────────────────────────────────────────────----const BOT_USER_AGENTS = [-- /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,-- /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,-- /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,-- /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,-- /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,-- /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,-- /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,-- /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,-- /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,-- /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,-- /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,-- /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,-- /statuscake/i, /monitoring/i, /health.?check/i,--];----const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];----/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */--const HIGH_REQUEST_RATE = 60;----/** The family name of the first bot pattern the user agent matches, or null. */--function botName(ua) {-- if (!ua) return null;-- for (const re of BOT_USER_AGENTS) {-- if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';-- }-- return null;--}----/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */--function classifyRequest(req) {-- const ua = (req.headers && req.headers['user-agent']) || '';-- if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };-- if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };-- const path = String(req.path || req.url || '').toLowerCase();-- if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };-- return { isBot: false, botType: null, confidence: 0.1 };--}----// ── User agent → class ───────────────────────────────────────----function parseUserAgent(ua) {-- if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };-- let device = 'desktop';-- if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';-- else if (/ipad|tablet/i.test(ua)) device = 'tablet';---- let browser = 'other';-- if (/edg\//i.test(ua)) browser = 'edge';-- else if (/opr\/|opera/i.test(ua)) browser = 'opera';-- else if (/firefox\//i.test(ua)) browser = 'firefox';-- else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';-- else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';-- else if (/trident|msie/i.test(ua)) browser = 'ie';---- let os = 'other';-- if (/windows/i.test(ua)) os = 'windows';-- else if (/macintosh|mac os/i.test(ua)) os = 'macos';-- else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';-- else if (/android/i.test(ua)) os = 'android';-- else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';---- return { device, browser, os };--}----const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;----/**-- * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,-- * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps-- * to itself), so a scrub can run over rows that are already reduced.-- */--function uaClass(ua) {-- if (ua == null || ua === '') return 'none';-- const s = String(ua);-- if (UA_CLASS_RE.test(s)) return s;-- const bot = botName(s);-- if (bot) return `bot:${bot}`;-- const p = parseUserAgent(s);-- return `${p.browser}/${p.os}/${p.device}`;--}----// ── Referer → origin ─────────────────────────────────────────----/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */--function refererOrigin(ref) {-- if (!ref) return null;-- try {-- const u = new URL(String(ref));-- if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;-- return u.origin;-- } catch {-- return null;-- }--}----// ── Country ──────────────────────────────────────────────────----/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */--function countryCode(v) {-- const s = String(v || '').trim().toUpperCase();-- return /^[A-Z]{2}$/.test(s) ? s : null;--}----// ── Paths → route templates ──────────────────────────────────----/**-- * Words after which the next path segment is a parameter even when it looks like a word-- * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.-- */--const DEFAULT_PARAM_PREFIXES = [-- 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',-- 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',-- 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',-- 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',-- 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',--];----const MAX_SEGMENTS = 8;--const MAX_TEMPLATE_LENGTH = 200;--const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;--const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;----/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */--function looksLikeId(s) {-- if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…-- if (/^\d+$/.test(s)) return true; // 123-- if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid-- if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes-- if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…-- if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid-- if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers-- if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names-- if (s.length > 32) return true;-- return false;--}----/**-- * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by-- * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated-- * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.-- */--function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES } = {}) {-- if (raw == null || raw === '') return '/';-- let p = String(raw);-- if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {-- try { p = new URL(p).pathname; } catch { return '/'; }-- }-- p = p.split(/[?#]/)[0];-- const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);-- const segs = p.split('/').filter(Boolean);-- const out = [];-- let expectParam = false;-- for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {-- let s = segs[i];-- try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }-- if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }-- if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }-- if (expectParam) { out.push(':param'); expectParam = false; continue; }-- if (looksLikeId(s)) { out.push(':id'); continue; }-- out.push(s);-- expectParam = prefixes.has(s.toLowerCase());-- }-- if (segs.length > MAX_SEGMENTS) out.push('*');-- const t = '/' + out.join('/');-- return t.length > MAX_TEMPLATE_LENGTH ? t.slice(0, MAX_TEMPLATE_LENGTH) : t;--}----/**-- * The route template for a finished request: the matched Express route (mount path + route path)-- * when there is one, else the normalised original path. Both go through normalisePath, because a-- * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.-- */--function routeTemplate(req, fallbackPath, opts) {-- const r = req && req.route;-- if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {-- return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);-- }-- return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);--}----module.exports = {-- BOT_USER_AGENTS,-- HONEYPOT_PATHS,-- HIGH_REQUEST_RATE,-- DEFAULT_PARAM_PREFIXES,-- botName,-- classifyRequest,-- parseUserAgent,-- uaClass,-- refererOrigin,-- countryCode,-- looksLikeId,-- normalisePath,-- routeTemplate,--};ddeleted file mode 100644@@@ -1,209 +0,0 @@--'use strict';--/**-- * Raw analytics retention and the one-time scrub (ADR-021).-- *-- * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded-- * batches, yielding to the event loop between batches. Rollups-- * (analytics_hourly / analytics_daily) are never touched. Also clears-- * leftovers: visitor hashes older than yesterday, salts of past days,-- * and analytics_rate_tracking (IP counters live in memory now).-- * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →-- * NULL (authenticated = 1 where a user id was present), path → route-- * template, referer → origin, user_agent → class, session ids that are-- * not rotating ids → NULL.-- * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /-- * top_referers JSON (counts merged, never changed).-- * inspect(db, { days }) read-only counts for a dry run.-- * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.-- *-- * Used by the server (scheduled prune) and scripts/analytics-prune.js (CLI).-- * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).-- */----const { ensureSchema } = require('./schema');--const privacy = require('./privacy');----const MAX_DAYS = 30;--const DEFAULT_BATCH = 5000;--const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;----const pad = (n) => String(n).padStart(2, '0');--function sqlTime(ms) {-- const d = new Date(ms);-- return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;--}----function checkDays(days) {-- const n = Number(days);-- if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {-- throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);-- }-- return n;--}----/** The created_at bound: rows strictly older than this are pruned. */--function cutoffFor(days, nowMs = Date.now()) {-- return sqlTime(nowMs - checkDays(days) * 86400000);--}----const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);--const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);--const tick = () => new Promise((r) => setImmediate(r));----/**-- * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write-- * transaction). Returns { cutoff, deleted, batches, complete }.-- */--async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {-- const cutoff = cutoffFor(days, now());-- const out = { cutoff, deleted: 0, batches: 0, complete: true };-- if (!tableExists(db, 'analytics_events')) return out;-- const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');-- for (;;) {-- if (out.batches >= maxBatches) { out.complete = false; break; }-- const n = del.run(cutoff, batchSize).changes;-- out.batches++;-- out.deleted += n;-- if (n < batchSize) break;-- await tick();-- }-- const today = sqlTime(now()).slice(0, 10);-- const yesterday = sqlTime(now() - 86400000).slice(0, 10);-- if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);-- if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);-- if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();-- return out;--}----/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */--function registerFunctions(db, opts) {-- db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));-- db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));-- db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));-- db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));--}----/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */--async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes } = {}) {-- ensureSchema(db);-- registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);-- const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();-- const out = { rows: 0, batches: 0 };-- if (lo == null) return out;-- const upd = db.prepare(`-- UPDATE analytics_events SET-- authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,-- ip = NULL, user_id = NULL, city = NULL,-- session_id = ov_rotating_session(session_id),-- path = ov_route_template(path),-- referer = ov_referer_origin(referer),-- user_agent = ov_ua_class(user_agent)-- WHERE id >= ? AND id < ?-- `);-- for (let from = lo; from <= hi; from += batchSize) {-- out.rows += upd.run(from, from + batchSize).changes;-- out.batches++;-- await tick();-- }-- return out;--}----/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */--function reduceTopList(json, key, fn) {-- let list;-- try { list = JSON.parse(json); } catch { return null; }-- if (!Array.isArray(list)) return null;-- const merged = new Map();-- for (const item of list) {-- if (!item || typeof item !== 'object') continue;-- const k = fn(item[key]);-- if (k == null) continue;-- merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));-- }-- const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);-- const text = JSON.stringify(next);-- return text === json ? null : text;--}----/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */--function scrubRollups(db, { paramPrefixes } = {}) {-- const opts = paramPrefixes ? { paramPrefixes } : undefined;-- const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));-- const out = { hourly: 0, daily: 0 };-- for (const table of ['analytics_hourly', 'analytics_daily']) {-- if (!tableExists(db, table)) continue;-- const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();-- const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);-- db.transaction(() => {-- for (const r of rows) {-- const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;-- const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;-- if (paths == null && refs == null) continue;-- upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);-- out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;-- }-- })();-- }-- return out;--}----/** Sums of every rollup counter; equal before and after a prune or scrub. */--function rollupTotals(db) {-- const out = {};-- if (tableExists(db, 'analytics_hourly')) {-- out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,-- TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,-- TOTAL(error_count) AS error_count FROM analytics_hourly`).get();-- }-- if (tableExists(db, 'analytics_daily')) {-- out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,-- TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,-- TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();-- }-- return out;--}----/** Read-only counts for a dry run. */--function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes } = {}) {-- const cutoff = cutoffFor(days, now());-- const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };-- const pageSize = db.pragma('page_size', { simple: true });-- out.bytes = db.pragma('page_count', { simple: true }) * pageSize;-- out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;-- out.rollups = rollupTotals(db);-- if (!tableExists(db, 'analytics_events')) return out;-- registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);-- const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();-- out.events = e.n;-- out.oldest = e.oldest;-- out.newest = e.newest;-- out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);-- out.remaining = out.events - out.older;-- const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))-- .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';-- const r = db.prepare(`-- SELECT-- COUNT(*) FILTER (WHERE ${personal}) AS personal,-- COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,-- COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,-- COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,-- COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions-- FROM analytics_events WHERE created_at >= ?-- `).get(cutoff);-- out.toScrub = r;-- out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;-- return out;--}----module.exports = {-- MAX_DAYS,-- DEFAULT_BATCH,-- checkDays,-- cutoffFor,-- pruneRawEvents,-- scrubEvents,-- scrubRollups,-- rollupTotals,-- inspect,-- reduceTopList,--};ddeleted file mode 100644@@@ -1,123 +0,0 @@--'use strict';--/**-- * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the-- * existing analytics.db files and the admin dashboards keep working; the columns that used to hold-- * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and-- * user_agent / referer / path hold the reduced forms (class, origin, route template).-- *-- * Added:-- * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)-- * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;-- * rows are deleted once the day's rollup is final (≤ ~25 h)-- * analytics_day_salts the random salt of the current UTC day (so a restart does not-- * double-count); deleted together with that day's hashes-- * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)-- */----const ANALYTICS_SCHEMA = `-- CREATE TABLE IF NOT EXISTS analytics_events (-- id INTEGER PRIMARY KEY AUTOINCREMENT,-- service TEXT NOT NULL,-- event_type TEXT NOT NULL DEFAULT 'pageview',-- path TEXT,-- method TEXT DEFAULT 'GET',-- status_code INTEGER,-- response_time_ms INTEGER,-- user_id INTEGER,-- session_id TEXT,-- ip TEXT,-- country TEXT,-- city TEXT,-- user_agent TEXT,-- referer TEXT,-- is_bot INTEGER DEFAULT 0,-- bot_type TEXT,-- device_type TEXT,-- browser TEXT,-- os TEXT,-- created_at DATETIME DEFAULT CURRENT_TIMESTAMP-- );---- CREATE TABLE IF NOT EXISTS analytics_hourly (-- id INTEGER PRIMARY KEY AUTOINCREMENT,-- service TEXT NOT NULL,-- hour TEXT NOT NULL,-- pageviews INTEGER DEFAULT 0,-- api_calls INTEGER DEFAULT 0,-- unique_visitors INTEGER DEFAULT 0,-- unique_users INTEGER DEFAULT 0,-- bot_hits INTEGER DEFAULT 0,-- avg_response_ms INTEGER DEFAULT 0,-- error_count INTEGER DEFAULT 0,-- bandwidth_bytes INTEGER DEFAULT 0,-- top_paths TEXT,-- top_referers TEXT,-- UNIQUE(service, hour)-- );---- CREATE TABLE IF NOT EXISTS analytics_daily (-- id INTEGER PRIMARY KEY AUTOINCREMENT,-- service TEXT NOT NULL,-- date TEXT NOT NULL,-- pageviews INTEGER DEFAULT 0,-- api_calls INTEGER DEFAULT 0,-- unique_visitors INTEGER DEFAULT 0,-- unique_users INTEGER DEFAULT 0,-- new_users INTEGER DEFAULT 0,-- bot_hits INTEGER DEFAULT 0,-- avg_response_ms INTEGER DEFAULT 0,-- error_count INTEGER DEFAULT 0,-- top_paths TEXT,-- top_referers TEXT,-- top_countries TEXT,-- device_breakdown TEXT,-- browser_breakdown TEXT,-- UNIQUE(service, date)-- );---- CREATE TABLE IF NOT EXISTS analytics_rate_tracking (-- ip TEXT NOT NULL,-- window_start INTEGER NOT NULL,-- hit_count INTEGER DEFAULT 1,-- PRIMARY KEY (ip, window_start)-- );---- CREATE TABLE IF NOT EXISTS analytics_visitor_days (-- service TEXT NOT NULL,-- day TEXT NOT NULL,-- hour TEXT NOT NULL,-- vhash TEXT NOT NULL,-- authenticated INTEGER NOT NULL DEFAULT 0,-- PRIMARY KEY (service, hour, vhash)-- );---- CREATE TABLE IF NOT EXISTS analytics_day_salts (-- service TEXT NOT NULL,-- day TEXT NOT NULL,-- salt TEXT NOT NULL,-- PRIMARY KEY (service, day)-- );---- CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);-- CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);-- CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);---- -- Indexes over the retired personal columns.-- DROP INDEX IF EXISTS idx_analytics_events_ip;-- DROP INDEX IF EXISTS idx_analytics_events_user;--`;----/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */--function ensureSchema(db) {-- db.exec(ANALYTICS_SCHEMA);-- const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);-- if (!cols.includes('authenticated')) {-- db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');-- }--}----module.exports = { ANALYTICS_SCHEMA, ensureSchema };ddeleted file mode 100644@@@ -1,728 +0,0 @@--'use strict';--/**-- * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the openvibe-shared-- * AnalyticsTracker (same tables, same getStats/getOverview/getBotAnalysis shapes), minus the personal-- * data:-- *-- * raw row service, event_type, route template, method, status, response time, rotating session-- * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,-- * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.-- * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after-- * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.-- * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped-- * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).-- * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.-- * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as-- * the day is over); hashes go to analytics_visitor_days only, never to raw events,-- * and are deleted by the first aggregation after their day ends, right after that day's-- * final hourly/daily rollup — so no hash or salt outlives its day by more than about-- * one aggregation interval (1 h). Rollups keep only the counts.-- *-- * Retention of raw events (30 days) is ./retention.js; this tracker only writes and rolls up.-- *-- * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).-- */----const crypto = require('crypto');--const { ensureSchema } = require('./schema');--const privacy = require('./privacy');--const retention = require('./retention');----const SESSION_IDLE_MS = 30 * 60 * 1000;--const MAX_TRACKED_CLIENTS = 200000;--const MAX_BUFFER = 5000;--const SESSION_ID_RE = /^[0-9a-f]{16}$/;----const pad = (n) => String(n).padStart(2, '0');--/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */--function sqlTime(ms) {-- const d = new Date(ms);-- return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;--}--const dayOf = (ms) => sqlTime(ms).slice(0, 10);--const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';--const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');----function isStaticOrHealth(path) {-- return path === '/api/health' || path === '/health' ||-- /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);--}----class AnalyticsTracker {-- /**-- * @param {object} db better-sqlite3 handle (its own analytics database)-- * @param {string} service service name ('live', 'openvibe-yt', …)-- * @param {object} [opts]-- * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter-- * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:-- * run the raw-event prune on a timer (Tools). Live passes false and schedules-- * retention.pruneRawEvents through server/utils/jobs.js instead.-- * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)-- * @param {function} [opts.now] clock (tests)-- */-- constructor(db, service, opts = {}) {-- this.db = db;-- this.service = service;-- this._now = opts.now || Date.now;-- this._pathOpts = opts.paramPrefixes-- ? { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...opts.paramPrefixes]) }-- : undefined;-- this._buffer = [];-- this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }-- this._sessions = new Map(); // vhash -> { sid, last }-- this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)-- this._saltDay = null;-- this._salt = null;-- this._timers = [];---- ensureSchema(db);-- try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }-- try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }-- db.exec('DELETE FROM analytics_rate_tracking');---- this._insertEvent = db.prepare(`-- INSERT INTO analytics_events-- (service, event_type, path, method, status_code, response_time_ms,-- user_id, session_id, ip, country, city, user_agent, referer,-- is_bot, bot_type, device_type, browser, os, authenticated, created_at)-- VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)-- `);-- this._upsertVisitor = db.prepare(`-- INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)-- ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)-- `);-- this._write = db.transaction((events, visitors) => {-- for (const e of events) {-- this._insertEvent.run(-- e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,-- e.session_id, e.country, e.user_agent, e.referer,-- e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,-- );-- }-- for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);-- });---- if (opts.timers !== false) {-- const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };-- const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };-- every(() => { this.flush(); this._sweep(); }, 5000);-- every(() => this.aggregate(), 60 * 60 * 1000);-- once(() => this.aggregate(), 10_000);-- if (opts.retention) {-- const r = opts.retention;-- const run = () => retention.pruneRawEvents(db, { days: r.days || retention.MAX_DAYS })-- .then((out) => { if (out.deleted) console.log(`[Analytics:${service}] pruned ${out.deleted} raw events older than ${out.cutoff}`); })-- .catch((err) => console.error(`[Analytics:${service}] prune error:`, err.message));-- once(run, r.initialDelayMs != null ? r.initialDelayMs : 5 * 60 * 1000);-- every(run, r.intervalMs || 24 * 60 * 60 * 1000);-- }-- }-- }---- // ── Per-request reducers ──────────────────────────────────---- _daySalt(day) {-- if (this._saltDay === day) return this._salt;-- let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);-- if (!row) {-- this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')-- .run(this.service, day, crypto.randomBytes(32).toString('hex'));-- row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);-- }-- // Earlier days' salts are never needed again: their hashes are already written.-- this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);-- this._saltDay = day;-- this._salt = row.salt;-- this._sessions.clear();-- return this._salt;-- }---- _visitorHash(ip, ua, day) {-- return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);-- }---- _sessionId(vhash, nowMs) {-- let s = this._sessions.get(vhash);-- if (!s || nowMs - s.last > SESSION_IDLE_MS) {-- s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };-- this._sessions.set(vhash, s);-- }-- s.last = nowMs;-- return s.sid;-- }---- /** Hits from this client in the current and previous minute (memory only). */-- _rateHit(ip, nowMs) {-- const minute = Math.floor(nowMs / 60000);-- let r = this._rates.get(ip);-- if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }-- if (r.minute !== minute) {-- r.prev = r.minute === minute - 1 ? r.count : 0;-- r.count = 0;-- r.minute = minute;-- }-- r.count++;-- return r.count + r.prev;-- }---- _sweep() {-- const nowMs = this._now();-- const minute = Math.floor(nowMs / 60000);-- for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);-- for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);-- if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();-- if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();-- }---- _push(event, visitor) {-- if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();-- this._buffer.push(event);-- if (visitor) {-- const key = `${visitor.hour}|${visitor.vhash}`;-- const prev = this._visitors.get(key);-- if (!prev) this._visitors.set(key, visitor);-- else if (visitor.authenticated) prev.authenticated = true;-- }-- if (this._buffer.length >= 100) this.flush();-- }---- /**-- * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted-- * router is relative by the time the response finishes).-- */-- record(req, res, entryPath, responseTimeMs) {-- const nowMs = this._now();-- const headers = req.headers || {};-- const ua = String(headers['user-agent'] || '');-- const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');-- const bot = privacy.classifyRequest({ headers, path: entryPath });-- if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {-- bot.isBot = true;-- bot.botType = 'rate_limit';-- bot.confidence = 0.80;-- }-- const parsed = privacy.parseUserAgent(ua);-- const day = dayOf(nowMs);-- const vhash = this._visitorHash(ip, ua, day);-- const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));-- const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');-- this._push({-- service: this.service,-- event_type: isApi ? 'api_call' : 'pageview',-- path: privacy.routeTemplate(req, entryPath, this._pathOpts),-- method: req.method,-- status_code: res.statusCode,-- response_time_ms: responseTimeMs,-- session_id: this._sessionId(vhash, nowMs),-- country: privacy.countryCode(headers['cf-ipcountry']),-- user_agent: privacy.uaClass(ua),-- referer: privacy.refererOrigin(headers.referer || headers.referrer),-- is_bot: bot.isBot,-- bot_type: bot.botType,-- device_type: parsed.device,-- browser: parsed.browser,-- os: parsed.os,-- authenticated,-- created_at: sqlTime(nowMs),-- }, { day, hour: hourOf(nowMs), vhash, authenticated });-- }---- /** Express middleware: one row per finished request (health checks and static assets skipped). */-- middleware() {-- return (req, res, next) => {-- const entryPath = req.path || '';-- if (isStaticOrHealth(entryPath)) return next();-- const started = Date.now();-- let done = false;-- const finish = () => {-- if (done) return;-- done = true;-- try { this.record(req, res, entryPath, Date.now() - started); }-- catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }-- };-- res.once('finish', finish);-- res.once('close', finish);-- next();-- };-- }---- /**-- * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,-- * referer and user_agent are reduced like a request's; session_id is kept only if it is one of-- * this tracker's rotating ids.-- */-- trackEvent(eventType, data = {}) {-- const nowMs = this._now();-- const ua = data.user_agent ? String(data.user_agent) : '';-- const parsed = ua ? privacy.parseUserAgent(ua) : {};-- this._push({-- service: this.service,-- event_type: String(eventType || 'event').slice(0, 64),-- path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,-- method: data.method || null,-- status_code: data.status_code || null,-- response_time_ms: data.response_time_ms || null,-- session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,-- country: privacy.countryCode(data.country),-- user_agent: ua ? privacy.uaClass(ua) : null,-- referer: privacy.refererOrigin(data.referer),-- is_bot: !!data.is_bot,-- bot_type: data.bot_type || null,-- device_type: data.device_type || parsed.device || null,-- browser: data.browser || parsed.browser || null,-- os: data.os || parsed.os || null,-- authenticated: !!data.authenticated,-- created_at: sqlTime(nowMs),-- }, null);-- }---- /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */-- flush() {-- if (!this._buffer.length && !this._visitors.size) return;-- const events = this._buffer.splice(0);-- const visitors = [...this._visitors.values()];-- this._visitors.clear();-- try {-- this._write(events, visitors);-- } catch (err) {-- console.error(`[Analytics:${this.service}] Flush error:`, err.message);-- this._buffer = events.slice(-MAX_BUFFER);-- for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);-- }-- }---- // ── Rollups ───────────────────────────────────────────────---- /**-- * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has-- * visitor hashes (finalised here), after which those hashes are deleted.-- */-- aggregate() {-- try {-- this.flush();-- const nowMs = this._now();-- const today = dayOf(nowMs);-- const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);-- const days = new Set([today, dayOf(nowMs - 86400000)]);-- const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);-- for (const p of pending) { hours.add(p.hour); days.add(p.day); }-- for (const h of hours) this._aggregateHour(h);-- for (const d of days) this._aggregateDay(d);-- this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);-- this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);-- } catch (err) {-- console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);-- }-- }---- _aggregateHour(hourStart) {-- const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);-- const svc = this.service;-- const stats = this.db.prepare(`-- SELECT-- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,-- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,-- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at < ?-- `).get(svc, hourStart, nextHour);-- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;---- // Uniques only grow within a period, so never lower a stored count (restarts, or rows-- // counted before this tracker when the old one still stored IPs).-- const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);-- const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};---- const topPaths = this.db.prepare(`-- SELECT path, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0-- GROUP BY path ORDER BY cnt DESC LIMIT 10-- `).all(svc, hourStart, nextHour);-- const topReferers = this.db.prepare(`-- SELECT referer, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0-- GROUP BY referer ORDER BY cnt DESC LIMIT 10-- `).all(svc, hourStart, nextHour);---- this.db.prepare(`-- INSERT INTO analytics_hourly-- (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)-- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)-- ON CONFLICT(service, hour) DO UPDATE SET-- pageviews = excluded.pageviews, api_calls = excluded.api_calls,-- unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,-- bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,-- error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers-- `).run(-- svc, hourStart, stats.pageviews, stats.api_calls,-- Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),-- stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,-- JSON.stringify(topPaths), JSON.stringify(topReferers),-- );-- }---- _aggregateDay(date) {-- const dayStart = date + ' 00:00:00';-- const dayEnd = date + ' 23:59:59';-- const svc = this.service;-- const q = (sql) => this.db.prepare(sql);-- const stats = q(`-- SELECT-- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,-- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,-- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ?-- `).get(svc, dayStart, dayEnd);-- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;---- const v = q(`-- SELECT COUNT(DISTINCT vhash) AS visitors,-- COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users-- FROM analytics_visitor_days WHERE service = ? AND day = ?-- `).get(svc, date);-- const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};---- const range = [svc, dayStart, dayEnd];-- const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);-- const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);-- const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);-- const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);-- const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);---- q(`-- INSERT INTO analytics_daily-- (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,-- avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)-- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)-- ON CONFLICT(service, date) DO UPDATE SET-- pageviews = excluded.pageviews, api_calls = excluded.api_calls,-- unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,-- new_users = excluded.new_users, bot_hits = excluded.bot_hits,-- avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,-- top_paths = excluded.top_paths, top_referers = excluded.top_referers,-- top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,-- browser_breakdown = excluded.browser_breakdown-- `).run(-- svc, date, stats.pageviews, stats.api_calls,-- Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),-- // New users needs a user id history, which analytics no longer has: keep what was-- // recorded before, NULL (not measured) from now on.-- prev.new_users != null ? prev.new_users : null,-- stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,-- JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),-- JSON.stringify(devices), JSON.stringify(browsers),-- );-- }---- // ── Reads (admin dashboards) ─────────────────────────────-- // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,-- // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based-- // numbers (daily/hourly unique_visitors) are true unique visitors per period.---- getStats(options = {}) {-- const { days = 30, hours, service } = options;-- const svc = service || this.service;-- const nowMs = this._now();-- const rangeMs = hours ? hours * 3600000 : days * 86400000;-- const rangeCutoff = sqlTime(nowMs - rangeMs);-- const isSubDay = hours && hours < 24;-- const effectiveDays = hours ? hours / 24 : days;-- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;-- const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;-- const q = (sql) => this.db.prepare(sql);-- const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };---- let daily = [];-- if (!isSubDay) {-- daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));-- }---- const hourlyLimit = hours ? Math.min(hours, 168) : 24;-- const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);-- const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);---- let timeBuckets = [];-- if (isSubDay) {-- timeBuckets = q(`-- SELECT ${bucketExpr} as bucket,-- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,-- COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,-- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,-- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- GROUP BY bucket ORDER BY bucket ASC-- `).all(svc, rangeCutoff);-- }---- let summary;-- if (hours && hours < 48) {-- summary = q(`-- SELECT-- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,-- COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,-- COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,-- COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,-- COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- `).get(svc, rangeCutoff);-- } else {-- summary = q(`-- SELECT-- SUM(pageviews) as total_pageviews,-- SUM(api_calls) as total_api_calls,-- SUM(unique_visitors) as total_unique_visitors,-- SUM(unique_users) as total_unique_users,-- SUM(bot_hits) as total_bot_hits,-- SUM(error_count) as total_errors,-- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms-- FROM analytics_daily-- WHERE service = ? AND date >= ?-- `).get(svc, dayOf(nowMs - rangeMs));-- }---- const realtime = q(`-- SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots-- FROM analytics_events WHERE service = ? AND created_at >= ?-- `).get(svc, sqlTime(nowMs - 300000));---- const human = [svc, rangeCutoff];-- const topPages = q(`-- SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors-- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY path ORDER BY hits DESC LIMIT 20-- `).all(...human);-- const topReferers = q(`-- SELECT referer, COUNT(*) as hits FROM analytics_events-- WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0-- GROUP BY referer ORDER BY hits DESC LIMIT 15-- `).all(...human);-- const botBreakdown = q(`-- SELECT bot_type, COUNT(*) as hits FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 1-- GROUP BY bot_type ORDER BY hits DESC LIMIT 15-- `).all(...human);-- const statusCodes = q(`-- SELECT CASE-- WHEN status_code >= 200 AND status_code < 300 THEN '2xx'-- WHEN status_code >= 300 AND status_code < 400 THEN '3xx'-- WHEN status_code >= 400 AND status_code < 500 THEN '4xx'-- WHEN status_code >= 500 THEN '5xx'-- ELSE 'other' END as group_code,-- COUNT(*) as cnt-- FROM analytics_events WHERE service = ? AND created_at >= ?-- GROUP BY group_code ORDER BY cnt DESC-- `).all(...human);-- const breakdown = (col) => q(`-- SELECT ${col}, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY ${col} ORDER BY cnt DESC-- `).all(...human);-- const deviceBreakdown = breakdown('device_type');-- const browserBreakdown = breakdown('browser');-- const osBreakdown = breakdown('os');-- const countryBreakdown = q(`-- SELECT country, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0-- GROUP BY country ORDER BY cnt DESC LIMIT 20-- `).all(...human);---- const responsePercentiles = safe(() => {-- const vals = q(`-- SELECT response_time_ms FROM analytics_events-- WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0-- ORDER BY response_time_ms ASC-- `).pluck().all(...human);-- if (!vals.length) return {};-- const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];-- return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };-- }, {});---- const bandwidth = safe(() => {-- const bw = q(`-- SELECT COUNT(*) as total_requests,-- COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,-- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,-- SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes-- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0-- `).get(...human);-- return bw && {-- total_requests: bw.total_requests || 0,-- page_requests: bw.page_requests || 0,-- api_requests: bw.api_requests || 0,-- estimated_bytes: bw.estimated_bytes || 0,-- };-- }, {});---- const authBreakdown = safe(() => q(`-- SELECT-- COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,-- COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,-- COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,-- COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions-- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0-- `).get(...human), {});---- const errorTrend = safe(() => (isSubDay-- ? q(`-- SELECT ${bucketExpr} as bucket, COUNT(*) AS total,-- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,-- COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors-- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY bucket ORDER BY bucket ASC-- `).all(...human)-- : q(`-- SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors-- FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC-- `).all(svc, hourlyCutoff)), []);---- const peakHours = safe(() => q(`-- SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,-- COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms-- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY hour_of_day ORDER BY hour_of_day ASC-- `).all(...human), []);---- const topApiEndpoints = safe(() => q(`-- SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,-- COUNT(*) FILTER (WHERE status_code >= 400) AS errors-- FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0-- GROUP BY path, method ORDER BY hits DESC LIMIT 20-- `).all(...human), []);---- const sessionCount = safe(() => q(`-- SELECT COUNT(DISTINCT session_id) FROM analytics_events-- WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0-- `).pluck().get(...human), 0);---- // New vs returning needs a visitor identity that outlives a day; analytics keeps none.-- const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };---- const slowestEndpoints = safe(() => q(`-- SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms-- FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0-- GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10-- `).all(...human), []);---- const authTrend = safe(() => q(`-- SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,-- COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,-- COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,-- COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,-- COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits-- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY bucket ORDER BY bucket ASC-- `).all(...human), []);---- return {-- service: svc,-- period_days: effectiveDays,-- period_hours: hours || null,-- summary: summary || {},-- realtime: realtime || {},-- daily,-- hourly,-- timeBuckets,-- topPages,-- topReferers,-- botBreakdown,-- statusCodes,-- deviceBreakdown,-- browserBreakdown,-- osBreakdown,-- countryBreakdown,-- responsePercentiles,-- bandwidth,-- authBreakdown,-- errorTrend,-- peakHours,-- topApiEndpoints,-- sessionCount,-- visitorTypes,-- slowestEndpoints,-- authTrend,-- };-- }---- getOverview(days = 30) {-- const cutoff = dayOf(this._now() - days * 86400000);-- const services = this.db.prepare(`-- SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,-- SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,-- SUM(bot_hits) as bot_hits, SUM(error_count) as errors,-- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms-- FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC-- `).all(cutoff);-- const totals = this.db.prepare(`-- SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,-- SUM(bot_hits) as bot_hits, SUM(error_count) as errors-- FROM analytics_daily WHERE date >= ?-- `).get(cutoff);-- const dailyTrend = this.db.prepare(`-- SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,-- SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits-- FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC-- `).all(cutoff);-- const realtime = this.db.prepare(`-- SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots-- FROM analytics_events WHERE created_at >= ? GROUP BY service-- `).all(sqlTime(this._now() - 300000));-- return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };-- }---- /**-- * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes-- * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is-- * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.-- */-- getBotAnalysis(days = 30) {-- const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);-- const topBotIPs = this.db.prepare(`-- SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,-- COUNT(DISTINCT session_id) AS sessions,-- MIN(created_at) as first_seen, MAX(created_at) as last_seen-- FROM analytics_events WHERE is_bot = 1 AND created_at >= ?-- GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25-- `).all(since);-- const botTrend = this.db.prepare(`-- SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits-- FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC-- `).all(since.slice(0, 10));-- const botTypes = this.db.prepare(`-- SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions-- FROM analytics_events WHERE is_bot = 1 AND created_at >= ?-- GROUP BY bot_type ORDER BY hits DESC-- `).all(since);-- const suspiciousIPs = this.db.prepare(`-- SELECT NULL AS ip, session_id, COUNT(*) as total_hits,-- COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,-- COUNT(DISTINCT path) as unique_paths,-- MIN(created_at) as first_seen, MAX(created_at) as last_seen-- FROM analytics_events-- WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL-- GROUP BY session_id HAVING total_hits > 500-- ORDER BY total_hits DESC LIMIT 20-- `).all(sqlTime(this._now() - 7 * 86400000));-- return { topBotIPs, botTrend, botTypes, suspiciousIPs };-- }---- destroy() {-- this.flush();-- for (const t of this._timers) { clearInterval(t); clearTimeout(t); }-- this._timers = [];-- this._rates.clear();-- this._sessions.clear();-- }--}----module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };@@@ -316,9 +316,10 @@ app.locals.urlRegistry = resolvedRegistry;config._registry = resolvedRegistry;// ── Analytics Tracking (ADR-021) ──────────────────────────────--// Same tables in network.db, on a connection of the tracker's own (server/analytics/network.js).--// Raw rows: route template, rotating session id, user-agent class, referer origin; never an IP or a--// user id. Pruned after 30 days by the analytics-prune job below; rollups are kept.++// openvibe-shared/analytics; same tables in network.db, on a connection of the tracker's own++// (server/analytics/network.js). Raw rows: route template, rotating session id, user-agent class,++// referer origin; never an IP or a user id. A Sec-GPC: 1 / DNT: 1 request is not recorded. Pruned after++// 30 days by the analytics-prune job below; rollups are kept.const analytics = networkAnalytics.openAnalytics(config.db.path);app.locals.analytics = analytics;app.use(analytics.middleware());@@@ -1,8 +1,11 @@'use strict';--// ADR-021 analytics bounds on OpenVibe.Network (server/analytics/, scripts/analytics-prune.js):++// ADR-021 analytics bounds on OpenVibe.Network (openvibe-shared/analytics wired by++// server/analytics/network.js, scripts/analytics-prune.js):++// - Network's path options (paramPrefixes, pathRules) template its usernames, anon tokens and slugs;// - real requests through Network's own routers (auth, OAuth, avatars, admin analytics) leave no IP,// user id, city, raw user agent, full referer, query value or username in any analytics table;--// the session id is a rotating id, paths are route templates;++// the session id is a rotating id, paths are route templates, every row is a valid analytics/event.v1;++// - a request with Sec-GPC: 1 or DNT: 1 is not recorded at all;// - the tracker runs on its own connection to network.db: the identity connection keeps busy_timeout 5000;// - the admin dashboards keep their shapes (bot rows carry ua_class / session_id, never an IP);// - the analytics-prune job deletes raw rows older than 30 days and leaves rollups alone;@@@ -20,8 +23,8 @@ const cookieParser = require('cookie-parser');const jwt = require('jsonwebtoken');const Database = require('better-sqlite3');const { initDb } = require('../server/db/database');--const { privacy, retention } = require('../server/analytics');--const { sqlTime } = require('../server/analytics/tracker');++const { privacy, retention, event } = require('openvibe-shared/analytics');++const { sqlTime } = require('openvibe-shared/analytics/tracker');const networkAnalytics = require('../server/analytics/network');const cli = require('../scripts/analytics-prune');@@@ -46,7 +49,8 @@ function dumpAnalytics(db) {(async () => {await check('Network paths: usernames, anon tokens and project slugs become parameters', () => {-- const t = (p) => privacy.normalisePath(networkAnalytics.preReducePath(p), networkAnalytics.PATH_OPTS);++ const opts = privacy.pathOptions(networkAnalytics.PATH_OPTS);++ const t = (p) => privacy.normalisePath(p, opts);const cases = {'/avatar/alex?s=96': '/avatar/:param','/api/auth/anon/k3yF00barBaz?x=1': '/api/auth/anon/:param',@@@ -114,6 +118,9 @@ function dumpAnalytics(db) {await get('/no/such/page/424242?email=alex%40example.com', hdr());assert.strictEqual(await get('/internal/users/by-username/carolsecretname', hdr()), 403);await get('/api/auth/me', hdr({ 'user-agent': 'curl/8.5.0' }));++ // Opted out (Sec-GPC / DNT): answered as usual, recorded nowhere.++ assert.strictEqual(await get('/avatar/gpcsecretname', hdr({ 'sec-gpc': '1', 'x-forwarded-for': '198.51.100.61' })), 200);++ assert.strictEqual(await get('/internal/users/by-username/dntsecretname', hdr({ dnt: '1', 'x-forwarded-for': '198.51.100.62' })), 403);await new Promise((r) => setTimeout(r, 50));analytics.flush();@@@ -129,6 +136,7 @@ function dumpAnalytics(db) {assert.ok(/^[0-9a-f]{16}$/.test(r.session_id), r.session_id);assert.strictEqual(r.country, 'NL');assert.ok(/^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/.test(r.user_agent), r.user_agent);++ assert.deepStrictEqual(event.checkRow(r), [], JSON.stringify(r));}assert.deepStrictEqual(rows.map((r) => r.path), ['/api/auth/me', '/avatar/:username', '/avatar/:param', '/oauth/authorize',@@@ -142,7 +150,8 @@ function dumpAnalytics(db) {const everything = dumpAnalytics(db);for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', 'Amsterdam', 'supersecret', 'secret-query','/search', 'statesecret', 'openvibe.live%2Fcb', USERNAME, 'bobsecretname', 'carolsecretname', '0123456789abcdefanon', '424242',-- 'alex%40example.com', '[email protected]', 'Mozilla/5.0', 'curl/8.5.0', 'Bearer', token.slice(0, 20)]) {++ 'alex%40example.com', '[email protected]', 'Mozilla/5.0', 'curl/8.5.0', 'Bearer', token.slice(0, 20),++ 'gpcsecretname', 'dntsecretname', '198.51.100.61', '198.51.100.62']) {assert.ok(!everything.includes(needle), `found ${needle} in the analytics tables`);}assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
@@ -366,9 +366,9 @@ Google-style account management supporting up to 5 accounts:
@@ -366,9 +366,9 @@ Google-style account management supporting up to 5 accounts:
Network records one row per finished request in the `analytics_*` tables of `network.db` and rolls
Network records one row per finished request in the `analytics_*` tables of `network.db` and rolls
them up hourly and daily for the admin analytics pages (`/api/admin/analytics`, which also gathers
them up hourly and daily for the admin analytics pages (`/api/admin/analytics`, which also gathers
Live, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (OpenVibe.Contracts
Live, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (OpenVibe.Contracts
-`docs/adr/ADR-021-analytics.md`). `server/analytics/` holds the same module as OpenVibe.Live
-(`server/analytics/`) and OpenVibe.Tools (`apps/_shared/analytics/`); Network's wiring is
-`server/analytics/network.js`.
+`docs/adr/ADR-021-analytics.md`). The module is `openvibe-shared/analytics` (openvibe-shared v1.4.0;
+the same one Live and Tools use). `server/analytics/network.js` holds only Network's wiring: its own
+connection, its path options (`paramPrefixes`, `pathRules`) and the prune job.
- **Stored:** event type, service, **route template** (the matched Express route, else the path
- **Stored:** event type, service, **route template** (the matched Express route, else the path
without its query and with ids, usernames and tokens replaced by `:id` / `:param`), method, status,
without its query and with ids, usernames and tokens replaced by `:id` / `:param`), method, status,
@@ -378,12 +378,15 @@ Live, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (Ope
@@ -378,12 +378,15 @@ Live, Tools, Games and Media). What a raw row may carry is bound by ADR-021 (Ope
The `ip`, `user_id` and `city` columns stay for compatibility and are always NULL. Per-IP counters
The `ip`, `user_id` and `city` columns stay for compatibility and are always NULL. Per-IP counters
for the bot rate check live in memory only. Unique visitors come from a daily-salted hash kept only
for the bot rate check live in memory only. Unique visitors come from a daily-salted hash kept only
until that day's final rollup. "New vs returning visitors" is no longer measured.
until that day's final rollup. "New vs returning visitors" is no longer measured.
+- **Opt-out:** a request with `Sec-GPC: 1` or `DNT: 1` is not recorded at all (no raw row, visitor
+ hash, session id or rate counter), so it is also missing from the rollups.
- **Admin panel:** the bot tables list user-agent classes and high-volume session ids, not IPs.
- **Admin panel:** the bot tables list user-agent classes and high-volume session ids, not IPs.
- **Connection:** the tracker opens its own connection to `network.db`, so its settings
- **Connection:** the tracker opens its own connection to `network.db`, so its settings
(`busy_timeout` 250, `secure_delete`) never apply to the identity connection.
(`busy_timeout` 250, `secure_delete`) never apply to the identity connection.
- **Retention:** raw events older than 30 days are deleted every night in batches of 5000 (the
- **Retention:** raw events older than 30 days are deleted every night in batches of 5000 (the
`analytics-prune` job: first run 5 minutes after boot, then every 24 h). Rollups are kept.
`analytics-prune` job: first run 5 minutes after boot, then every 24 h). Rollups are kept.
-- **Operator CLI:** `scripts/analytics-prune.js` runs a dry run by default and changes nothing.
+- **Operator CLI:** `scripts/analytics-prune.js` (a wrapper over `openvibe-shared/analytics/prune-cli`
+ with Network's database and path rules) runs a dry run by default and changes nothing.
`--apply` needs `--backup <new file>` (a verified, owner-only copy of all of `network.db`) or an
`--apply` needs `--backup <new file>` (a verified, owner-only copy of all of `network.db`) or an
explicit `--no-backup`. `--scrub` also rewrites the rows the old tracker wrote and the rollups'
explicit `--no-backup`. `--scrub` also rewrites the rows the old tracker wrote and the rollups'
top-path and referer lists. Rollup totals are compared before and after the run. VACUUM rewrites
top-path and referer lists. Rollup totals are compared before and after the run. VACUUM rewrites
@@ -22,7 +22,7 @@
@@ -22,7 +22,7 @@
"multer": "^1.4.5-lts.1",
"multer": "^1.4.5-lts.1",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.30.1",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.30.1",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.4.0",
"uuid": "^11.1.0",
"uuid": "^11.1.0",
"web-push": "^3.6.7"
"web-push": "^3.6.7"
}
}
@@ -1490,8 +1490,9 @@
@@ -1490,8 +1490,9 @@
}
}
},
},
"node_modules/openvibe-shared": {
"node_modules/openvibe-shared": {
- "version": "1.3.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
+ "version": "1.4.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.4.0",
+ "integrity": "sha512-H7jAsUklS1p57d9y6ozME++HmsX18iPYZ/U3P6bLoL1E2oSl6DbSPzLsVUkA7RYoYmAtcI3AwJ4hTNlAr66Vdw==",
"license": "MIT",
"license": "MIT",
"dependencies": {
"dependencies": {
"jsonwebtoken": "^9.0.0"
"jsonwebtoken": "^9.0.0"
@@ -25,7 +25,7 @@
@@ -25,7 +25,7 @@
"multer": "^1.4.5-lts.1",
"multer": "^1.4.5-lts.1",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.30.1",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.30.1",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.4.0",
"uuid": "^11.1.0",
"uuid": "^11.1.0",
"web-push": "^3.6.7"
"web-push": "^3.6.7"
}
}
@@ -1,161 +1,54 @@
@@ -1,161 +1,54 @@
#!/usr/bin/env node
#!/usr/bin/env node
/**
/**
* Raw analytics retention and one-time scrub for Network's analytics tables (ADR-021). They live in
* Raw analytics retention and one-time scrub for Network's analytics tables (ADR-021). They live in
- * network.db, Network's identity database, next to users and sessions.
+ * network.db, Network's identity database, next to users and sessions. The command itself is
+ * openvibe-shared/analytics/prune-cli; this wrapper passes Network's better-sqlite3, database and path
+ * rules (/avatar/<name>, /users/by-username/<name>, /api/auth/anon/<token>; server/analytics/network.js).
*
*
* node scripts/analytics-prune.js # dry run: counts only, changes nothing
* node scripts/analytics-prune.js # dry run: counts only, changes nothing
* node scripts/analytics-prune.js --scrub # dry run including what the scrub would rewrite
* node scripts/analytics-prune.js --scrub # dry run including what the scrub would rewrite
* node scripts/analytics-prune.js --apply --backup <file> # online backup to <file>, then prune
* node scripts/analytics-prune.js --apply --backup <file> # online backup to <file>, then prune
* node scripts/analytics-prune.js --apply --scrub --backup <file>
* node scripts/analytics-prune.js --apply --scrub --backup <file>
* node scripts/analytics-prune.js --apply --no-backup # prune without a backup (explicit)
* node scripts/analytics-prune.js --apply --no-backup # prune without a backup (explicit)
+ * node scripts/analytics-prune.js --help # every option
*
*
- * Options:
- * --db <file> database (default: $DB_PATH, else data/network.db; relative paths from the repo root)
- * --days <n> keep raw events newer than n days, 1..30 (default 30)
- * --scrub after pruning, rewrite the remaining rows: ip/user_id/city → NULL, path → route
- * template (with Network's path rules: /avatar/<name>, /users/by-username/<name>,
- * /api/auth/anon/<token>), referer → origin, user_agent → class, legacy session
- * ids → NULL; same path/referer reduction in the rollups' top lists (counts untouched)
- * --no-vacuum skip the VACUUM that follows an --apply (VACUUM rewrites the file so pruned and
- * scrubbed values do not survive in free pages; it needs ~2x the database size free).
- * VACUUM rewrites all of network.db and holds its write lock meanwhile: Network's own
- * writes (sign-ins, sessions) wait 5 s and then fail. Vacuum with the service stopped,
- * or pass --no-vacuum while it is up.
- * --batch <n> rows per write batch (default 5000)
+ * Default database: $DB_PATH, else data/network.db. Relative --db and --backup paths are from the repo
+ * root. The backup is a full copy of network.db (accounts, sessions): mode 0600.
*
*
- * --apply refuses to run without --backup <file> (a new file, mode 0600 — it is a full copy of
- * network.db; sqlite online backup, verified) or an explicit --no-backup. Rollup totals are compared
- * before and after; any difference exits 1. Prune and scrub are safe while the server is up (short
- * batches; the tracker retries a busy flush). Run as the service user so no file changes owner.
+ * VACUUM (after an --apply, unless --no-vacuum) rewrites all of network.db and holds its write lock
+ * meanwhile: Network's own writes (sign-ins, sessions) wait 5 s and then fail. Vacuum with the service
+ * stopped, or pass --no-vacuum while it is up. Run as the service user so no file changes owner.
*/
*/
'use strict';
'use strict';
-const fs = require('fs');
const path = require('path');
const path = require('path');
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const retention = require('../server/analytics/retention');
+const cli = require('openvibe-shared/analytics/prune-cli');
const networkAnalytics = require('../server/analytics/network');
const networkAnalytics = require('../server/analytics/network');
const ROOT = path.join(__dirname, '..');
const ROOT = path.join(__dirname, '..');
-const HELP_LINES = 29; // header lines 3..29
-const { paramPrefixes } = networkAnalytics.PATH_OPTS;
-function parseArgs(argv) {
- const a = { apply: false, scrub: false, vacuum: true, backup: null, noBackup: false, days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, db: null };
- for (let i = 0; i < argv.length; i++) {
- const k = argv[i];
- const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };
- if (k === '--apply') a.apply = true;
- else if (k === '--scrub') a.scrub = true;
- else if (k === '--no-vacuum') a.vacuum = false;
- else if (k === '--no-backup') a.noBackup = true;
- else if (k === '--backup') a.backup = val();
- else if (k === '--days') a.days = Number(val());
- else if (k === '--batch') a.batch = Number(val());
- else if (k === '--db') a.db = val();
- else if (k === '-h' || k === '--help') a.help = true;
- else throw new Error(`unknown option ${k}`);
- }
- retention.checkDays(a.days);
- if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');
- if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');
- return a;
-}
+const VACUUM_WARNING = `
+On Network, VACUUM rewrites all of network.db and holds its write lock meanwhile: Network's own writes
+(sign-ins, sessions) wait 5 s and then fail. Vacuum with the service stopped, or pass --no-vacuum while
+it is up. Default database: $DB_PATH, else data/network.db; relative paths are from the repo root. The
+backup is a full copy of network.db (mode 0600). Run as the service user so no file changes owner.`;
/** The database the server uses: --db, else $DB_PATH, else data/network.db (relative to the repo root). */
/** The database the server uses: --db, else $DB_PATH, else data/network.db (relative to the repo root). */
function defaultDbPath(args, env = process.env) {
function defaultDbPath(args, env = process.env) {
return path.resolve(ROOT, args.db || env.DB_PATH || path.join('data', 'network.db'));
return path.resolve(ROOT, args.db || env.DB_PATH || path.join('data', 'network.db'));
}
}
-const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;
-function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }
-function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }
-const sameTotals = (a, b) => JSON.stringify(a) === JSON.stringify(b);
-
-async function main(argv, log = console.log) {
- let args;
- try { args = parseArgs(argv); } catch (e) { log(`error: ${e.message}`); return 2; }
- if (args.help) { log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, HELP_LINES).join('\n')); return 0; }
- const dbPath = defaultDbPath(args);
- if (!fs.existsSync(dbPath)) { log(`error: ${dbPath} does not exist`); return 2; }
-
- if (args.apply && !args.backup && !args.noBackup) {
- log('refusing --apply without a backup: pass --backup <new file> (sqlite online backup) or --no-backup');
- return 2;
- }
-
- const db = new Database(dbPath, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });
- try {
- db.pragma('busy_timeout = 5000');
- const walBytes = fileBytes(dbPath + '-wal');
- const before = retention.inspect(db, { days: args.days, paramPrefixes });
- log(`database ${dbPath}`);
- log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);
- log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);
- log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);
- if (before.toScrub) {
- const t = before.toScrub;
- log(`scrub of the rows that stay: ${t.personal} with ip/user_id/city, ${t.paths} paths to template, ${t.referers} referers to origin, ${t.user_agents} user agents to class, ${t.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);
- }
- log(`rate rows ${before.rateRows || 0} (IP counters; the server keeps them in memory now)`);
- for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);
-
- if (!args.apply) {
- log('\ndry run: nothing changed. Re-run with --apply --backup <file> (or --no-backup).');
- return 0;
- }
-
- if (args.backup) {
- const target = path.resolve(args.backup);
- if (fs.existsSync(target)) { log(`error: backup target ${target} already exists; choose a new file`); return 2; }
- const need = before.bytes + walBytes;
- const free = freeBytes(path.dirname(target));
- if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(target)}, the backup needs about ${mb(need)}`); return 2; }
- // A full copy of network.db (accounts, sessions): owner-only from the first byte.
- const umask = process.umask(0o077);
- try { await db.backup(target); } finally { process.umask(umask); }
- fs.chmodSync(target, 0o600);
- const b = new Database(target, { readonly: true });
- try {
- const ok = b.pragma('quick_check', { simple: true });
- const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()
- ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;
- if (ok !== 'ok' || n < before.events) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }
- } finally { b.close(); }
- log(`backup ${target} (${mb(fileBytes(target))}, mode 0600, quick_check ok)`);
- }
-
- db.pragma('secure_delete = ON');
- const totalsBefore = retention.rollupTotals(db);
- const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });
- log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);
- if (args.scrub) {
- await networkAnalytics.prescrub(db, { batchSize: args.batch });
- const s = await retention.scrubEvents(db, { batchSize: args.batch, paramPrefixes });
- const r = retention.scrubRollups(db, { paramPrefixes });
- log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);
- }
- const totalsAfter = retention.rollupTotals(db);
- if (!sameTotals(totalsBefore, totalsAfter)) {
- log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);
- return 1;
- }
- log('rollups totals unchanged');
- if (args.vacuum && (pruned.deleted || args.scrub)) {
- db.pragma('wal_checkpoint(TRUNCATE)');
- db.exec('VACUUM');
- db.pragma('wal_checkpoint(TRUNCATE)');
- log(`vacuumed ${mb(fileBytes(dbPath))}`);
- }
- const after = retention.inspect(db, { days: args.days, paramPrefixes });
- log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);
- return 0;
- } finally {
- db.close();
- }
+function main(argv, log) {
+ return cli.main(argv, {
+ Database,
+ log,
+ root: ROOT,
+ defaultDb: defaultDbPath,
+ ...networkAnalytics.PATH_OPTS,
+ usage: cli.USAGE + '\n' + VACUUM_WARNING,
+ });
}
}
-if (require.main === module) {
- main(process.argv.slice(2)).then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });
-}
+if (require.main === module) cli.run(main);
-module.exports = { main, parseArgs, defaultDbPath };
+module.exports = { main, defaultDbPath };
deleted file mode 100644
deleted file mode 100644
@@ -1,8 +0,0 @@
@@ -1,8 +0,0 @@
-'use strict';
-// Request analytics within ADR-021 (see tracker.js, privacy.js, retention.js).
-const { AnalyticsTracker } = require('./tracker');
-const privacy = require('./privacy');
-const retention = require('./retention');
-const { ensureSchema } = require('./schema');
-
-module.exports = { AnalyticsTracker, privacy, retention, ensureSchema };
@@ -1,114 +1,52 @@
@@ -1,114 +1,52 @@
'use strict';
'use strict';
/**
/**
- * OpenVibe.Network's wiring of the ADR-021 analytics module. The other files in this directory are
- * the module shared with OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);
- * keep them identical to those copies and put anything Network-specific here.
+ * OpenVibe.Network's wiring of the ADR-021 analytics module, openvibe-shared/analytics (the one module
+ * Live, Tools and Network use). Only what is Network-specific lives here:
*
*
- * openAnalytics(dbPath) the tracker for service 'openvibe-network' (NetworkAnalyticsTracker).
- * Network's analytics tables (analytics_events / _hourly / _daily …) live
- * in network.db, as they always
+ * openAnalytics(dbPath) the tracker for service 'openvibe-network'. Network's analytics tables
+ * (analytics_events / _hourly / _daily …) live in network.db, as they always
* have; the tracker gets a better-sqlite3 connection of its own so its
* have; the tracker gets a better-sqlite3 connection of its own so its
* per-connection settings (busy_timeout 250, secure_delete) never apply to
* per-connection settings (busy_timeout 250, secure_delete) never apply to
* the identity connection (busy_timeout 5000).
* the identity connection (busy_timeout 5000).
- * schedulePrune(tracker) the nightly `analytics-prune` job: raw events older than 30 days are
- * deleted in bounded batches (retention.pruneRawEvents); rollups are kept.
+ * schedulePrune(tracker) the nightly `analytics-prune` job (retention.schedulePrune): raw events
+ * older than 30 days are deleted in bounded batches; rollups are kept.
* First run 5 minutes after boot, then every 24 hours; timers are unref'd.
* First run 5 minutes after boot, then every 24 hours; timers are unref'd.
- * prescrub(db) Network's part of the one-time scrub (scripts/analytics-prune.js --scrub):
- * the path rules below applied to legacy rows and rollup top lists, before
- * the shared scrub templates them.
- * PATH_OPTS the normaliser options (param prefixes) the CLI passes to the shared scrub.
+ * PATH_OPTS { paramPrefixes, pathRules }: the path options the tracker and
+ * scripts/analytics-prune.js (prune-cli, for the one-time scrub) both use.
+ *
+ * A request with `Sec-GPC: 1` or `DNT: 1` is not recorded (openvibe-shared v1.4.0).
*/
*/
const path = require('path');
const path = require('path');
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
-const { AnalyticsTracker } = require('./tracker');
-const privacy = require('./privacy');
-const retention = require('./retention');
+const { AnalyticsTracker, retention } = require('openvibe-shared/analytics');
const SERVICE = 'openvibe-network';
const SERVICE = 'openvibe-network';
/**
/**
- * Words whose next path segment is a parameter on Network (on top of privacy.DEFAULT_PARAM_PREFIXES).
+ * Words whose next path segment is a parameter on Network (on top of the package's default prefixes).
* They matter when no Express route matched (a rate-limited or rejected request is templated from
* They matter when no Express route matched (a rate-limited or rejected request is templated from
* its raw path), and to the one-time scrub of rows the old tracker stored raw: /avatar/<username>,
* its raw path), and to the one-time scrub of rows the old tracker stored raw: /avatar/<username>,
* /api/auth/anon/<token>, /api/v1/projects/<slug>.
* /api/auth/anon/<token>, /api/v1/projects/<slug>.
*/
*/
const PARAM_PREFIXES = ['avatar', 'anon', 'projects'];
const PARAM_PREFIXES = ['avatar', 'anon', 'projects'];
-const PATH_OPTS = { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...PARAM_PREFIXES]) };
/**
/**
* A parameter behind two fixed words, which the shared normaliser cannot see: it turns `users` into a
* A parameter behind two fixed words, which the shared normaliser cannot see: it turns `users` into a
* prefix, so `/users/by-username/alex` would keep `alex`. Applied to the raw path before it.
* prefix, so `/users/by-username/alex` would keep `alex`. Applied to the raw path before it.
*/
*/
const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];
const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];
-function preReducePath(p) {
- let s = String(p);
- for (const [re, to] of PATH_RULES) s = s.replace(re, to);
- return s;
-}
-class NetworkAnalyticsTracker extends AnalyticsTracker {
- /** The raw path is only used when no Express route matched (a request refused before routing). */
- record(req, res, entryPath, responseTimeMs) {
- return super.record(req, res, preReducePath(entryPath || ''), responseTimeMs);
- }
-}
+const PATH_OPTS = { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES };
function openAnalytics(dbPath, opts = {}) {
function openAnalytics(dbPath, opts = {}) {
const db = new Database(path.resolve(dbPath));
const db = new Database(path.resolve(dbPath));
db.pragma('journal_mode = WAL');
db.pragma('journal_mode = WAL');
- return new NetworkAnalyticsTracker(db, SERVICE, { retention: false, paramPrefixes: PARAM_PREFIXES, ...opts });
-}
-
-const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
-const tick = () => new Promise((r) => setImmediate(r));
-
-/** PATH_RULES over legacy raw rows (id-range batches) and rollup top lists. Returns { rows, hourly, daily }. */
-async function prescrub(db, { batchSize = retention.DEFAULT_BATCH } = {}) {
- const out = { rows: 0, hourly: 0, daily: 0 };
- if (!tableExists(db, 'analytics_events')) return out;
- db.function('ov_network_path', { deterministic: true }, (p) => (p == null ? null : preReducePath(p)));
- const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();
- const upd = db.prepare("UPDATE analytics_events SET path = ov_network_path(path) WHERE id >= ? AND id < ? AND path LIKE '%/by-username/%' AND path IS NOT ov_network_path(path)");
- for (let from = lo; lo != null && from <= hi; from += batchSize) {
- out.rows += upd.run(from, from + batchSize).changes;
- await tick();
- }
- for (const table of ['analytics_hourly', 'analytics_daily']) {
- if (!tableExists(db, table)) continue;
- const rows = db.prepare(`SELECT id, top_paths FROM ${table} WHERE top_paths LIKE '%/by-username/%'`).all();
- const set = db.prepare(`UPDATE ${table} SET top_paths = ? WHERE id = ?`);
- db.transaction(() => {
- for (const r of rows) {
- const next = retention.reduceTopList(r.top_paths, 'path', preReducePath);
- if (next != null) { set.run(next, r.id); out[table === 'analytics_hourly' ? 'hourly' : 'daily']++; }
- }
- })();
- }
- return out;
+ return new AnalyticsTracker(db, SERVICE, { retention: false, paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES, ...opts });
}
}
-function schedulePrune(tracker, { initialDelayMs = 5 * 60 * 1000, intervalMs = 24 * 60 * 60 * 1000, log = console.log } = {}) {
- let running = false;
- const run = async () => {
- if (running) return null;
- running = true;
- try {
- const out = await retention.pruneRawEvents(tracker.db, { days: retention.MAX_DAYS });
- if (out.deleted) log(`[Analytics] pruned ${out.deleted} raw events older than ${out.cutoff}`);
- return out;
- } catch (err) {
- console.error('[Analytics] analytics-prune error:', err.message);
- return null;
- } finally {
- running = false;
- }
- };
- const first = setTimeout(run, initialDelayMs);
- const every = setInterval(run, intervalMs);
- first.unref();
- every.unref();
- return { run, stop() { clearTimeout(first); clearInterval(every); } };
+function schedulePrune(tracker, opts = {}) {
+ return retention.schedulePrune(tracker.db, { label: 'Analytics', ...opts });
}
}
-module.exports = { SERVICE, PARAM_PREFIXES, PATH_OPTS, NetworkAnalyticsTracker, preReducePath, openAnalytics, schedulePrune, prescrub };
+module.exports = { SERVICE, PARAM_PREFIXES, PATH_RULES, PATH_OPTS, openAnalytics, schedulePrune };
deleted file mode 100644
deleted file mode 100644
@@ -1,211 +0,0 @@
@@ -1,211 +0,0 @@
-'use strict';
-/**
- * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):
- * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a
- * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a
- * precise location or a user/subject id.
- *
- * This file holds the pure reducers that turn a request into those fields. It has no dependencies,
- * so the same file serves OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/);
- * keep the two copies identical.
- */
-
-// ── Bots ──────────────────────────────────────────────────────
-
-const BOT_USER_AGENTS = [
- /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,
- /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,
- /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,
- /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,
- /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,
- /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,
- /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,
- /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,
- /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,
- /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,
- /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,
- /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,
- /statuscake/i, /monitoring/i, /health.?check/i,
-];
-
-const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];
-
-/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */
-const HIGH_REQUEST_RATE = 60;
-
-/** The family name of the first bot pattern the user agent matches, or null. */
-function botName(ua) {
- if (!ua) return null;
- for (const re of BOT_USER_AGENTS) {
- if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';
- }
- return null;
-}
-
-/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */
-function classifyRequest(req) {
- const ua = (req.headers && req.headers['user-agent']) || '';
- if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };
- if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };
- const path = String(req.path || req.url || '').toLowerCase();
- if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };
- return { isBot: false, botType: null, confidence: 0.1 };
-}
-
-// ── User agent → class ───────────────────────────────────────
-
-function parseUserAgent(ua) {
- if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };
- let device = 'desktop';
- if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';
- else if (/ipad|tablet/i.test(ua)) device = 'tablet';
-
- let browser = 'other';
- if (/edg\//i.test(ua)) browser = 'edge';
- else if (/opr\/|opera/i.test(ua)) browser = 'opera';
- else if (/firefox\//i.test(ua)) browser = 'firefox';
- else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';
- else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';
- else if (/trident|msie/i.test(ua)) browser = 'ie';
-
- let os = 'other';
- if (/windows/i.test(ua)) os = 'windows';
- else if (/macintosh|mac os/i.test(ua)) os = 'macos';
- else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';
- else if (/android/i.test(ua)) os = 'android';
- else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';
-
- return { device, browser, os };
-}
-
-const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;
-
-/**
- * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,
- * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps
- * to itself), so a scrub can run over rows that are already reduced.
- */
-function uaClass(ua) {
- if (ua == null || ua === '') return 'none';
- const s = String(ua);
- if (UA_CLASS_RE.test(s)) return s;
- const bot = botName(s);
- if (bot) return `bot:${bot}`;
- const p = parseUserAgent(s);
- return `${p.browser}/${p.os}/${p.device}`;
-}
-
-// ── Referer → origin ─────────────────────────────────────────
-
-/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */
-function refererOrigin(ref) {
- if (!ref) return null;
- try {
- const u = new URL(String(ref));
- if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
- return u.origin;
- } catch {
- return null;
- }
-}
-
-// ── Country ──────────────────────────────────────────────────
-
-/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */
-function countryCode(v) {
- const s = String(v || '').trim().toUpperCase();
- return /^[A-Z]{2}$/.test(s) ? s : null;
-}
-
-// ── Paths → route templates ──────────────────────────────────
-
-/**
- * Words after which the next path segment is a parameter even when it looks like a word
- * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.
- */
-const DEFAULT_PARAM_PREFIXES = [
- 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',
- 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',
- 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',
- 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',
- 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',
-];
-
-const MAX_SEGMENTS = 8;
-const MAX_TEMPLATE_LENGTH = 200;
-const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;
-const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;
-
-/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */
-function looksLikeId(s) {
- if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…
- if (/^\d+$/.test(s)) return true; // 123
- if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid
- if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes
- if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…
- if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid
- if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers
- if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names
- if (s.length > 32) return true;
- return false;
-}
-
-/**
- * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by
- * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated
- * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.
- */
-function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES } = {}) {
- if (raw == null || raw === '') return '/';
- let p = String(raw);
- if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {
- try { p = new URL(p).pathname; } catch { return '/'; }
- }
- p = p.split(/[?#]/)[0];
- const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);
- const segs = p.split('/').filter(Boolean);
- const out = [];
- let expectParam = false;
- for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {
- let s = segs[i];
- try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }
- if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }
- if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }
- if (expectParam) { out.push(':param'); expectParam = false; continue; }
- if (looksLikeId(s)) { out.push(':id'); continue; }
- out.push(s);
- expectParam = prefixes.has(s.toLowerCase());
- }
- if (segs.length > MAX_SEGMENTS) out.push('*');
- const t = '/' + out.join('/');
- return t.length > MAX_TEMPLATE_LENGTH ? t.slice(0, MAX_TEMPLATE_LENGTH) : t;
-}
-
-/**
- * The route template for a finished request: the matched Express route (mount path + route path)
- * when there is one, else the normalised original path. Both go through normalisePath, because a
- * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.
- */
-function routeTemplate(req, fallbackPath, opts) {
- const r = req && req.route;
- if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {
- return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);
- }
- return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);
-}
-
-module.exports = {
- BOT_USER_AGENTS,
- HONEYPOT_PATHS,
- HIGH_REQUEST_RATE,
- DEFAULT_PARAM_PREFIXES,
- botName,
- classifyRequest,
- parseUserAgent,
- uaClass,
- refererOrigin,
- countryCode,
- looksLikeId,
- normalisePath,
- routeTemplate,
-};
deleted file mode 100644
deleted file mode 100644
@@ -1,209 +0,0 @@
@@ -1,209 +0,0 @@
-'use strict';
-/**
- * Raw analytics retention and the one-time scrub (ADR-021).
- *
- * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded
- * batches, yielding to the event loop between batches. Rollups
- * (analytics_hourly / analytics_daily) are never touched. Also clears
- * leftovers: visitor hashes older than yesterday, salts of past days,
- * and analytics_rate_tracking (IP counters live in memory now).
- * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →
- * NULL (authenticated = 1 where a user id was present), path → route
- * template, referer → origin, user_agent → class, session ids that are
- * not rotating ids → NULL.
- * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /
- * top_referers JSON (counts merged, never changed).
- * inspect(db, { days }) read-only counts for a dry run.
- * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.
- *
- * Used by the server (scheduled prune) and scripts/analytics-prune.js (CLI).
- * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).
- */
-
-const { ensureSchema } = require('./schema');
-const privacy = require('./privacy');
-
-const MAX_DAYS = 30;
-const DEFAULT_BATCH = 5000;
-const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;
-
-const pad = (n) => String(n).padStart(2, '0');
-function sqlTime(ms) {
- const d = new Date(ms);
- return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
-}
-
-function checkDays(days) {
- const n = Number(days);
- if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {
- throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);
- }
- return n;
-}
-
-/** The created_at bound: rows strictly older than this are pruned. */
-function cutoffFor(days, nowMs = Date.now()) {
- return sqlTime(nowMs - checkDays(days) * 86400000);
-}
-
-const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
-const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);
-const tick = () => new Promise((r) => setImmediate(r));
-
-/**
- * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write
- * transaction). Returns { cutoff, deleted, batches, complete }.
- */
-async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {
- const cutoff = cutoffFor(days, now());
- const out = { cutoff, deleted: 0, batches: 0, complete: true };
- if (!tableExists(db, 'analytics_events')) return out;
- const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');
- for (;;) {
- if (out.batches >= maxBatches) { out.complete = false; break; }
- const n = del.run(cutoff, batchSize).changes;
- out.batches++;
- out.deleted += n;
- if (n < batchSize) break;
- await tick();
- }
- const today = sqlTime(now()).slice(0, 10);
- const yesterday = sqlTime(now() - 86400000).slice(0, 10);
- if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);
- if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);
- if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();
- return out;
-}
-
-/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */
-function registerFunctions(db, opts) {
- db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));
- db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));
- db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));
- db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));
-}
-
-/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */
-async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes } = {}) {
- ensureSchema(db);
- registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);
- const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();
- const out = { rows: 0, batches: 0 };
- if (lo == null) return out;
- const upd = db.prepare(`
- UPDATE analytics_events SET
- authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,
- ip = NULL, user_id = NULL, city = NULL,
- session_id = ov_rotating_session(session_id),
- path = ov_route_template(path),
- referer = ov_referer_origin(referer),
- user_agent = ov_ua_class(user_agent)
- WHERE id >= ? AND id < ?
- `);
- for (let from = lo; from <= hi; from += batchSize) {
- out.rows += upd.run(from, from + batchSize).changes;
- out.batches++;
- await tick();
- }
- return out;
-}
-
-/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */
-function reduceTopList(json, key, fn) {
- let list;
- try { list = JSON.parse(json); } catch { return null; }
- if (!Array.isArray(list)) return null;
- const merged = new Map();
- for (const item of list) {
- if (!item || typeof item !== 'object') continue;
- const k = fn(item[key]);
- if (k == null) continue;
- merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));
- }
- const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);
- const text = JSON.stringify(next);
- return text === json ? null : text;
-}
-
-/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */
-function scrubRollups(db, { paramPrefixes } = {}) {
- const opts = paramPrefixes ? { paramPrefixes } : undefined;
- const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));
- const out = { hourly: 0, daily: 0 };
- for (const table of ['analytics_hourly', 'analytics_daily']) {
- if (!tableExists(db, table)) continue;
- const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();
- const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);
- db.transaction(() => {
- for (const r of rows) {
- const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;
- const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;
- if (paths == null && refs == null) continue;
- upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);
- out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;
- }
- })();
- }
- return out;
-}
-
-/** Sums of every rollup counter; equal before and after a prune or scrub. */
-function rollupTotals(db) {
- const out = {};
- if (tableExists(db, 'analytics_hourly')) {
- out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
- TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,
- TOTAL(error_count) AS error_count FROM analytics_hourly`).get();
- }
- if (tableExists(db, 'analytics_daily')) {
- out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
- TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,
- TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();
- }
- return out;
-}
-
-/** Read-only counts for a dry run. */
-function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes } = {}) {
- const cutoff = cutoffFor(days, now());
- const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };
- const pageSize = db.pragma('page_size', { simple: true });
- out.bytes = db.pragma('page_count', { simple: true }) * pageSize;
- out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;
- out.rollups = rollupTotals(db);
- if (!tableExists(db, 'analytics_events')) return out;
- registerFunctions(db, paramPrefixes ? { paramPrefixes } : undefined);
- const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();
- out.events = e.n;
- out.oldest = e.oldest;
- out.newest = e.newest;
- out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);
- out.remaining = out.events - out.older;
- const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))
- .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';
- const r = db.prepare(`
- SELECT
- COUNT(*) FILTER (WHERE ${personal}) AS personal,
- COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,
- COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,
- COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,
- COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions
- FROM analytics_events WHERE created_at >= ?
- `).get(cutoff);
- out.toScrub = r;
- out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;
- return out;
-}
-
-module.exports = {
- MAX_DAYS,
- DEFAULT_BATCH,
- checkDays,
- cutoffFor,
- pruneRawEvents,
- scrubEvents,
- scrubRollups,
- rollupTotals,
- inspect,
- reduceTopList,
-};
deleted file mode 100644
deleted file mode 100644
@@ -1,123 +0,0 @@
@@ -1,123 +0,0 @@
-'use strict';
-/**
- * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the
- * existing analytics.db files and the admin dashboards keep working; the columns that used to hold
- * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and
- * user_agent / referer / path hold the reduced forms (class, origin, route template).
- *
- * Added:
- * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)
- * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;
- * rows are deleted once the day's rollup is final (≤ ~25 h)
- * analytics_day_salts the random salt of the current UTC day (so a restart does not
- * double-count); deleted together with that day's hashes
- * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)
- */
-
-const ANALYTICS_SCHEMA = `
- CREATE TABLE IF NOT EXISTS analytics_events (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- service TEXT NOT NULL,
- event_type TEXT NOT NULL DEFAULT 'pageview',
- path TEXT,
- method TEXT DEFAULT 'GET',
- status_code INTEGER,
- response_time_ms INTEGER,
- user_id INTEGER,
- session_id TEXT,
- ip TEXT,
- country TEXT,
- city TEXT,
- user_agent TEXT,
- referer TEXT,
- is_bot INTEGER DEFAULT 0,
- bot_type TEXT,
- device_type TEXT,
- browser TEXT,
- os TEXT,
- created_at DATETIME DEFAULT CURRENT_TIMESTAMP
- );
-
- CREATE TABLE IF NOT EXISTS analytics_hourly (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- service TEXT NOT NULL,
- hour TEXT NOT NULL,
- pageviews INTEGER DEFAULT 0,
- api_calls INTEGER DEFAULT 0,
- unique_visitors INTEGER DEFAULT 0,
- unique_users INTEGER DEFAULT 0,
- bot_hits INTEGER DEFAULT 0,
- avg_response_ms INTEGER DEFAULT 0,
- error_count INTEGER DEFAULT 0,
- bandwidth_bytes INTEGER DEFAULT 0,
- top_paths TEXT,
- top_referers TEXT,
- UNIQUE(service, hour)
- );
-
- CREATE TABLE IF NOT EXISTS analytics_daily (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- service TEXT NOT NULL,
- date TEXT NOT NULL,
- pageviews INTEGER DEFAULT 0,
- api_calls INTEGER DEFAULT 0,
- unique_visitors INTEGER DEFAULT 0,
- unique_users INTEGER DEFAULT 0,
- new_users INTEGER DEFAULT 0,
- bot_hits INTEGER DEFAULT 0,
- avg_response_ms INTEGER DEFAULT 0,
- error_count INTEGER DEFAULT 0,
- top_paths TEXT,
- top_referers TEXT,
- top_countries TEXT,
- device_breakdown TEXT,
- browser_breakdown TEXT,
- UNIQUE(service, date)
- );
-
- CREATE TABLE IF NOT EXISTS analytics_rate_tracking (
- ip TEXT NOT NULL,
- window_start INTEGER NOT NULL,
- hit_count INTEGER DEFAULT 1,
- PRIMARY KEY (ip, window_start)
- );
-
- CREATE TABLE IF NOT EXISTS analytics_visitor_days (
- service TEXT NOT NULL,
- day TEXT NOT NULL,
- hour TEXT NOT NULL,
- vhash TEXT NOT NULL,
- authenticated INTEGER NOT NULL DEFAULT 0,
- PRIMARY KEY (service, hour, vhash)
- );
-
- CREATE TABLE IF NOT EXISTS analytics_day_salts (
- service TEXT NOT NULL,
- day TEXT NOT NULL,
- salt TEXT NOT NULL,
- PRIMARY KEY (service, day)
- );
-
- CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);
- CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);
- CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);
-
- -- Indexes over the retired personal columns.
- DROP INDEX IF EXISTS idx_analytics_events_ip;
- DROP INDEX IF EXISTS idx_analytics_events_user;
-`;
-
-/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */
-function ensureSchema(db) {
- db.exec(ANALYTICS_SCHEMA);
- const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);
- if (!cols.includes('authenticated')) {
- db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');
- }
-}
-
-module.exports = { ANALYTICS_SCHEMA, ensureSchema };
deleted file mode 100644
deleted file mode 100644
@@ -1,728 +0,0 @@
@@ -1,728 +0,0 @@
-'use strict';
-/**
- * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the openvibe-shared
- * AnalyticsTracker (same tables, same getStats/getOverview/getBotAnalysis shapes), minus the personal
- * data:
- *
- * raw row service, event_type, route template, method, status, response time, rotating session
- * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,
- * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.
- * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after
- * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.
- * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped
- * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).
- * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.
- * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as
- * the day is over); hashes go to analytics_visitor_days only, never to raw events,
- * and are deleted by the first aggregation after their day ends, right after that day's
- * final hourly/daily rollup — so no hash or salt outlives its day by more than about
- * one aggregation interval (1 h). Rollups keep only the counts.
- *
- * Retention of raw events (30 days) is ./retention.js; this tracker only writes and rolls up.
- *
- * Same file in OpenVibe.Live (server/analytics/) and OpenVibe.Tools (apps/_shared/analytics/).
- */
-
-const crypto = require('crypto');
-const { ensureSchema } = require('./schema');
-const privacy = require('./privacy');
-const retention = require('./retention');
-
-const SESSION_IDLE_MS = 30 * 60 * 1000;
-const MAX_TRACKED_CLIENTS = 200000;
-const MAX_BUFFER = 5000;
-const SESSION_ID_RE = /^[0-9a-f]{16}$/;
-
-const pad = (n) => String(n).padStart(2, '0');
-/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */
-function sqlTime(ms) {
- const d = new Date(ms);
- return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
-}
-const dayOf = (ms) => sqlTime(ms).slice(0, 10);
-const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';
-const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');
-
-function isStaticOrHealth(path) {
- return path === '/api/health' || path === '/health' ||
- /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);
-}
-
-class AnalyticsTracker {
- /**
- * @param {object} db better-sqlite3 handle (its own analytics database)
- * @param {string} service service name ('live', 'openvibe-yt', …)
- * @param {object} [opts]
- * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter
- * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:
- * run the raw-event prune on a timer (Tools). Live passes false and schedules
- * retention.pruneRawEvents through server/utils/jobs.js instead.
- * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)
- * @param {function} [opts.now] clock (tests)
- */
- constructor(db, service, opts = {}) {
- this.db = db;
- this.service = service;
- this._now = opts.now || Date.now;
- this._pathOpts = opts.paramPrefixes
- ? { paramPrefixes: new Set([...privacy.DEFAULT_PARAM_PREFIXES, ...opts.paramPrefixes]) }
- : undefined;
- this._buffer = [];
- this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }
- this._sessions = new Map(); // vhash -> { sid, last }
- this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)
- this._saltDay = null;
- this._salt = null;
- this._timers = [];
-
- ensureSchema(db);
- try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }
- try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }
- db.exec('DELETE FROM analytics_rate_tracking');
-
- this._insertEvent = db.prepare(`
- INSERT INTO analytics_events
- (service, event_type, path, method, status_code, response_time_ms,
- user_id, session_id, ip, country, city, user_agent, referer,
- is_bot, bot_type, device_type, browser, os, authenticated, created_at)
- VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `);
- this._upsertVisitor = db.prepare(`
- INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)
- ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)
- `);
- this._write = db.transaction((events, visitors) => {
- for (const e of events) {
- this._insertEvent.run(
- e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,
- e.session_id, e.country, e.user_agent, e.referer,
- e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,
- );
- }
- for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);
- });
-
- if (opts.timers !== false) {
- const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
- const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
- every(() => { this.flush(); this._sweep(); }, 5000);
- every(() => this.aggregate(), 60 * 60 * 1000);
- once(() => this.aggregate(), 10_000);
- if (opts.retention) {
- const r = opts.retention;
- const run = () => retention.pruneRawEvents(db, { days: r.days || retention.MAX_DAYS })
- .then((out) => { if (out.deleted) console.log(`[Analytics:${service}] pruned ${out.deleted} raw events older than ${out.cutoff}`); })
- .catch((err) => console.error(`[Analytics:${service}] prune error:`, err.message));
- once(run, r.initialDelayMs != null ? r.initialDelayMs : 5 * 60 * 1000);
- every(run, r.intervalMs || 24 * 60 * 60 * 1000);
- }
- }
- }
-
- // ── Per-request reducers ──────────────────────────────────
-
- _daySalt(day) {
- if (this._saltDay === day) return this._salt;
- let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
- if (!row) {
- this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')
- .run(this.service, day, crypto.randomBytes(32).toString('hex'));
- row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
- }
- // Earlier days' salts are never needed again: their hashes are already written.
- this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);
- this._saltDay = day;
- this._salt = row.salt;
- this._sessions.clear();
- return this._salt;
- }
-
- _visitorHash(ip, ua, day) {
- return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);
- }
-
- _sessionId(vhash, nowMs) {
- let s = this._sessions.get(vhash);
- if (!s || nowMs - s.last > SESSION_IDLE_MS) {
- s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };
- this._sessions.set(vhash, s);
- }
- s.last = nowMs;
- return s.sid;
- }
-
- /** Hits from this client in the current and previous minute (memory only). */
- _rateHit(ip, nowMs) {
- const minute = Math.floor(nowMs / 60000);
- let r = this._rates.get(ip);
- if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }
- if (r.minute !== minute) {
- r.prev = r.minute === minute - 1 ? r.count : 0;
- r.count = 0;
- r.minute = minute;
- }
- r.count++;
- return r.count + r.prev;
- }
-
- _sweep() {
- const nowMs = this._now();
- const minute = Math.floor(nowMs / 60000);
- for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);
- for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);
- if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();
- if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();
- }
-
- _push(event, visitor) {
- if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();
- this._buffer.push(event);
- if (visitor) {
- const key = `${visitor.hour}|${visitor.vhash}`;
- const prev = this._visitors.get(key);
- if (!prev) this._visitors.set(key, visitor);
- else if (visitor.authenticated) prev.authenticated = true;
- }
- if (this._buffer.length >= 100) this.flush();
- }
-
- /**
- * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted
- * router is relative by the time the response finishes).
- */
- record(req, res, entryPath, responseTimeMs) {
- const nowMs = this._now();
- const headers = req.headers || {};
- const ua = String(headers['user-agent'] || '');
- const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');
- const bot = privacy.classifyRequest({ headers, path: entryPath });
- if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {
- bot.isBot = true;
- bot.botType = 'rate_limit';
- bot.confidence = 0.80;
- }
- const parsed = privacy.parseUserAgent(ua);
- const day = dayOf(nowMs);
- const vhash = this._visitorHash(ip, ua, day);
- const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));
- const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');
- this._push({
- service: this.service,
- event_type: isApi ? 'api_call' : 'pageview',
- path: privacy.routeTemplate(req, entryPath, this._pathOpts),
- method: req.method,
- status_code: res.statusCode,
- response_time_ms: responseTimeMs,
- session_id: this._sessionId(vhash, nowMs),
- country: privacy.countryCode(headers['cf-ipcountry']),
- user_agent: privacy.uaClass(ua),
- referer: privacy.refererOrigin(headers.referer || headers.referrer),
- is_bot: bot.isBot,
- bot_type: bot.botType,
- device_type: parsed.device,
- browser: parsed.browser,
- os: parsed.os,
- authenticated,
- created_at: sqlTime(nowMs),
- }, { day, hour: hourOf(nowMs), vhash, authenticated });
- }
-
- /** Express middleware: one row per finished request (health checks and static assets skipped). */
- middleware() {
- return (req, res, next) => {
- const entryPath = req.path || '';
- if (isStaticOrHealth(entryPath)) return next();
- const started = Date.now();
- let done = false;
- const finish = () => {
- if (done) return;
- done = true;
- try { this.record(req, res, entryPath, Date.now() - started); }
- catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }
- };
- res.once('finish', finish);
- res.once('close', finish);
- next();
- };
- }
-
- /**
- * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,
- * referer and user_agent are reduced like a request's; session_id is kept only if it is one of
- * this tracker's rotating ids.
- */
- trackEvent(eventType, data = {}) {
- const nowMs = this._now();
- const ua = data.user_agent ? String(data.user_agent) : '';
- const parsed = ua ? privacy.parseUserAgent(ua) : {};
- this._push({
- service: this.service,
- event_type: String(eventType || 'event').slice(0, 64),
- path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,
- method: data.method || null,
- status_code: data.status_code || null,
- response_time_ms: data.response_time_ms || null,
- session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,
- country: privacy.countryCode(data.country),
- user_agent: ua ? privacy.uaClass(ua) : null,
- referer: privacy.refererOrigin(data.referer),
- is_bot: !!data.is_bot,
- bot_type: data.bot_type || null,
- device_type: data.device_type || parsed.device || null,
- browser: data.browser || parsed.browser || null,
- os: data.os || parsed.os || null,
- authenticated: !!data.authenticated,
- created_at: sqlTime(nowMs),
- }, null);
- }
-
- /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */
- flush() {
- if (!this._buffer.length && !this._visitors.size) return;
- const events = this._buffer.splice(0);
- const visitors = [...this._visitors.values()];
- this._visitors.clear();
- try {
- this._write(events, visitors);
- } catch (err) {
- console.error(`[Analytics:${this.service}] Flush error:`, err.message);
- this._buffer = events.slice(-MAX_BUFFER);
- for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);
- }
- }
-
- // ── Rollups ───────────────────────────────────────────────
-
- /**
- * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has
- * visitor hashes (finalised here), after which those hashes are deleted.
- */
- aggregate() {
- try {
- this.flush();
- const nowMs = this._now();
- const today = dayOf(nowMs);
- const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);
- const days = new Set([today, dayOf(nowMs - 86400000)]);
- const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);
- for (const p of pending) { hours.add(p.hour); days.add(p.day); }
- for (const h of hours) this._aggregateHour(h);
- for (const d of days) this._aggregateDay(d);
- this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);
- this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);
- } catch (err) {
- console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);
- }
- }
-
- _aggregateHour(hourStart) {
- const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);
- const svc = this.service;
- const stats = this.db.prepare(`
- SELECT
- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at < ?
- `).get(svc, hourStart, nextHour);
- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
-
- // Uniques only grow within a period, so never lower a stored count (restarts, or rows
- // counted before this tracker when the old one still stored IPs).
- const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);
- const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};
-
- const topPaths = this.db.prepare(`
- SELECT path, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0
- GROUP BY path ORDER BY cnt DESC LIMIT 10
- `).all(svc, hourStart, nextHour);
- const topReferers = this.db.prepare(`
- SELECT referer, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0
- GROUP BY referer ORDER BY cnt DESC LIMIT 10
- `).all(svc, hourStart, nextHour);
-
- this.db.prepare(`
- INSERT INTO analytics_hourly
- (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- ON CONFLICT(service, hour) DO UPDATE SET
- pageviews = excluded.pageviews, api_calls = excluded.api_calls,
- unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
- bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,
- error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers
- `).run(
- svc, hourStart, stats.pageviews, stats.api_calls,
- Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
- stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
- JSON.stringify(topPaths), JSON.stringify(topReferers),
- );
- }
-
- _aggregateDay(date) {
- const dayStart = date + ' 00:00:00';
- const dayEnd = date + ' 23:59:59';
- const svc = this.service;
- const q = (sql) => this.db.prepare(sql);
- const stats = q(`
- SELECT
- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ?
- `).get(svc, dayStart, dayEnd);
- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
-
- const v = q(`
- SELECT COUNT(DISTINCT vhash) AS visitors,
- COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users
- FROM analytics_visitor_days WHERE service = ? AND day = ?
- `).get(svc, date);
- const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};
-
- const range = [svc, dayStart, dayEnd];
- const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);
- const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);
- const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);
- const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);
- const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);
-
- q(`
- INSERT INTO analytics_daily
- (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,
- avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- ON CONFLICT(service, date) DO UPDATE SET
- pageviews = excluded.pageviews, api_calls = excluded.api_calls,
- unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
- new_users = excluded.new_users, bot_hits = excluded.bot_hits,
- avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,
- top_paths = excluded.top_paths, top_referers = excluded.top_referers,
- top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,
- browser_breakdown = excluded.browser_breakdown
- `).run(
- svc, date, stats.pageviews, stats.api_calls,
- Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
- // New users needs a user id history, which analytics no longer has: keep what was
- // recorded before, NULL (not measured) from now on.
- prev.new_users != null ? prev.new_users : null,
- stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
- JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),
- JSON.stringify(devices), JSON.stringify(browsers),
- );
- }
-
- // ── Reads (admin dashboards) ─────────────────────────────
- // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,
- // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based
- // numbers (daily/hourly unique_visitors) are true unique visitors per period.
-
- getStats(options = {}) {
- const { days = 30, hours, service } = options;
- const svc = service || this.service;
- const nowMs = this._now();
- const rangeMs = hours ? hours * 3600000 : days * 86400000;
- const rangeCutoff = sqlTime(nowMs - rangeMs);
- const isSubDay = hours && hours < 24;
- const effectiveDays = hours ? hours / 24 : days;
- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;
- const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;
- const q = (sql) => this.db.prepare(sql);
- const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };
-
- let daily = [];
- if (!isSubDay) {
- daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));
- }
-
- const hourlyLimit = hours ? Math.min(hours, 168) : 24;
- const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);
- const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);
-
- let timeBuckets = [];
- if (isSubDay) {
- timeBuckets = q(`
- SELECT ${bucketExpr} as bucket,
- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,
- COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,
- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- GROUP BY bucket ORDER BY bucket ASC
- `).all(svc, rangeCutoff);
- }
-
- let summary;
- if (hours && hours < 48) {
- summary = q(`
- SELECT
- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,
- COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,
- COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,
- COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,
- COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- `).get(svc, rangeCutoff);
- } else {
- summary = q(`
- SELECT
- SUM(pageviews) as total_pageviews,
- SUM(api_calls) as total_api_calls,
- SUM(unique_visitors) as total_unique_visitors,
- SUM(unique_users) as total_unique_users,
- SUM(bot_hits) as total_bot_hits,
- SUM(error_count) as total_errors,
- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
- FROM analytics_daily
- WHERE service = ? AND date >= ?
- `).get(svc, dayOf(nowMs - rangeMs));
- }
-
- const realtime = q(`
- SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
- FROM analytics_events WHERE service = ? AND created_at >= ?
- `).get(svc, sqlTime(nowMs - 300000));
-
- const human = [svc, rangeCutoff];
- const topPages = q(`
- SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors
- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY path ORDER BY hits DESC LIMIT 20
- `).all(...human);
- const topReferers = q(`
- SELECT referer, COUNT(*) as hits FROM analytics_events
- WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0
- GROUP BY referer ORDER BY hits DESC LIMIT 15
- `).all(...human);
- const botBreakdown = q(`
- SELECT bot_type, COUNT(*) as hits FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 1
- GROUP BY bot_type ORDER BY hits DESC LIMIT 15
- `).all(...human);
- const statusCodes = q(`
- SELECT CASE
- WHEN status_code >= 200 AND status_code < 300 THEN '2xx'
- WHEN status_code >= 300 AND status_code < 400 THEN '3xx'
- WHEN status_code >= 400 AND status_code < 500 THEN '4xx'
- WHEN status_code >= 500 THEN '5xx'
- ELSE 'other' END as group_code,
- COUNT(*) as cnt
- FROM analytics_events WHERE service = ? AND created_at >= ?
- GROUP BY group_code ORDER BY cnt DESC
- `).all(...human);
- const breakdown = (col) => q(`
- SELECT ${col}, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY ${col} ORDER BY cnt DESC
- `).all(...human);
- const deviceBreakdown = breakdown('device_type');
- const browserBreakdown = breakdown('browser');
- const osBreakdown = breakdown('os');
- const countryBreakdown = q(`
- SELECT country, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0
- GROUP BY country ORDER BY cnt DESC LIMIT 20
- `).all(...human);
-
- const responsePercentiles = safe(() => {
- const vals = q(`
- SELECT response_time_ms FROM analytics_events
- WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
- ORDER BY response_time_ms ASC
- `).pluck().all(...human);
- if (!vals.length) return {};
- const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];
- return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };
- }, {});
-
- const bandwidth = safe(() => {
- const bw = q(`
- SELECT COUNT(*) as total_requests,
- COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,
- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,
- SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes
- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
- `).get(...human);
- return bw && {
- total_requests: bw.total_requests || 0,
- page_requests: bw.page_requests || 0,
- api_requests: bw.api_requests || 0,
- estimated_bytes: bw.estimated_bytes || 0,
- };
- }, {});
-
- const authBreakdown = safe(() => q(`
- SELECT
- COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,
- COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,
- COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,
- COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions
- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
- `).get(...human), {});
-
- const errorTrend = safe(() => (isSubDay
- ? q(`
- SELECT ${bucketExpr} as bucket, COUNT(*) AS total,
- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
- COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors
- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY bucket ORDER BY bucket ASC
- `).all(...human)
- : q(`
- SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors
- FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC
- `).all(svc, hourlyCutoff)), []);
-
- const peakHours = safe(() => q(`
- SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,
- COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms
- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY hour_of_day ORDER BY hour_of_day ASC
- `).all(...human), []);
-
- const topApiEndpoints = safe(() => q(`
- SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,
- COUNT(*) FILTER (WHERE status_code >= 400) AS errors
- FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0
- GROUP BY path, method ORDER BY hits DESC LIMIT 20
- `).all(...human), []);
-
- const sessionCount = safe(() => q(`
- SELECT COUNT(DISTINCT session_id) FROM analytics_events
- WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0
- `).pluck().get(...human), 0);
-
- // New vs returning needs a visitor identity that outlives a day; analytics keeps none.
- const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };
-
- const slowestEndpoints = safe(() => q(`
- SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms
- FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
- GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10
- `).all(...human), []);
-
- const authTrend = safe(() => q(`
- SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,
- COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,
- COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,
- COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,
- COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits
- FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY bucket ORDER BY bucket ASC
- `).all(...human), []);
-
- return {
- service: svc,
- period_days: effectiveDays,
- period_hours: hours || null,
- summary: summary || {},
- realtime: realtime || {},
- daily,
- hourly,
- timeBuckets,
- topPages,
- topReferers,
- botBreakdown,
- statusCodes,
- deviceBreakdown,
- browserBreakdown,
- osBreakdown,
- countryBreakdown,
- responsePercentiles,
- bandwidth,
- authBreakdown,
- errorTrend,
- peakHours,
- topApiEndpoints,
- sessionCount,
- visitorTypes,
- slowestEndpoints,
- authTrend,
- };
- }
-
- getOverview(days = 30) {
- const cutoff = dayOf(this._now() - days * 86400000);
- const services = this.db.prepare(`
- SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
- SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,
- SUM(bot_hits) as bot_hits, SUM(error_count) as errors,
- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
- FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC
- `).all(cutoff);
- const totals = this.db.prepare(`
- SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,
- SUM(bot_hits) as bot_hits, SUM(error_count) as errors
- FROM analytics_daily WHERE date >= ?
- `).get(cutoff);
- const dailyTrend = this.db.prepare(`
- SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
- SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits
- FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
- `).all(cutoff);
- const realtime = this.db.prepare(`
- SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
- FROM analytics_events WHERE created_at >= ? GROUP BY service
- `).all(sqlTime(this._now() - 300000));
- return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };
- }
-
- /**
- * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes
- * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is
- * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.
- */
- getBotAnalysis(days = 30) {
- const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);
- const topBotIPs = this.db.prepare(`
- SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,
- COUNT(DISTINCT session_id) AS sessions,
- MIN(created_at) as first_seen, MAX(created_at) as last_seen
- FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
- GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25
- `).all(since);
- const botTrend = this.db.prepare(`
- SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits
- FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
- `).all(since.slice(0, 10));
- const botTypes = this.db.prepare(`
- SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions
- FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
- GROUP BY bot_type ORDER BY hits DESC
- `).all(since);
- const suspiciousIPs = this.db.prepare(`
- SELECT NULL AS ip, session_id, COUNT(*) as total_hits,
- COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,
- COUNT(DISTINCT path) as unique_paths,
- MIN(created_at) as first_seen, MAX(created_at) as last_seen
- FROM analytics_events
- WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL
- GROUP BY session_id HAVING total_hits > 500
- ORDER BY total_hits DESC LIMIT 20
- `).all(sqlTime(this._now() - 7 * 86400000));
- return { topBotIPs, botTrend, botTypes, suspiciousIPs };
- }
-
- destroy() {
- this.flush();
- for (const t of this._timers) { clearInterval(t); clearTimeout(t); }
- this._timers = [];
- this._rates.clear();
- this._sessions.clear();
- }
-}
-
-module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };
@@ -316,9 +316,10 @@ app.locals.urlRegistry = resolvedRegistry;
@@ -316,9 +316,10 @@ app.locals.urlRegistry = resolvedRegistry;
config._registry = resolvedRegistry;
config._registry = resolvedRegistry;
// ── Analytics Tracking (ADR-021) ──────────────────────────────
// ── Analytics Tracking (ADR-021) ──────────────────────────────
-// Same tables in network.db, on a connection of the tracker's own (server/analytics/network.js).
-// Raw rows: route template, rotating session id, user-agent class, referer origin; never an IP or a
-// user id. Pruned after 30 days by the analytics-prune job below; rollups are kept.
+// openvibe-shared/analytics; same tables in network.db, on a connection of the tracker's own
+// (server/analytics/network.js). Raw rows: route template, rotating session id, user-agent class,
+// referer origin; never an IP or a user id. A Sec-GPC: 1 / DNT: 1 request is not recorded. Pruned after
+// 30 days by the analytics-prune job below; rollups are kept.
const analytics = networkAnalytics.openAnalytics(config.db.path);
const analytics = networkAnalytics.openAnalytics(config.db.path);
app.locals.analytics = analytics;
app.locals.analytics = analytics;
app.use(analytics.middleware());
app.use(analytics.middleware());
@@ -1,8 +1,11 @@
@@ -1,8 +1,11 @@
'use strict';
'use strict';
-// ADR-021 analytics bounds on OpenVibe.Network (server/analytics/, scripts/analytics-prune.js):
+// ADR-021 analytics bounds on OpenVibe.Network (openvibe-shared/analytics wired by
+// server/analytics/network.js, scripts/analytics-prune.js):
+// - Network's path options (paramPrefixes, pathRules) template its usernames, anon tokens and slugs;
// - real requests through Network's own routers (auth, OAuth, avatars, admin analytics) leave no IP,
// - real requests through Network's own routers (auth, OAuth, avatars, admin analytics) leave no IP,
// user id, city, raw user agent, full referer, query value or username in any analytics table;
// user id, city, raw user agent, full referer, query value or username in any analytics table;
-// the session id is a rotating id, paths are route templates;
+// the session id is a rotating id, paths are route templates, every row is a valid analytics/event.v1;
+// - a request with Sec-GPC: 1 or DNT: 1 is not recorded at all;
// - the tracker runs on its own connection to network.db: the identity connection keeps busy_timeout 5000;
// - the tracker runs on its own connection to network.db: the identity connection keeps busy_timeout 5000;
// - the admin dashboards keep their shapes (bot rows carry ua_class / session_id, never an IP);
// - the admin dashboards keep their shapes (bot rows carry ua_class / session_id, never an IP);
// - the analytics-prune job deletes raw rows older than 30 days and leaves rollups alone;
// - the analytics-prune job deletes raw rows older than 30 days and leaves rollups alone;
@@ -20,8 +23,8 @@ const cookieParser = require('cookie-parser');
@@ -20,8 +23,8 @@ const cookieParser = require('cookie-parser');
const jwt = require('jsonwebtoken');
const jwt = require('jsonwebtoken');
const Database = require('better-sqlite3');
const Database = require('better-sqlite3');
const { initDb } = require('../server/db/database');
const { initDb } = require('../server/db/database');
-const { privacy, retention } = require('../server/analytics');
-const { sqlTime } = require('../server/analytics/tracker');
+const { privacy, retention, event } = require('openvibe-shared/analytics');
+const { sqlTime } = require('openvibe-shared/analytics/tracker');
const networkAnalytics = require('../server/analytics/network');
const networkAnalytics = require('../server/analytics/network');
const cli = require('../scripts/analytics-prune');
const cli = require('../scripts/analytics-prune');
@@ -46,7 +49,8 @@ function dumpAnalytics(db) {
@@ -46,7 +49,8 @@ function dumpAnalytics(db) {
(async () => {
(async () => {
await check('Network paths: usernames, anon tokens and project slugs become parameters', () => {
await check('Network paths: usernames, anon tokens and project slugs become parameters', () => {
- const t = (p) => privacy.normalisePath(networkAnalytics.preReducePath(p), networkAnalytics.PATH_OPTS);
+ const opts = privacy.pathOptions(networkAnalytics.PATH_OPTS);
+ const t = (p) => privacy.normalisePath(p, opts);
const cases = {
const cases = {
'/avatar/alex?s=96': '/avatar/:param',
'/avatar/alex?s=96': '/avatar/:param',
'/api/auth/anon/k3yF00barBaz?x=1': '/api/auth/anon/:param',
'/api/auth/anon/k3yF00barBaz?x=1': '/api/auth/anon/:param',
@@ -114,6 +118,9 @@ function dumpAnalytics(db) {
@@ -114,6 +118,9 @@ function dumpAnalytics(db) {
await get('/no/such/page/424242?email=alex%40example.com', hdr());
await get('/no/such/page/424242?email=alex%40example.com', hdr());
assert.strictEqual(await get('/internal/users/by-username/carolsecretname', hdr()), 403);
assert.strictEqual(await get('/internal/users/by-username/carolsecretname', hdr()), 403);
await get('/api/auth/me', hdr({ 'user-agent': 'curl/8.5.0' }));
await get('/api/auth/me', hdr({ 'user-agent': 'curl/8.5.0' }));
+ // Opted out (Sec-GPC / DNT): answered as usual, recorded nowhere.
+ assert.strictEqual(await get('/avatar/gpcsecretname', hdr({ 'sec-gpc': '1', 'x-forwarded-for': '198.51.100.61' })), 200);
+ assert.strictEqual(await get('/internal/users/by-username/dntsecretname', hdr({ dnt: '1', 'x-forwarded-for': '198.51.100.62' })), 403);
await new Promise((r) => setTimeout(r, 50));
await new Promise((r) => setTimeout(r, 50));
analytics.flush();
analytics.flush();
@@ -129,6 +136,7 @@ function dumpAnalytics(db) {
@@ -129,6 +136,7 @@ function dumpAnalytics(db) {
assert.ok(/^[0-9a-f]{16}$/.test(r.session_id), r.session_id);
assert.ok(/^[0-9a-f]{16}$/.test(r.session_id), r.session_id);
assert.strictEqual(r.country, 'NL');
assert.strictEqual(r.country, 'NL');
assert.ok(/^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/.test(r.user_agent), r.user_agent);
assert.ok(/^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/.test(r.user_agent), r.user_agent);
+ assert.deepStrictEqual(event.checkRow(r), [], JSON.stringify(r));
}
}
assert.deepStrictEqual(rows.map((r) => r.path), [
assert.deepStrictEqual(rows.map((r) => r.path), [
'/api/auth/me', '/avatar/:username', '/avatar/:param', '/oauth/authorize',
'/api/auth/me', '/avatar/:username', '/avatar/:param', '/oauth/authorize',
@@ -142,7 +150,8 @@ function dumpAnalytics(db) {
@@ -142,7 +150,8 @@ function dumpAnalytics(db) {
const everything = dumpAnalytics(db);
const everything = dumpAnalytics(db);
for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', 'Amsterdam', 'supersecret', 'secret-query',
for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', 'Amsterdam', 'supersecret', 'secret-query',
'/search', 'statesecret', 'openvibe.live%2Fcb', USERNAME, 'bobsecretname', 'carolsecretname', '0123456789abcdefanon', '424242',
'/search', 'statesecret', 'openvibe.live%2Fcb', USERNAME, 'bobsecretname', 'carolsecretname', '0123456789abcdefanon', '424242',
- 'alex%40example.com', '[email protected]', 'Mozilla/5.0', 'curl/8.5.0', 'Bearer', token.slice(0, 20)]) {
+ 'alex%40example.com', '[email protected]', 'Mozilla/5.0', 'curl/8.5.0', 'Bearer', token.slice(0, 20),
+ 'gpcsecretname', 'dntsecretname', '198.51.100.61', '198.51.100.62']) {
assert.ok(!everything.includes(needle), `found ${needle} in the analytics tables`);
assert.ok(!everything.includes(needle), `found ${needle} in the analytics tables`);
}
}
assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);