openvibers/openvibe.shared Commits
analytics: the ADR-021 module moves into Shared (register item 26, C-80); Sec-GPC/DNT requests are not recorded; analytics/event.v1 schema
Changes
@@@ -24,7 +24,7 @@ jobs:git archive --format=tar.gz --prefix=OpenVibe.Shared-ci/ -o "$RUNNER_TEMP/openvibe-shared.tgz" HEADmkdir "$RUNNER_TEMP/consumer" && cd "$RUNNER_TEMP/consumer" && npm init -y >/dev/nullnpm install --no-audit --no-fund "$RUNNER_TEMP/openvibe-shared.tgz"-- node -e "require('openvibe-shared'); require('openvibe-shared/navbar'); require('openvibe-shared/legal'); require('openvibe-shared/metrics'); require('openvibe-shared/ready'); const f = require('openvibe-shared/files'); for (const n of f.BROWSER) require('fs').accessSync(f.path(n)); console.log('ok', require('openvibe-shared/package.json').version)"++ node -e "require('openvibe-shared'); require('openvibe-shared/navbar'); require('openvibe-shared/legal'); require('openvibe-shared/metrics'); require('openvibe-shared/ready'); require('openvibe-shared/analytics'); require('openvibe-shared/analytics/prune-cli'); require('openvibe-shared/analytics/event.v1.json'); const f = require('openvibe-shared/files'); for (const n of f.BROWSER) require('fs').accessSync(f.path(n)); console.log('ok', require('openvibe-shared/package.json').version)"# Secret scan (gitleaks) and dependency audit, the workflow every OpenVibers repo calls (Track Q).security:uses: ./.github/workflows/security.yml@@@ -23,7 +23,8 @@ were reconciled first.| Kind | Files ||---|---|| Browser scripts, served at `/shared/<file>` (listed in `files.js`) | `navbar.js`, `nav-icons.js`, `theme-loader.js`, `footer.js`, `notification-ui.js`, `account-switcher.js`, `user-card.js`, `ov-mark.js`, `ov-icons.js`, `history.js`, `sso-client.js`, `panels.js`, `ui.js`, `island.js`, `tooltip.js`, `openvibe-sw.js` |--| Node modules (`require('openvibe-shared/<name>')`) | `index` (`.`), `analytics`, `app-icon`, `auth-client`, `brand`, `builtin-themes`, `chrome-ssr`, `legal`, `middleware`, `notifications`, `seo`, `theme-sync`, `url-resolver`, `files`, `release`, `metrics`, `ready`; `footer` and `icons` (= `ov-icons.js`) work on both sides |++| Node modules (`require('openvibe-shared/<name>')`) | `index` (`.`), `analytics` (+ `analytics/{privacy,tracker,retention,schema,event,prune-cli}`), `app-icon`, `auth-client`, `brand`, `builtin-themes`, `chrome-ssr`, `legal`, `middleware`, `notifications`, `seo`, `theme-sync`, `url-resolver`, `files`, `release`, `metrics`, `ready`; `footer` and `icons` (= `ov-icons.js`) work on both sides |++| Schemas | `docs/schemas/analytics-event.v1.json` (`analytics/event.v1`, exported as `openvibe-shared/analytics/event.v1.json`) || Generators | `scripts/build-nav-icons.py` (Font Awesome glyphs → `nav-icons.js`, `ov-icons.js`), `scripts/build-navbar-icons.js` (navbar.js's built-in glyphs), `scripts/build-theme-loader.js`, `scripts/build-app-icons.js` |Every module file has an `exports` entry, so `require('openvibe-shared/navbar')`,@@@ -72,6 +73,29 @@ A check fails when it throws, times out, returns `false`, a string (the reason)`{ ok: false, error }`. Name a dependency required only when the service really cannot servewithout it.++### Analytics (server, ADR-021)++++```js++const Database = require('better-sqlite3'); // the service's own; Shared has no native dependency++const { AnalyticsTracker } = require('openvibe-shared/analytics');++const db = new Database('data/analytics.db');++db.pragma('journal_mode = WAL');++const analytics = new AnalyticsTracker(db, 'live', {++ retention: { days: 30 }, // nightly prune (the default); false if you schedule it yourself++ paramPrefixes: ['avatar'], // extra words whose next segment is a parameter++ pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']], // parameters the segment rules can't see++});++app.use(analytics.middleware());++```++++A raw event has no IP, user or subject id, or city. It has a route template (never a raw URL), a++referer origin, a user-agent class, a rotating session id and a country. Raw events are kept for++30 days, and the rollups for longer. A request with `Sec-GPC: 1` or `DNT: 1` is not recorded at++all. The row shape is `analytics/event.v1` (`docs/schemas/analytics-event.v1.json`), and++`require('openvibe-shared/analytics/event').checkRow(row)` checks a stored row against it. The++prune and one-time scrub command is `openvibe-shared/analytics/prune-cli`: a service wraps it in++its own `scripts/analytics-prune.js` and passes in its `Database` and default paths.++### Browser side: two options1. **From the Network** (what most pages do today):@@@ -131,7 +155,12 @@ Tests are plain Node with stubbed browser globals. Some tests guard the release:- **Generated blocks**: the generated navbar and theme-loader blocks must be up to date.- **Browser bundles**: browser files must be free of `require()` and Node globals(`test/browser-bundles.test.js`).--- **Exports**: every export must resolve.++- **Exports**: every export must resolve, and every module file (including `analytics/*.js`) must++ have one.++- **Analytics (ADR-021)**: no personal data in any table after real requests, opt-out requests++ leave nothing, every row is a valid `analytics/event.v1`, prune/scrub keep rollups intact, and++ the prune CLI's dry run changes nothing (`test/analytics-*.test.js`, with `better-sqlite3` as a++ dev dependency).CI runs on Node 22.22.1 and installs a tag-style tarball into an empty project.ddeleted file mode 100644@@@ -1,1066 +0,0 @@--'use strict';----// ═══════════════════════════════════════════════════════════════--// OpenVibe — Unified Analytics Module--// Shared analytics tracking for all OpenVibe services.--// Each service stores events in its own SQLite DB using the--// same schema. The admin panel aggregates via internal API.--// ═══════════════════════════════════════════════════════════════----// ── Bot Detection ────────────────────────────────────────────----const BOT_USER_AGENTS = [-- /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,-- /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,-- /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,-- /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,-- /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,-- /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,-- /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,-- /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,-- /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,-- /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,-- /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,-- /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,-- /statuscake/i, /monitoring/i, /health.?check/i,--];----const SUSPICIOUS_PATTERNS = {-- // Hits per minute thresholds-- highRequestRate: 60,-- // Pages per session in under a minute-- rapidPageViews: 20,-- // No JS execution (missing beacon)-- noJsExecution: true,-- // Accessing well-known bot traps-- honeypotPaths: ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'],--};----/**-- * Classify whether a request is from a bot.-- * Returns: { isBot: boolean, botType: string|null, confidence: number }-- */--function classifyRequest(req) {-- const ua = req.headers['user-agent'] || '';-- const ip = req.ip || req.connection?.remoteAddress || '';---- // 1. Known bot user agents-- for (const pattern of BOT_USER_AGENTS) {-- if (pattern.test(ua)) {-- return { isBot: true, botType: 'known_crawler', confidence: 0.95 };-- }-- }---- // 2. Missing or empty user agent-- if (!ua || ua.length < 10) {-- return { isBot: true, botType: 'no_useragent', confidence: 0.85 };-- }---- // 3. Honeypot paths-- const path = req.path || req.url || '';-- if (SUSPICIOUS_PATTERNS.honeypotPaths.some(p => path.toLowerCase().startsWith(p))) {-- return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };-- }---- // 4. Looks human (default)-- return { isBot: false, botType: null, confidence: 0.1 };--}------// ── Schema ───────────────────────────────────────────────────----const ANALYTICS_SCHEMA = `-- -- Page views / API hits-- CREATE TABLE IF NOT EXISTS analytics_events (-- id INTEGER PRIMARY KEY AUTOINCREMENT,-- service TEXT NOT NULL,-- event_type TEXT NOT NULL DEFAULT 'pageview',-- path TEXT,-- method TEXT DEFAULT 'GET',-- status_code INTEGER,-- response_time_ms INTEGER,-- user_id INTEGER,-- session_id TEXT,-- ip TEXT,-- country TEXT,-- city TEXT,-- user_agent TEXT,-- referer TEXT,-- is_bot INTEGER DEFAULT 0,-- bot_type TEXT,-- device_type TEXT,-- browser TEXT,-- os TEXT,-- created_at DATETIME DEFAULT CURRENT_TIMESTAMP-- );---- -- Aggregated hourly stats (materialized by cron/timer)-- CREATE TABLE IF NOT EXISTS analytics_hourly (-- id INTEGER PRIMARY KEY AUTOINCREMENT,-- service TEXT NOT NULL,-- hour TEXT NOT NULL,-- pageviews INTEGER DEFAULT 0,-- api_calls INTEGER DEFAULT 0,-- unique_visitors INTEGER DEFAULT 0,-- unique_users INTEGER DEFAULT 0,-- bot_hits INTEGER DEFAULT 0,-- avg_response_ms INTEGER DEFAULT 0,-- error_count INTEGER DEFAULT 0,-- bandwidth_bytes INTEGER DEFAULT 0,-- top_paths TEXT,-- top_referers TEXT,-- UNIQUE(service, hour)-- );---- -- Daily aggregates-- CREATE TABLE IF NOT EXISTS analytics_daily (-- id INTEGER PRIMARY KEY AUTOINCREMENT,-- service TEXT NOT NULL,-- date TEXT NOT NULL,-- pageviews INTEGER DEFAULT 0,-- api_calls INTEGER DEFAULT 0,-- unique_visitors INTEGER DEFAULT 0,-- unique_users INTEGER DEFAULT 0,-- new_users INTEGER DEFAULT 0,-- bot_hits INTEGER DEFAULT 0,-- avg_response_ms INTEGER DEFAULT 0,-- error_count INTEGER DEFAULT 0,-- top_paths TEXT,-- top_referers TEXT,-- top_countries TEXT,-- device_breakdown TEXT,-- browser_breakdown TEXT,-- UNIQUE(service, date)-- );---- -- Rate tracking for bot detection-- CREATE TABLE IF NOT EXISTS analytics_rate_tracking (-- ip TEXT NOT NULL,-- window_start INTEGER NOT NULL,-- hit_count INTEGER DEFAULT 1,-- PRIMARY KEY (ip, window_start)-- );---- -- Indexes-- CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_ip ON analytics_events(ip, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_events_user ON analytics_events(user_id, created_at);-- CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);-- CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);--`;------// ── Device/Browser Parsing (lightweight) ─────────────────────----function parseUserAgent(ua) {-- if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };---- // Device-- let device = 'desktop';-- if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';-- else if (/ipad|tablet/i.test(ua)) device = 'tablet';---- // Browser-- let browser = 'other';-- if (/edg\//i.test(ua)) browser = 'edge';-- else if (/opr\/|opera/i.test(ua)) browser = 'opera';-- else if (/firefox\//i.test(ua)) browser = 'firefox';-- else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';-- else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';-- else if (/trident|msie/i.test(ua)) browser = 'ie';---- // OS-- let os = 'other';-- if (/windows/i.test(ua)) os = 'windows';-- else if (/macintosh|mac os/i.test(ua)) os = 'macos';-- else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';-- else if (/android/i.test(ua)) os = 'android';-- else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';---- return { device, browser, os };--}------// ── Analytics Tracker Class ──────────────────────────────────----class AnalyticsTracker {-- /**-- * @param {object} db - better-sqlite3 database instance-- * @param {string} service - service name (e.g., 'live', 'openvibe-network', 'tools')-- */-- constructor(db, service) {-- this.db = db;-- this.service = service;-- this._buffer = [];-- this._flushInterval = null;-- this._aggregateInterval = null;---- // Initialize schema-- this.db.exec(ANALYTICS_SCHEMA);---- // Prepared statements-- this._insertEvent = this.db.prepare(`-- INSERT INTO analytics_events-- (service, event_type, path, method, status_code, response_time_ms,-- user_id, session_id, ip, country, city, user_agent, referer,-- is_bot, bot_type, device_type, browser, os)-- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)-- `);---- this._insertMany = this.db.transaction((events) => {-- for (const e of events) {-- this._insertEvent.run(-- e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,-- e.user_id, e.session_id, e.ip, e.country, e.city, e.user_agent, e.referer,-- e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os-- );-- }-- });---- this._updateRate = this.db.prepare(`-- INSERT INTO analytics_rate_tracking (ip, window_start, hit_count)-- VALUES (?, ?, 1)-- ON CONFLICT(ip, window_start) DO UPDATE SET hit_count = hit_count + 1-- `);---- this._getRate = this.db.prepare(-- 'SELECT SUM(hit_count) as total FROM analytics_rate_tracking WHERE ip = ? AND window_start >= ?'-- );---- this._cleanRates = this.db.prepare(-- 'DELETE FROM analytics_rate_tracking WHERE window_start < ?'-- );---- // Start flush timer (write buffered events every 5s)-- this._flushInterval = setInterval(() => this.flush(), 5000);---- // Hourly aggregation timer-- this._aggregateInterval = setInterval(() => this.aggregate(), 60 * 60 * 1000);---- // Initial aggregation on startup (catch up)-- setTimeout(() => this.aggregate(), 10_000);---- // Clean old rate tracking every 10 min-- setInterval(() => {-- const cutoff = Math.floor(Date.now() / 1000) - 300;-- try { this._cleanRates.run(cutoff); } catch {}-- }, 10 * 60 * 1000);-- }---- /**-- * Express middleware for automatic request tracking.-- * Tracks page views and API calls with response times.-- */-- middleware() {-- return (req, res, next) => {-- // Skip health checks and static assets-- const path = req.path || '';-- if (path === '/api/health' || path === '/health' ||-- /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path)) {-- return next();-- }---- const startTime = Date.now();---- // Hook into response finish-- const originalEnd = res.end;-- res.end = (...args) => {-- res.end = originalEnd;-- res.end(...args);---- const responseTime = Date.now() - startTime;-- const botCheck = classifyRequest(req);-- const parsed = parseUserAgent(req.headers['user-agent'] || '');---- const isApi = path.startsWith('/api/') || path.startsWith('/internal/') || path.startsWith('/oauth/');-- const eventType = isApi ? 'api_call' : 'pageview';---- // Rate-based bot detection-- const ip = req.ip || '';-- const windowStart = Math.floor(Date.now() / 60000); // 1-minute windows-- try {-- this._updateRate.run(ip, windowStart);-- if (!botCheck.isBot) {-- const rateData = this._getRate.get(ip, windowStart - 1);-- if (rateData && rateData.total > SUSPICIOUS_PATTERNS.highRequestRate) {-- botCheck.isBot = true;-- botCheck.botType = 'rate_limit';-- botCheck.confidence = 0.80;-- }-- }-- } catch {}---- // Get session ID from cookie or generate from IP+UA-- const sessionId = req.cookies?.openvibe_session ||-- req.cookies?.ov_token?.slice(-16) ||-- null;---- this._buffer.push({-- service: this.service,-- event_type: eventType,-- path: path.substring(0, 500),-- method: req.method,-- status_code: res.statusCode,-- response_time_ms: responseTime,-- user_id: req.user?.id || req.user?.sub || null,-- session_id: sessionId,-- ip: ip,-- country: req.headers['cf-ipcountry'] || null,-- city: null,-- user_agent: (req.headers['user-agent'] || '').substring(0, 500),-- referer: (req.headers['referer'] || '').substring(0, 500),-- is_bot: botCheck.isBot,-- bot_type: botCheck.botType,-- device_type: parsed.device,-- browser: parsed.browser,-- os: parsed.os,-- });---- // Auto-flush if buffer gets large-- if (this._buffer.length >= 100) {-- this.flush();-- }-- };---- next();-- };-- }---- /**-- * Track a custom event (not from HTTP request).-- */-- trackEvent(eventType, data = {}) {-- this._buffer.push({-- service: this.service,-- event_type: eventType,-- path: data.path || null,-- method: data.method || null,-- status_code: data.status_code || null,-- response_time_ms: data.response_time_ms || null,-- user_id: data.user_id || null,-- session_id: data.session_id || null,-- ip: data.ip || null,-- country: data.country || null,-- city: data.city || null,-- user_agent: data.user_agent || null,-- referer: data.referer || null,-- is_bot: data.is_bot || false,-- bot_type: data.bot_type || null,-- device_type: data.device_type || null,-- browser: data.browser || null,-- os: data.os || null,-- });-- }---- /**-- * Flush buffered events to database.-- */-- flush() {-- if (this._buffer.length === 0) return;-- const events = this._buffer.splice(0);-- try {-- this._insertMany(events);-- } catch (err) {-- console.error(`[Analytics:${this.service}] Flush error:`, err.message);-- }-- }---- /**-- * Run hourly + daily aggregation.-- */-- aggregate() {-- try {-- this._aggregateHourly();-- this._aggregateDaily();-- // Prune raw events older than 90 days-- this.db.prepare("DELETE FROM analytics_events WHERE created_at < datetime('now', '-90 days')").run();-- } catch (err) {-- console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);-- }-- }---- _aggregateHourly() {-- // Aggregate last 2 hours (to catch stragglers)-- const hours = [];-- const now = new Date();-- for (let i = 0; i < 2; i++) {-- const d = new Date(now.getTime() - i * 3600000);-- hours.push(d.toISOString().slice(0, 13) + ':00:00');-- }---- const upsert = this.db.prepare(`-- INSERT OR REPLACE INTO analytics_hourly-- (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)-- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)-- `);---- for (const hour of hours) {-- const nextHour = new Date(new Date(hour).getTime() + 3600000).toISOString().slice(0, 19).replace('T', ' ');-- const hourStart = hour.replace('T', ' ');---- const stats = this.db.prepare(`-- SELECT-- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,-- COUNT(DISTINCT ip) AS unique_visitors,-- COUNT(DISTINCT user_id) AS unique_users,-- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,-- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at < ?-- `).get(this.service, hourStart, nextHour);---- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) continue;---- const topPaths = this.db.prepare(`-- SELECT path, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0-- GROUP BY path ORDER BY cnt DESC LIMIT 10-- `).all(this.service, hourStart, nextHour);---- const topReferers = this.db.prepare(`-- SELECT referer, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0-- GROUP BY referer ORDER BY cnt DESC LIMIT 10-- `).all(this.service, hourStart, nextHour);---- upsert.run(-- this.service, hourStart,-- stats.pageviews, stats.api_calls, stats.unique_visitors, stats.unique_users,-- stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,-- JSON.stringify(topPaths), JSON.stringify(topReferers)-- );-- }-- }---- _aggregateDaily() {-- // Aggregate last 2 days-- const days = [];-- const now = new Date();-- for (let i = 0; i < 2; i++) {-- const d = new Date(now.getTime() - i * 86400000);-- days.push(d.toISOString().slice(0, 10));-- }---- const upsert = this.db.prepare(`-- INSERT OR REPLACE INTO analytics_daily-- (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,-- avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)-- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)-- `);---- for (const date of days) {-- const dayStart = date + ' 00:00:00';-- const dayEnd = date + ' 23:59:59';---- const stats = this.db.prepare(`-- SELECT-- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,-- COUNT(DISTINCT ip) AS unique_visitors,-- COUNT(DISTINCT user_id) AS unique_users,-- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,-- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ?-- `).get(this.service, dayStart, dayEnd);---- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) continue;---- const topPaths = this.db.prepare(`-- SELECT path, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0-- GROUP BY path ORDER BY cnt DESC LIMIT 15-- `).all(this.service, dayStart, dayEnd);---- const topReferers = this.db.prepare(`-- SELECT referer, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0-- GROUP BY referer ORDER BY cnt DESC LIMIT 10-- `).all(this.service, dayStart, dayEnd);---- const topCountries = this.db.prepare(`-- SELECT country, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0-- GROUP BY country ORDER BY cnt DESC LIMIT 10-- `).all(this.service, dayStart, dayEnd);---- const devices = this.db.prepare(`-- SELECT device_type, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0-- GROUP BY device_type ORDER BY cnt DESC-- `).all(this.service, dayStart, dayEnd);---- const browsers = this.db.prepare(`-- SELECT browser, COUNT(*) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0-- GROUP BY browser ORDER BY cnt DESC-- `).all(this.service, dayStart, dayEnd);---- // New users (first seen this day)-- const newUsers = this.db.prepare(`-- SELECT COUNT(DISTINCT user_id) as cnt FROM analytics_events-- WHERE service = ? AND created_at >= ? AND created_at <= ? AND user_id IS NOT NULL-- AND user_id NOT IN (-- SELECT DISTINCT user_id FROM analytics_events-- WHERE service = ? AND created_at < ? AND user_id IS NOT NULL-- )-- `).get(this.service, dayStart, dayEnd, this.service, dayStart);---- upsert.run(-- this.service, date, stats.pageviews, stats.api_calls,-- stats.unique_visitors, stats.unique_users,-- newUsers?.cnt || 0, stats.bot_hits,-- stats.avg_response_ms || 0, stats.error_count,-- JSON.stringify(topPaths), JSON.stringify(topReferers),-- JSON.stringify(topCountries), JSON.stringify(devices),-- JSON.stringify(browsers)-- );-- }-- }---- /**-- * Get analytics data for the admin dashboard.-- * @param {object} options-- * @param {number} options.days - Number of days to look back (default 30)-- * @param {number} options.hours - If set, overrides days with an hour-based range (e.g. 1, 12, 24, 72)-- * @param {string} options.service - Service name override-- */-- getStats(options = {}) {-- const { days = 30, hours, service } = options;-- const svc = service || this.service;---- // Compute cutoff timestamp for raw event queries-- const rangeMs = hours ? hours * 3600000 : days * 86400000;-- const rangeCutoff = new Date(Date.now() - rangeMs).toISOString().slice(0, 19).replace('T', ' ');-- const isSubDay = hours && hours < 24;-- const effectiveDays = hours ? hours / 24 : days;---- // Daily trend (use daily aggregates for multi-day, raw events for sub-day)-- let daily = [];-- if (!isSubDay) {-- const dateCutoff = new Date(Date.now() - rangeMs).toISOString().slice(0, 10);-- daily = this.db.prepare(`-- SELECT * FROM analytics_daily-- WHERE service = ? AND date >= ?-- ORDER BY date ASC-- `).all(svc, dateCutoff);-- }---- // Hourly trend (always useful — scope to the requested range)-- const hourlyLimit = hours ? Math.min(hours, 168) : 24;-- const hourlyCutoff = new Date(Date.now() - hourlyLimit * 3600000).toISOString().slice(0, 19).replace('T', ' ');-- const hourly = this.db.prepare(`-- SELECT * FROM analytics_hourly-- WHERE service = ? AND hour >= ?-- ORDER BY hour ASC-- `).all(svc, hourlyCutoff);---- // For sub-day ranges, generate fine-grained time buckets from raw events-- let timeBuckets = [];-- if (isSubDay) {-- // 5-minute buckets for ≤1h, 15-min for ≤12h, 30-min for ≤24h-- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;-- timeBuckets = this.db.prepare(`-- SELECT-- strftime('%Y-%m-%d %H:', created_at) ||-- CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT) as bucket,-- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,-- COUNT(DISTINCT ip) FILTER (WHERE is_bot = 0) AS unique_visitors,-- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,-- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- GROUP BY bucket-- ORDER BY bucket ASC-- `).all(svc, rangeCutoff);-- }---- // Summary totals — for sub-day ranges, compute from raw events-- let summary;-- if (hours && hours < 48) {-- summary = this.db.prepare(`-- SELECT-- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,-- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,-- COUNT(DISTINCT ip) FILTER (WHERE is_bot = 0) AS total_unique_visitors,-- COUNT(DISTINCT user_id) FILTER (WHERE is_bot = 0 AND user_id IS NOT NULL) AS total_unique_users,-- COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,-- COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- `).get(svc, rangeCutoff);-- } else {-- const dateCutoff = new Date(Date.now() - rangeMs).toISOString().slice(0, 10);-- summary = this.db.prepare(`-- SELECT-- SUM(pageviews) as total_pageviews,-- SUM(api_calls) as total_api_calls,-- SUM(unique_visitors) as total_unique_visitors,-- SUM(unique_users) as total_unique_users,-- SUM(bot_hits) as total_bot_hits,-- SUM(error_count) as total_errors,-- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms-- FROM analytics_daily-- WHERE service = ? AND date >= ?-- `).get(svc, dateCutoff);-- }---- // Real-time (last 5 min)-- const fiveMinAgo = new Date(Date.now() - 300000).toISOString().slice(0, 19).replace('T', ' ');-- const realtime = this.db.prepare(`-- SELECT-- COUNT(*) as requests,-- COUNT(DISTINCT ip) as visitors,-- COUNT(*) FILTER (WHERE is_bot = 1) as bots-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- `).get(svc, fiveMinAgo);---- // Top pages (human only)-- const topPages = this.db.prepare(`-- SELECT path, COUNT(*) as hits, COUNT(DISTINCT ip) as visitors-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY path ORDER BY hits DESC LIMIT 20-- `).all(svc, rangeCutoff);---- // Top referers-- const topReferers = this.db.prepare(`-- SELECT referer, COUNT(*) as hits-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- AND referer IS NOT NULL AND referer != '' AND is_bot = 0-- GROUP BY referer ORDER BY hits DESC LIMIT 15-- `).all(svc, rangeCutoff);---- // Bot breakdown-- const botBreakdown = this.db.prepare(`-- SELECT bot_type, COUNT(*) as hits-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 1-- GROUP BY bot_type ORDER BY hits DESC LIMIT 15-- `).all(svc, rangeCutoff);---- // Status code distribution-- const statusCodes = this.db.prepare(`-- SELECT-- CASE-- WHEN status_code >= 200 AND status_code < 300 THEN '2xx'-- WHEN status_code >= 300 AND status_code < 400 THEN '3xx'-- WHEN status_code >= 400 AND status_code < 500 THEN '4xx'-- WHEN status_code >= 500 THEN '5xx'-- ELSE 'other'-- END as group_code,-- COUNT(*) as cnt-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- GROUP BY group_code ORDER BY cnt DESC-- `).all(svc, rangeCutoff);---- // Device/browser/OS breakdown-- const deviceBreakdown = this.db.prepare(`-- SELECT device_type, COUNT(*) as cnt-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY device_type ORDER BY cnt DESC-- `).all(svc, rangeCutoff);---- const browserBreakdown = this.db.prepare(`-- SELECT browser, COUNT(*) as cnt-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY browser ORDER BY cnt DESC-- `).all(svc, rangeCutoff);---- const osBreakdown = this.db.prepare(`-- SELECT os, COUNT(*) as cnt-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY os ORDER BY cnt DESC-- `).all(svc, rangeCutoff);---- // Country breakdown-- const countryBreakdown = this.db.prepare(`-- SELECT country, COUNT(*) as cnt-- FROM analytics_events-- WHERE service = ? AND created_at >= ?-- AND country IS NOT NULL AND is_bot = 0-- GROUP BY country ORDER BY cnt DESC LIMIT 20-- `).all(svc, rangeCutoff);---- // ── Enhanced Metrics ─────────────────────────────────---- // Response time percentiles (p50, p90, p95, p99)-- let responsePercentiles = {};-- try {-- const rtRows = this.db.prepare(`-- SELECT response_time_ms FROM analytics_events-- WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0-- ORDER BY response_time_ms ASC-- `).all(svc, rangeCutoff);-- if (rtRows.length > 0) {-- const vals = rtRows.map(r => r.response_time_ms);-- const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];-- responsePercentiles = { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };-- }-- } catch {}---- // Bandwidth estimation (response_time_ms * rough factor isn't great,-- // but we can count requests and estimate from content-length if tracked;-- // for now, count total requests * avg page weight heuristic)-- let bandwidth = {};-- try {-- const bwData = this.db.prepare(`-- SELECT-- COUNT(*) as total_requests,-- COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,-- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,-- SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- `).get(svc, rangeCutoff);-- if (bwData) {-- bandwidth = {-- total_requests: bwData.total_requests || 0,-- page_requests: bwData.page_requests || 0,-- api_requests: bwData.api_requests || 0,-- estimated_bytes: bwData.estimated_bytes || 0,-- };-- }-- } catch {}---- // Authenticated vs anonymous sessions-- let authBreakdown = {};-- try {-- authBreakdown = this.db.prepare(`-- SELECT-- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS authenticated,-- COUNT(*) FILTER (WHERE user_id IS NULL) AS anonymous,-- COUNT(DISTINCT user_id) FILTER (WHERE user_id IS NOT NULL) AS unique_authenticated_users,-- COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- `).get(svc, rangeCutoff) || {};-- } catch {}---- // Error rate over time (hourly for multi-day, bucketed for sub-day)-- let errorTrend = [];-- try {-- if (isSubDay) {-- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;-- errorTrend = this.db.prepare(`-- SELECT-- strftime('%Y-%m-%d %H:', created_at) ||-- CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT) as bucket,-- COUNT(*) AS total,-- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,-- COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY bucket ORDER BY bucket ASC-- `).all(svc, rangeCutoff);-- } else {-- errorTrend = this.db.prepare(`-- SELECT hour as bucket,-- (pageviews + api_calls) as total,-- error_count as errors-- FROM analytics_hourly-- WHERE service = ? AND hour >= ?-- ORDER BY hour ASC-- `).all(svc, hourlyCutoff);-- }-- } catch {}---- // Peak hours (which hours of day get most traffic)-- let peakHours = [];-- try {-- peakHours = this.db.prepare(`-- SELECT-- CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day,-- COUNT(*) AS hits,-- COUNT(DISTINCT ip) AS visitors,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY hour_of_day-- ORDER BY hour_of_day ASC-- `).all(svc, rangeCutoff);-- } catch {}---- // Top API endpoints (separate from top pages)-- let topApiEndpoints = [];-- try {-- topApiEndpoints = this.db.prepare(`-- SELECT path, method, COUNT(*) as hits,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,-- COUNT(*) FILTER (WHERE status_code >= 400) AS errors-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0-- GROUP BY path, method ORDER BY hits DESC LIMIT 20-- `).all(svc, rangeCutoff);-- } catch {}---- // Unique sessions count-- let sessionCount = 0;-- try {-- const sc = this.db.prepare(`-- SELECT COUNT(DISTINCT session_id) as cnt-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0-- `).get(svc, rangeCutoff);-- sessionCount = sc?.cnt || 0;-- } catch {}---- // New vs returning visitors (first-time IPs in this period vs seen before)-- let visitorTypes = {};-- try {-- visitorTypes = this.db.prepare(`-- SELECT-- COUNT(DISTINCT ip) FILTER (-- WHERE ip NOT IN (-- SELECT DISTINCT ip FROM analytics_events-- WHERE service = ? AND created_at < ? AND is_bot = 0-- )-- ) AS new_visitors,-- COUNT(DISTINCT ip) FILTER (-- WHERE ip IN (-- SELECT DISTINCT ip FROM analytics_events-- WHERE service = ? AND created_at < ? AND is_bot = 0-- )-- ) AS returning_visitors-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- `).get(svc, rangeCutoff, svc, rangeCutoff, svc, rangeCutoff) || {};-- } catch {}---- // Slowest endpoints-- let slowestEndpoints = [];-- try {-- slowestEndpoints = this.db.prepare(`-- SELECT path, method,-- COUNT(*) as hits,-- CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,-- MAX(response_time_ms) AS max_ms-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0-- GROUP BY path, method-- HAVING hits >= 3-- ORDER BY avg_ms DESC LIMIT 10-- `).all(svc, rangeCutoff);-- } catch {}---- // Authenticated vs anonymous time trend-- let authTrend = [];-- try {-- if (isSubDay) {-- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;-- authTrend = this.db.prepare(`-- SELECT-- strftime('%Y-%m-%d %H:', created_at) ||-- CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT) as bucket,-- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NOT NULL) AS auth_visitors,-- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NULL) AS anon_visitors,-- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS auth_hits,-- COUNT(*) FILTER (WHERE user_id IS NULL) AS anon_hits-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY bucket ORDER BY bucket ASC-- `).all(svc, rangeCutoff);-- } else {-- authTrend = this.db.prepare(`-- SELECT-- strftime('%Y-%m-%d', created_at) AS bucket,-- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NOT NULL) AS auth_visitors,-- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NULL) AS anon_visitors,-- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS auth_hits,-- COUNT(*) FILTER (WHERE user_id IS NULL) AS anon_hits-- FROM analytics_events-- WHERE service = ? AND created_at >= ? AND is_bot = 0-- GROUP BY bucket ORDER BY bucket ASC-- `).all(svc, rangeCutoff);-- }-- } catch {}---- return {-- service: svc,-- period_days: effectiveDays,-- period_hours: hours || null,-- summary: summary || {},-- realtime: realtime || {},-- daily,-- hourly,-- timeBuckets,-- topPages,-- topReferers,-- botBreakdown,-- statusCodes,-- deviceBreakdown,-- browserBreakdown,-- osBreakdown,-- countryBreakdown,-- // Enhanced metrics-- responsePercentiles,-- bandwidth,-- authBreakdown,-- errorTrend,-- peakHours,-- topApiEndpoints,-- sessionCount,-- visitorTypes,-- slowestEndpoints,-- authTrend,-- };-- }---- /**-- * Get a cross-service overview (for the admin panel).-- */-- getOverview(days = 30) {-- const cutoff = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);---- const services = this.db.prepare(`-- SELECT-- service,-- SUM(pageviews) as pageviews,-- SUM(api_calls) as api_calls,-- SUM(unique_visitors) as unique_visitors,-- SUM(unique_users) as unique_users,-- SUM(bot_hits) as bot_hits,-- SUM(error_count) as errors,-- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms-- FROM analytics_daily-- WHERE date >= ?-- GROUP BY service-- ORDER BY pageviews DESC-- `).all(cutoff);---- // Totals across all services-- const totals = this.db.prepare(`-- SELECT-- SUM(pageviews) as pageviews,-- SUM(api_calls) as api_calls,-- SUM(unique_visitors) as unique_visitors,-- SUM(bot_hits) as bot_hits,-- SUM(error_count) as errors-- FROM analytics_daily-- WHERE date >= ?-- `).get(cutoff);---- // Daily trend (all services combined)-- const dailyTrend = this.db.prepare(`-- SELECT date,-- SUM(pageviews) as pageviews,-- SUM(api_calls) as api_calls,-- SUM(unique_visitors) as unique_visitors,-- SUM(bot_hits) as bot_hits-- FROM analytics_daily-- WHERE date >= ?-- GROUP BY date-- ORDER BY date ASC-- `).all(cutoff);---- // Real-time all services-- const fiveMinAgo = new Date(Date.now() - 300000).toISOString().slice(0, 19).replace('T', ' ');-- const realtime = this.db.prepare(`-- SELECT-- service,-- COUNT(*) as requests,-- COUNT(DISTINCT ip) as visitors,-- COUNT(*) FILTER (WHERE is_bot = 1) as bots-- FROM analytics_events-- WHERE created_at >= ?-- GROUP BY service-- `).all(fiveMinAgo);---- return {-- period_days: days,-- totals: totals || {},-- services,-- dailyTrend,-- realtime,-- };-- }---- /**-- * Get bot analysis data.-- */-- getBotAnalysis(days = 30) {-- const daysCast = Math.min(Math.max(1, days), 365);---- // Top bot IPs-- const topBotIPs = this.db.prepare(`-- SELECT ip, COUNT(*) as hits, bot_type, user_agent,-- MIN(created_at) as first_seen, MAX(created_at) as last_seen-- FROM analytics_events-- WHERE is_bot = 1 AND created_at >= datetime('now', '-' || ? || ' days')-- GROUP BY ip ORDER BY hits DESC LIMIT 25-- `).all(daysCast);---- // Bot vs human ratio over time-- const botTrend = this.db.prepare(`-- SELECT date,-- SUM(pageviews) + SUM(api_calls) as human_hits,-- SUM(bot_hits) as bot_hits-- FROM analytics_daily-- WHERE date >= date('now', '-' || ? || ' days')-- GROUP BY date ORDER BY date ASC-- `).all(daysCast);---- // Bot type breakdown-- const botTypes = this.db.prepare(`-- SELECT bot_type, COUNT(*) as hits, COUNT(DISTINCT ip) as unique_ips-- FROM analytics_events-- WHERE is_bot = 1 AND created_at >= datetime('now', '-' || ? || ' days')-- GROUP BY bot_type ORDER BY hits DESC-- `).all(daysCast);---- // Suspicious behavior patterns-- const suspiciousIPs = this.db.prepare(`-- SELECT ip, COUNT(*) as total_hits,-- COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,-- COUNT(DISTINCT path) as unique_paths,-- MIN(created_at) as first_seen,-- MAX(created_at) as last_seen-- FROM analytics_events-- WHERE created_at >= datetime('now', '-7 days') AND is_bot = 0-- GROUP BY ip-- HAVING total_hits > 500-- ORDER BY total_hits DESC LIMIT 20-- `).all();---- return {-- topBotIPs,-- botTrend,-- botTypes,-- suspiciousIPs,-- };-- }---- /**-- * Destroy tracker (clean up timers).-- */-- destroy() {-- this.flush();-- if (this._flushInterval) clearInterval(this._flushInterval);-- if (this._aggregateInterval) clearInterval(this._aggregateInterval);-- }--}------module.exports = {-- AnalyticsTracker,-- classifyRequest,-- parseUserAgent,-- ANALYTICS_SCHEMA,-- BOT_USER_AGENTS,-- SUSPICIOUS_PATTERNS,--};nnew file mode 100644@@@ -0,0 +1,130 @@++'use strict';++/**++ * analytics/event.v1: the raw analytics event shape (ADR-021). The JSON schema is++ * docs/schemas/analytics-event.v1.json, also exported as 'openvibe-shared/analytics/event.v1.json'.++ *++ * EVENT_V1 the schema object++ * toEvent(row) an analytics_events row -> an event.v1 object (column renames, 0/1 -> boolean,++ * created_at -> RFC 3339). The retired columns ip/user_id/city are not part++ * of the event; checkRow() reports them.++ * validateEvent(ev) [] when ev conforms to event.v1, else a list of 'field: problem' strings++ * checkRow(row) validateEvent(toEvent(row)) plus any non-NULL ip / user_id / city++ *++ * The validator interprets the schema file itself, for the keywords it uses (type, enum, pattern,++ * maxLength, minimum, maximum, required, properties, additionalProperties: false), and refuses a++ * schema with any other keyword, so the file and this code cannot drift apart. No dependencies.++ */++++const EVENT_V1 = require('../docs/schemas/analytics-event.v1.json');++++const RETIRED_COLUMNS = ['ip', 'user_id', 'city'];++const ANNOTATIONS = new Set(['$schema', '$id', 'title', 'description']);++const OBJECT_KEYWORDS = new Set(['type', 'required', 'additionalProperties', 'properties']);++const VALUE_KEYWORDS = new Set(['type', 'enum', 'pattern', 'maxLength', 'minimum', 'maximum']);++++function checkKeywords(node, allowed, where) {++ for (const k of Object.keys(node)) {++ if (!ANNOTATIONS.has(k) && !allowed.has(k)) throw new Error(`analytics event schema: unsupported keyword ${k} at ${where}`);++ }++}++++/** Compile the schema into per-field checks, once. */++function compile(schema) {++ checkKeywords(schema, OBJECT_KEYWORDS, '/');++ if (schema.type !== 'object' || schema.additionalProperties !== false) throw new Error('analytics event schema: expected a closed object');++ const fields = {};++ for (const [name, spec] of Object.entries(schema.properties)) {++ checkKeywords(spec, VALUE_KEYWORDS, `/properties/${name}`);++ const types = spec.type == null ? null : [].concat(spec.type);++ const re = spec.pattern ? new RegExp(spec.pattern, 'u') : null;++ fields[name] = (v) => {++ if (spec.enum && !spec.enum.some((e) => e === v)) return `not one of ${JSON.stringify(spec.enum)}`;++ if (types && !types.some((t) => typeOk(t, v))) return `expected ${types.join(' or ')}`;++ if (v === null) return null;++ if (typeof v === 'string') {++ if (spec.maxLength != null && [...v].length > spec.maxLength) return `longer than ${spec.maxLength}`;++ if (re && !re.test(v)) return `does not match ${spec.pattern}`;++ }++ if (typeof v === 'number') {++ if (spec.minimum != null && v < spec.minimum) return `below ${spec.minimum}`;++ if (spec.maximum != null && v > spec.maximum) return `above ${spec.maximum}`;++ }++ return null;++ };++ }++ return { fields, required: schema.required || [] };++}++++function typeOk(t, v) {++ switch (t) {++ case 'null': return v === null;++ case 'string': return typeof v === 'string';++ case 'boolean': return typeof v === 'boolean';++ case 'integer': return Number.isInteger(v);++ case 'number': return typeof v === 'number' && Number.isFinite(v);++ default: throw new Error(`analytics event schema: unsupported type ${t}`);++ }++}++++const COMPILED = compile(EVENT_V1);++++/** [] when `ev` is a valid analytics/event.v1 object; otherwise one 'field: problem' string per issue. */++function validateEvent(ev) {++ if (!ev || typeof ev !== 'object' || Array.isArray(ev)) return ['event: expected an object'];++ const errors = [];++ for (const k of COMPILED.required) if (!(k in ev)) errors.push(`${k}: required`);++ for (const [k, v] of Object.entries(ev)) {++ const check = COMPILED.fields[k];++ if (!check) { errors.push(`${k}: not allowed`); continue; }++ const problem = check(v);++ if (problem) errors.push(`${k}: ${problem}`);++ }++ return errors;++}++++const nullable = (v) => (v === undefined || v === '' ? null : v);++const bool = (v) => v === true || v === 1 || v === '1';++++/** 'YYYY-MM-DD HH:MM:SS' (SQLite CURRENT_TIMESTAMP, UTC) -> 'YYYY-MM-DDTHH:MM:SSZ'; ISO input passes. */++function isoTime(v) {++ if (v == null) return v;++ const s = String(v);++ if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(s)) return s.replace(' ', 'T') + 'Z';++ return s;++}++++/** An analytics_events row as an analytics/event.v1 object. */++function toEvent(row) {++ return {++ event: row.event_type,++ service: row.service,++ route: nullable(row.path),++ method: nullable(row.method),++ status: nullable(row.status_code),++ duration_ms: nullable(row.response_time_ms),++ session_id: nullable(row.session_id),++ country: nullable(row.country),++ ua_class: nullable(row.user_agent),++ device: nullable(row.device_type),++ browser: nullable(row.browser),++ os: nullable(row.os),++ referer_origin: nullable(row.referer),++ is_bot: bool(row.is_bot),++ bot_type: nullable(row.bot_type),++ authenticated: bool(row.authenticated),++ timestamp: isoTime(row.created_at),++ };++}++++/** Problems with a stored row: retired personal columns that are set, then the event.v1 checks. */++function checkRow(row) {++ const errors = RETIRED_COLUMNS.filter((c) => row[c] != null).map((c) => `${c}: must be NULL (ADR-021)`);++ return errors.concat(validateEvent(toEvent(row)));++}++++/** The enum of an event.v1 field, without null (for reducers that must stay inside the schema). */++function allowed(field) {++ return EVENT_V1.properties[field].enum.filter((v) => v !== null);++}++++module.exports = { EVENT_V1, RETIRED_COLUMNS, toEvent, validateEvent, checkRow, allowed };nnew file mode 100644@@@ -0,0 +1,49 @@++'use strict';++/**++ * openvibe-shared/analytics: request analytics within ADR-021 (OpenVibe.Contracts++ * docs/adr/ADR-021-analytics.md). A raw event has no IP, user or subject id, or city. Paths are route++ * templates and referers are reduced to origins. A user agent is stored as a class, sessions are++ * rotating ids, and raw events are pruned after 30 days. A request with `Sec-GPC: 1` or `DNT: 1` is++ * not recorded. The event shape is analytics/event.v1 (docs/schemas/analytics-event.v1.json).++ *++ * const { AnalyticsTracker } = require('openvibe-shared/analytics');++ * const tracker = new AnalyticsTracker(db, 'live', { retention: { days: 30 } }); // db: better-sqlite3++ * app.use(tracker.middleware());++ *++ * Parts (also available as subpaths): tracker, privacy, retention, schema, event, prune-cli.++ * No dependencies: the caller passes its own better-sqlite3 handle (and Database, for the CLI).++ */++++const { AnalyticsTracker } = require('./tracker');++const privacy = require('./privacy');++const retention = require('./retention');++const schema = require('./schema');++const event = require('./event');++++/**++ * @deprecated The pre-ADR-021 module's tuning object, kept so the 1.x export list stays complete++ * (register item C-80). It is frozen and read-only, and the tracker does not read it. It will be++ * removed in 2.0.0.++ */++const SUSPICIOUS_PATTERNS = Object.freeze({++ highRequestRate: privacy.HIGH_REQUEST_RATE,++ rapidPageViews: 20,++ noJsExecution: true,++ honeypotPaths: Object.freeze([...privacy.HONEYPOT_PATHS]),++});++++module.exports = {++ AnalyticsTracker,++ privacy,++ retention,++ schema,++ event,++ ensureSchema: schema.ensureSchema,++ optedOut: privacy.optedOut,++ // Names the pre-ADR module exported. They are kept for 1.x and now use the ADR-021 behaviour.++ classifyRequest: privacy.classifyRequest,++ parseUserAgent: privacy.parseUserAgent,++ ANALYTICS_SCHEMA: schema.ANALYTICS_SCHEMA,++ BOT_USER_AGENTS: privacy.BOT_USER_AGENTS,++ SUSPICIOUS_PATTERNS,++};nnew file mode 100644@@@ -0,0 +1,276 @@++'use strict';++/**++ * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):++ * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a++ * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a++ * precise location or a user/subject id.++ *++ * This file holds the pure reducers that turn a request into those fields, and the opt-out check++ * (Sec-GPC / DNT). No dependencies. The raw row shape is analytics/event.v1++ * (docs/schemas/analytics-event.v1.json, ./event.js).++ */++++// ── Opt-out (Global Privacy Control, Do Not Track) ───────────++++/** True when a header value is exactly "1" (either header may arrive repeated as an array). */++function headerIsOne(v) {++ if (Array.isArray(v)) return v.some(headerIsOne);++ return v != null && String(v).trim() === '1';++}++++/**++ * True when the request asks not to be tracked: `Sec-GPC: 1` (Global Privacy Control) or `DNT: 1`.++ * Such a request is not recorded at all — no raw row, no visitor hash, no session, no rate counter —++ * so it is also missing from every rollup (ADR-021 allows no per-request trace of an opted-out++ * visitor, and does not provide for a separate opted-out count).++ */++function optedOut(headers) {++ if (!headers) return false;++ return headerIsOne(headers['sec-gpc']) || headerIsOne(headers.dnt);++}++++// ── Bots ──────────────────────────────────────────────────────++++const BOT_USER_AGENTS = [++ /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,++ /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,++ /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,++ /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,++ /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,++ /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,++ /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,++ /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,++ /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,++ /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,++ /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,++ /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,++ /statuscake/i, /monitoring/i, /health.?check/i,++];++++const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];++++/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */++const HIGH_REQUEST_RATE = 60;++++/** The family name of the first bot pattern the user agent matches, or null. */++function botName(ua) {++ if (!ua) return null;++ for (const re of BOT_USER_AGENTS) {++ if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';++ }++ return null;++}++++/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */++function classifyRequest(req) {++ const ua = (req.headers && req.headers['user-agent']) || '';++ if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };++ if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };++ const path = String(req.path || req.url || '').toLowerCase();++ if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };++ return { isBot: false, botType: null, confidence: 0.1 };++}++++// ── User agent → class ───────────────────────────────────────++++function parseUserAgent(ua) {++ if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };++ let device = 'desktop';++ if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';++ else if (/ipad|tablet/i.test(ua)) device = 'tablet';++++ let browser = 'other';++ if (/edg\//i.test(ua)) browser = 'edge';++ else if (/opr\/|opera/i.test(ua)) browser = 'opera';++ else if (/firefox\//i.test(ua)) browser = 'firefox';++ else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';++ else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';++ else if (/trident|msie/i.test(ua)) browser = 'ie';++++ let os = 'other';++ if (/windows/i.test(ua)) os = 'windows';++ else if (/macintosh|mac os/i.test(ua)) os = 'macos';++ else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';++ else if (/android/i.test(ua)) os = 'android';++ else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';++++ return { device, browser, os };++}++++const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;++++/**++ * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,++ * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps++ * to itself), so a scrub can run over rows that are already reduced.++ */++function uaClass(ua) {++ if (ua == null || ua === '') return 'none';++ const s = String(ua);++ if (UA_CLASS_RE.test(s)) return s;++ const bot = botName(s);++ if (bot) return `bot:${bot}`;++ const p = parseUserAgent(s);++ return `${p.browser}/${p.os}/${p.device}`;++}++++// ── Referer → origin ─────────────────────────────────────────++++/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */++function refererOrigin(ref) {++ if (!ref) return null;++ try {++ const u = new URL(String(ref));++ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;++ return u.origin;++ } catch {++ return null;++ }++}++++// ── Country ──────────────────────────────────────────────────++++/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */++function countryCode(v) {++ const s = String(v || '').trim().toUpperCase();++ return /^[A-Z]{2}$/.test(s) ? s : null;++}++++// ── Paths → route templates ──────────────────────────────────++++/**++ * Words after which the next path segment is a parameter even when it looks like a word++ * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.++ */++const DEFAULT_PARAM_PREFIXES = [++ 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',++ 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',++ 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',++ 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',++ 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',++];++++const MAX_SEGMENTS = 8;++const MAX_TEMPLATE_LENGTH = 200;++const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;++const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;++++/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */++function looksLikeId(s) {++ if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…++ if (/^\d+$/.test(s)) return true; // 123++ if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid++ if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes++ if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…++ if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid++ if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers++ if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names++ if (s.length > 32) return true;++ return false;++}++++/**++ * Path options with a service's extra parameter words merged into the defaults:++ * `{ paramPrefixes: Set, pathRules }`, or undefined when there is nothing to add.++ */++function pathOptions({ paramPrefixes, pathRules } = {}) {++ const extra = paramPrefixes ? [...paramPrefixes] : [];++ const rules = pathRules && pathRules.length ? checkRules(pathRules) : null;++ if (!extra.length && !rules) return undefined;++ const out = { paramPrefixes: new Set([...DEFAULT_PARAM_PREFIXES, ...extra]) };++ if (rules) out.pathRules = rules;++ return out;++}++++function checkRules(rules) {++ for (const r of rules) {++ if (!Array.isArray(r) || !(r[0] instanceof RegExp) || typeof r[1] !== 'string') {++ throw new TypeError('pathRules entries are [RegExp, replacement string] pairs');++ }++ }++ return rules;++}++++/** Apply a service's pathRules ([RegExp, replacement] pairs) to a raw path, in order. */++function applyPathRules(p, rules) {++ let s = String(p);++ if (rules) for (const [re, to] of rules) { re.lastIndex = 0; s = s.replace(re, to); }++ return s;++}++++/**++ * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by++ * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated++ * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.++ *++ * Options (a service's own route shapes):++ * paramPrefixes words whose next segment is a parameter (default DEFAULT_PARAM_PREFIXES; pass the++ * merged set from pathOptions() to add to the defaults rather than replace them)++ * pathRules [RegExp, replacement] pairs applied to the raw path first, for a parameter the++ * segment rules cannot see (e.g. [/\/by-username\/[^/?#]+/gi, '/by-username/:username']).++ * A rule's output must be stable under the same rule, or the result is not idempotent.++ */++function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES, pathRules } = {}) {++ if (raw == null || raw === '') return '/';++ let p = String(raw);++ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {++ try { p = new URL(p).pathname; } catch { return '/'; }++ }++ p = p.split(/[?#]/)[0];++ if (pathRules) p = applyPathRules(p, pathRules);++ const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);++ const segs = p.split('/').filter(Boolean);++ const out = [];++ let expectParam = false;++ for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {++ let s = segs[i];++ try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }++ if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }++ if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }++ if (expectParam) { out.push(':param'); expectParam = false; continue; }++ if (looksLikeId(s)) { out.push(':id'); continue; }++ out.push(s);++ expectParam = prefixes.has(s.toLowerCase());++ }++ if (segs.length > MAX_SEGMENTS) out.push('*');++ // Too long: drop whole segments (never cut one in half) and end with `*`.++ if (('/' + out.join('/')).length > MAX_TEMPLATE_LENGTH) {++ if (out[out.length - 1] === '*') out.pop();++ while (out.length && ('/' + out.join('/')).length > MAX_TEMPLATE_LENGTH - 2) out.pop();++ out.push('*');++ }++ return '/' + out.join('/');++}++++/**++ * The route template for a finished request: the matched Express route (mount path + route path)++ * when there is one, else the normalised original path. Both go through normalisePath, because a++ * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.++ */++function routeTemplate(req, fallbackPath, opts) {++ const r = req && req.route;++ if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {++ return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);++ }++ return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);++}++++module.exports = {++ BOT_USER_AGENTS,++ HONEYPOT_PATHS,++ HIGH_REQUEST_RATE,++ DEFAULT_PARAM_PREFIXES,++ MAX_TEMPLATE_LENGTH,++ optedOut,++ botName,++ classifyRequest,++ parseUserAgent,++ uaClass,++ refererOrigin,++ countryCode,++ looksLikeId,++ pathOptions,++ applyPathRules,++ normalisePath,++ routeTemplate,++};nnew file mode 100644@@@ -0,0 +1,235 @@++'use strict';++/**++ * The analytics-prune command (ADR-021): raw-event retention and the one-time scrub of pre-ADR rows,++ * for one or more analytics databases. A service keeps a thin scripts/analytics-prune.js that passes++ * its better-sqlite3 and its defaults:++ *++ * const Database = require('better-sqlite3');++ * const cli = require('openvibe-shared/analytics/prune-cli');++ * const main = (argv, log) => cli.main(argv, { Database, log, defaultDb: process.env.ANALYTICS_DB || 'data/analytics.db' });++ * if (require.main === module) cli.run(main);++ * module.exports = { main };++ *++ * Options given by the service (second argument of main):++ * Database better-sqlite3's constructor (required; this package has no native dependency)++ * log line logger (default console.log)++ * defaultDb the database when no --db is given: a path, or (args, env) => path++ * targets (args) => [{ name, file }]: the databases when no --db is given (several apps)++ * options extra repeatable value flags, { '--app': 'apps' } -> args.apps = [...]++ * paramPrefixes the service's extra parameter words } as for the tracker, so legacy rows are++ * pathRules the service's [RegExp, replacement] } templated like new ones++ * backupMode file mode of a backup (default 0o600: a pre-scrub backup holds the old IPs)++ * usage the help text (default: USAGE below)++ * root base for relative --db / --backup paths (default process.cwd())++ * env environment (default process.env)++ *++ * Exit codes: 0 done (or dry run), 1 a check failed (backup unreadable, rollup totals changed),++ * 2 refused (bad arguments, no backup choice, existing backup target, missing database).++ */++++const fs = require('fs');++const path = require('path');++const retention = require('./retention');++++const USAGE = `Raw analytics retention and one-time scrub (ADR-021).++++ analytics-prune dry run: counts only, changes nothing++ analytics-prune --scrub dry run including what the scrub would rewrite++ analytics-prune --apply --backup <path> online backup, then prune++ analytics-prune --apply --scrub --backup <path>++ analytics-prune --apply --no-backup prune without a backup (explicit)++++Options:++ --db <file> an analytics database (repeatable; default: the service's own)++ --days <n> keep raw events newer than n days, 1..30 (default 30)++ --scrub after pruning, rewrite the remaining rows: ip/user_id/city -> NULL, path -> route++ template, referer -> origin, user_agent -> class, legacy session ids -> NULL; same++ path/referer reduction in the rollups' top lists (their counts are untouched)++ --backup <path> one database: a new file; several: a directory, each backup <name>.analytics.db++ in it (sqlite online backup, owner-only, verified with quick_check and a row count)++ --no-backup explicitly skip the backup++ --no-vacuum skip the VACUUM that follows an --apply (VACUUM rewrites the file so pruned and++ scrubbed values do not survive in free pages; it needs ~2x the database size free++ and holds the database's write lock while it runs)++ --batch <n> rows per write batch (default 5000)++++--apply refuses to run without --backup or --no-backup. Rollup totals are compared before and after++every database; any difference exits 1. Prune and scrub are safe while the service is up (short++batches; the tracker retries a busy flush).`;++++function parseArgs(argv, extra = {}) {++ const a = {++ apply: false, scrub: false, vacuum: true, backup: null, noBackup: false,++ days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, dbs: [], db: null,++ };++ for (const key of Object.values(extra)) a[key] = [];++ for (let i = 0; i < argv.length; i++) {++ const k = argv[i];++ const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };++ if (k === '--apply') a.apply = true;++ else if (k === '--scrub') a.scrub = true;++ else if (k === '--no-vacuum') a.vacuum = false;++ else if (k === '--no-backup') a.noBackup = true;++ else if (k === '--backup') a.backup = val();++ else if (k === '--days') a.days = Number(val());++ else if (k === '--batch') a.batch = Number(val());++ else if (k === '--db') a.dbs.push(val());++ else if (Object.prototype.hasOwnProperty.call(extra, k)) a[extra[k]].push(val());++ else if (k === '-h' || k === '--help') a.help = true;++ else throw new Error(`unknown option ${k}`);++ }++ a.db = a.dbs[0] || null;++ retention.checkDays(a.days);++ if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');++ if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');++ return a;++}++++/** [{ name, file }] for explicit --db paths: apps/<app>/data/analytics.db -> "<app>", else the file's base name. */++function namedTargets(files, root = process.cwd()) {++ const seen = new Map();++ return files.map((f) => {++ const file = path.resolve(root, f);++ let name = path.basename(path.dirname(file)) === 'data'++ ? path.basename(path.dirname(path.dirname(file)))++ : path.basename(file, path.extname(file));++ const n = (seen.get(name) || 0) + 1;++ seen.set(name, n);++ if (n > 1) name = `${name}-${n}`;++ return { name, file };++ });++}++++function resolveTargets(args, opts) {++ const root = opts.root || process.cwd();++ if (args.dbs.length) return namedTargets(args.dbs, root);++ if (opts.targets) return opts.targets(args);++ const d = typeof opts.defaultDb === 'function' ? opts.defaultDb(args, opts.env || process.env) : opts.defaultDb;++ if (!d) return [];++ return namedTargets([d], root);++}++++const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;++function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }++function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }++const sameTotals = (a, b) => JSON.stringify(a) === JSON.stringify(b);++++async function backupTo(Database, db, target, need, mode, expectRows, log) {++ if (fs.existsSync(target)) { log(`error: backup target ${target} already exists; choose a new path`); return 2; }++ const free = freeBytes(path.dirname(target));++ if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(target)}, the backup needs about ${mb(need)}`); return 2; }++ // The backup may hold pre-ADR rows (IPs, user ids): owner-only from the first byte.++ const umask = process.umask(0o077);++ try { await db.backup(target); } finally { process.umask(umask); }++ fs.chmodSync(target, mode);++ const b = new Database(target, { readonly: true });++ try {++ const ok = b.pragma('quick_check', { simple: true });++ const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()++ ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;++ if (ok !== 'ok' || n < expectRows) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }++ } finally { b.close(); }++ log(`backup ${target} (${mb(fileBytes(target))}, mode ${(mode & 0o777).toString(8).padStart(4, '0')}, quick_check ok)`);++ return 0;++}++++async function runOne(t, args, backupTarget, opts, log) {++ const { Database } = opts;++ const pathOpts = { paramPrefixes: opts.paramPrefixes, pathRules: opts.pathRules };++ if (!fs.existsSync(t.file)) { log(`error: ${t.file} does not exist`); return 2; }++ const db = new Database(t.file, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });++ try {++ db.pragma('busy_timeout = 5000');++ const walBytes = fileBytes(t.file + '-wal');++ const before = retention.inspect(db, { days: args.days, ...pathOpts });++ log(`\n[${t.name}] ${t.file}`);++ log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);++ log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);++ log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);++ if (before.toScrub) {++ const s = before.toScrub;++ log(`scrub of the rows that stay: ${s.personal} with ip/user_id/city, ${s.paths} paths to template, ${s.referers} referers to origin, ${s.user_agents} user agents to class, ${s.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);++ }++ log(`rate rows ${before.rateRows || 0} (IP counters; the tracker keeps them in memory now)`);++ for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);++ if (!args.apply) return 0;++++ if (backupTarget) {++ const code = await backupTo(Database, db, backupTarget, before.bytes + walBytes, opts.backupMode, before.events, log);++ if (code) return code;++ }++++ db.pragma('secure_delete = ON');++ const totalsBefore = retention.rollupTotals(db);++ const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });++ log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);++ if (args.scrub) {++ const s = await retention.scrubEvents(db, { batchSize: args.batch, ...pathOpts });++ const r = retention.scrubRollups(db, pathOpts);++ log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);++ }++ const totalsAfter = retention.rollupTotals(db);++ if (!sameTotals(totalsBefore, totalsAfter)) {++ log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);++ return 1;++ }++ log('rollups totals unchanged');++ if (args.vacuum && (pruned.deleted || args.scrub)) {++ db.pragma('wal_checkpoint(TRUNCATE)');++ db.exec('VACUUM');++ db.pragma('wal_checkpoint(TRUNCATE)');++ log(`vacuumed ${mb(fileBytes(t.file))}`);++ }++ const after = retention.inspect(db, { days: args.days, ...pathOpts });++ log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);++ return 0;++ } finally {++ db.close();++ }++}++++/** Run the command. Resolves the exit code; never calls process.exit. */++async function main(argv, opts = {}) {++ const log = opts.log || console.log;++ if (typeof opts.Database !== 'function') throw new TypeError('prune-cli: pass { Database } (better-sqlite3)');++ const o = { backupMode: 0o600, ...opts };++ let args;++ try { args = parseArgs(argv, o.options || {}); } catch (e) { log(`error: ${e.message}`); return 2; }++ if (args.help) { log(o.usage || USAGE); return 0; }++ const list = resolveTargets(args, o);++ if (!list.length) { log('no analytics databases found (use --db)'); return 2; }++ if (args.apply && !args.backup && !args.noBackup) {++ log('refusing --apply without a backup: pass --backup <new file or directory> (sqlite online backup) or --no-backup');++ return 2;++ }++ let backupFor = () => null;++ if (args.apply && args.backup) {++ const target = path.resolve(o.root || process.cwd(), args.backup);++ if (list.length === 1) {++ backupFor = () => target;++ } else {++ if (fs.existsSync(target) && !fs.statSync(target).isDirectory()) { log(`error: ${target} is a file; several databases need a directory`); return 2; }++ const clash = list.map((t) => path.join(target, `${t.name}.analytics.db`)).filter((f) => fs.existsSync(f));++ if (clash.length) { log(`error: backup targets already exist: ${clash.join(', ')}`); return 2; }++ fs.mkdirSync(target, { recursive: true, mode: 0o700 });++ backupFor = (t) => path.join(target, `${t.name}.analytics.db`);++ }++ }++ let worst = 0;++ for (const t of list) {++ const code = await runOne(t, args, backupFor(t), o, log);++ worst = Math.max(worst, code);++ if (code) { if (list.length > 1) log(`[${t.name}] stopped (exit ${code}); later databases not touched`); break; }++ }++ if (!args.apply && worst === 0) log('\ndry run: nothing changed. Re-run with --apply --backup <file|dir> (or --no-backup).');++ return worst;++}++++/** For a service's script: `if (require.main === module) run(main)` — exits with main's code. */++function run(mainFn, argv = process.argv.slice(2)) {++ return Promise.resolve()++ .then(() => mainFn(argv))++ .then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });++}++++module.exports = { USAGE, main, run, parseArgs, namedTargets, resolveTargets };nnew file mode 100644@@@ -0,0 +1,248 @@++'use strict';++/**++ * Raw analytics retention and the one-time scrub (ADR-021).++ *++ * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded++ * batches, yielding to the event loop between batches. Rollups++ * (analytics_hourly / analytics_daily) are never touched. Also clears++ * leftovers: visitor hashes older than yesterday, salts of past days,++ * and analytics_rate_tracking (IP counters live in memory now).++ * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →++ * NULL (authenticated = 1 where a user id was present), path → route++ * template, referer → origin, user_agent → class, session ids that are++ * not rotating ids → NULL.++ * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /++ * top_referers JSON (counts merged, never changed).++ * inspect(db, { days }) read-only counts for a dry run.++ * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.++ * schedulePrune(db, opts) the nightly prune on unref'd timers (first run 5 min after boot,++ * then every 24 h); returns { run, stop }.++ *++ * scrubEvents / scrubRollups / inspect take the service's path options ({ paramPrefixes, pathRules },++ * see privacy.normalisePath) so legacy rows are templated with the same rules as new ones.++ *++ * Used by the tracker (opts.retention), a service's own job runner, and ./prune-cli.js.++ */++++const { ensureSchema } = require('./schema');++const privacy = require('./privacy');++++const MAX_DAYS = 30;++const DEFAULT_BATCH = 5000;++const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;++++const pad = (n) => String(n).padStart(2, '0');++function sqlTime(ms) {++ const d = new Date(ms);++ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;++}++++function checkDays(days) {++ const n = Number(days);++ if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {++ throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);++ }++ return n;++}++++/** The created_at bound: rows strictly older than this are pruned. */++function cutoffFor(days, nowMs = Date.now()) {++ return sqlTime(nowMs - checkDays(days) * 86400000);++}++++const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);++const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);++const tick = () => new Promise((r) => setImmediate(r));++++/**++ * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write++ * transaction). Returns { cutoff, deleted, batches, complete }.++ */++async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {++ const cutoff = cutoffFor(days, now());++ const out = { cutoff, deleted: 0, batches: 0, complete: true };++ if (!tableExists(db, 'analytics_events')) return out;++ const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');++ for (;;) {++ if (out.batches >= maxBatches) { out.complete = false; break; }++ const n = del.run(cutoff, batchSize).changes;++ out.batches++;++ out.deleted += n;++ if (n < batchSize) break;++ await tick();++ }++ const today = sqlTime(now()).slice(0, 10);++ const yesterday = sqlTime(now() - 86400000).slice(0, 10);++ if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);++ if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);++ if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();++ return out;++}++++/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */++function registerFunctions(db, { paramPrefixes, pathRules } = {}) {++ const opts = privacy.pathOptions({ paramPrefixes, pathRules });++ db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));++ db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));++ db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));++ db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));++}++++/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */++async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes, pathRules } = {}) {++ ensureSchema(db);++ registerFunctions(db, { paramPrefixes, pathRules });++ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();++ const out = { rows: 0, batches: 0 };++ if (lo == null) return out;++ const upd = db.prepare(`++ UPDATE analytics_events SET++ authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,++ ip = NULL, user_id = NULL, city = NULL,++ session_id = ov_rotating_session(session_id),++ path = ov_route_template(path),++ referer = ov_referer_origin(referer),++ user_agent = ov_ua_class(user_agent)++ WHERE id >= ? AND id < ?++ `);++ for (let from = lo; from <= hi; from += batchSize) {++ out.rows += upd.run(from, from + batchSize).changes;++ out.batches++;++ await tick();++ }++ return out;++}++++/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */++function reduceTopList(json, key, fn) {++ let list;++ try { list = JSON.parse(json); } catch { return null; }++ if (!Array.isArray(list)) return null;++ const merged = new Map();++ for (const item of list) {++ if (!item || typeof item !== 'object') continue;++ const k = fn(item[key]);++ if (k == null) continue;++ merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));++ }++ const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);++ const text = JSON.stringify(next);++ return text === json ? null : text;++}++++/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */++function scrubRollups(db, { paramPrefixes, pathRules } = {}) {++ const opts = privacy.pathOptions({ paramPrefixes, pathRules });++ const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));++ const out = { hourly: 0, daily: 0 };++ for (const table of ['analytics_hourly', 'analytics_daily']) {++ if (!tableExists(db, table)) continue;++ const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();++ const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);++ db.transaction(() => {++ for (const r of rows) {++ const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;++ const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;++ if (paths == null && refs == null) continue;++ upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);++ out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;++ }++ })();++ }++ return out;++}++++/** Sums of every rollup counter; equal before and after a prune or scrub. */++function rollupTotals(db) {++ const out = {};++ if (tableExists(db, 'analytics_hourly')) {++ out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,++ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,++ TOTAL(error_count) AS error_count FROM analytics_hourly`).get();++ }++ if (tableExists(db, 'analytics_daily')) {++ out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,++ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,++ TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();++ }++ return out;++}++++/** Read-only counts for a dry run. */++function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes, pathRules } = {}) {++ const cutoff = cutoffFor(days, now());++ const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };++ const pageSize = db.pragma('page_size', { simple: true });++ out.bytes = db.pragma('page_count', { simple: true }) * pageSize;++ out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;++ out.rollups = rollupTotals(db);++ if (!tableExists(db, 'analytics_events')) return out;++ registerFunctions(db, { paramPrefixes, pathRules });++ const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();++ out.events = e.n;++ out.oldest = e.oldest;++ out.newest = e.newest;++ out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);++ out.remaining = out.events - out.older;++ const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))++ .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';++ const r = db.prepare(`++ SELECT++ COUNT(*) FILTER (WHERE ${personal}) AS personal,++ COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,++ COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,++ COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,++ COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions++ FROM analytics_events WHERE created_at >= ?++ `).get(cutoff);++ out.toScrub = r;++ out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;++ return out;++}++++/**++ * Run pruneRawEvents(db, { days }) on timers: first after `initialDelayMs` (default 5 min), then every++ * `intervalMs` (default 24 h). Timers are unref'd, so they never hold the process open. An overlapping++ * run is skipped (resolves null). `label` names the service in log lines.++ * Returns { run, stop }: run() prunes now (resolves the prune result, or null on error/overlap).++ */++function schedulePrune(db, {++ days = MAX_DAYS, batchSize = DEFAULT_BATCH, initialDelayMs = 5 * 60 * 1000, intervalMs = 24 * 60 * 60 * 1000,++ label = 'Analytics', log = console.log, logError = console.error, now = Date.now,++} = {}) {++ checkDays(days);++ let running = false;++ const run = async () => {++ if (running) return null;++ running = true;++ try {++ const out = await pruneRawEvents(db, { days, batchSize, now });++ if (out.deleted) log(`[${label}] pruned ${out.deleted} raw events older than ${out.cutoff}`);++ return out;++ } catch (err) {++ logError(`[${label}] analytics-prune error:`, err.message);++ return null;++ } finally {++ running = false;++ }++ };++ const first = setTimeout(run, initialDelayMs);++ const every = setInterval(run, intervalMs);++ if (first.unref) first.unref();++ if (every.unref) every.unref();++ return { run, stop() { clearTimeout(first); clearInterval(every); } };++}++++module.exports = {++ MAX_DAYS,++ DEFAULT_BATCH,++ checkDays,++ cutoffFor,++ pruneRawEvents,++ scrubEvents,++ scrubRollups,++ rollupTotals,++ inspect,++ reduceTopList,++ schedulePrune,++};nnew file mode 100644@@@ -0,0 +1,123 @@++'use strict';++/**++ * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the++ * existing analytics.db files and the admin dashboards keep working; the columns that used to hold++ * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and++ * user_agent / referer / path hold the reduced forms (class, origin, route template).++ *++ * Added:++ * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)++ * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;++ * rows are deleted once the day's rollup is final (≤ ~25 h)++ * analytics_day_salts the random salt of the current UTC day (so a restart does not++ * double-count); deleted together with that day's hashes++ * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)++ */++++const ANALYTICS_SCHEMA = `++ CREATE TABLE IF NOT EXISTS analytics_events (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ event_type TEXT NOT NULL DEFAULT 'pageview',++ path TEXT,++ method TEXT DEFAULT 'GET',++ status_code INTEGER,++ response_time_ms INTEGER,++ user_id INTEGER,++ session_id TEXT,++ ip TEXT,++ country TEXT,++ city TEXT,++ user_agent TEXT,++ referer TEXT,++ is_bot INTEGER DEFAULT 0,++ bot_type TEXT,++ device_type TEXT,++ browser TEXT,++ os TEXT,++ created_at DATETIME DEFAULT CURRENT_TIMESTAMP++ );++++ CREATE TABLE IF NOT EXISTS analytics_hourly (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ hour TEXT NOT NULL,++ pageviews INTEGER DEFAULT 0,++ api_calls INTEGER DEFAULT 0,++ unique_visitors INTEGER DEFAULT 0,++ unique_users INTEGER DEFAULT 0,++ bot_hits INTEGER DEFAULT 0,++ avg_response_ms INTEGER DEFAULT 0,++ error_count INTEGER DEFAULT 0,++ bandwidth_bytes INTEGER DEFAULT 0,++ top_paths TEXT,++ top_referers TEXT,++ UNIQUE(service, hour)++ );++++ CREATE TABLE IF NOT EXISTS analytics_daily (++ id INTEGER PRIMARY KEY AUTOINCREMENT,++ service TEXT NOT NULL,++ date TEXT NOT NULL,++ pageviews INTEGER DEFAULT 0,++ api_calls INTEGER DEFAULT 0,++ unique_visitors INTEGER DEFAULT 0,++ unique_users INTEGER DEFAULT 0,++ new_users INTEGER DEFAULT 0,++ bot_hits INTEGER DEFAULT 0,++ avg_response_ms INTEGER DEFAULT 0,++ error_count INTEGER DEFAULT 0,++ top_paths TEXT,++ top_referers TEXT,++ top_countries TEXT,++ device_breakdown TEXT,++ browser_breakdown TEXT,++ UNIQUE(service, date)++ );++++ CREATE TABLE IF NOT EXISTS analytics_rate_tracking (++ ip TEXT NOT NULL,++ window_start INTEGER NOT NULL,++ hit_count INTEGER DEFAULT 1,++ PRIMARY KEY (ip, window_start)++ );++++ CREATE TABLE IF NOT EXISTS analytics_visitor_days (++ service TEXT NOT NULL,++ day TEXT NOT NULL,++ hour TEXT NOT NULL,++ vhash TEXT NOT NULL,++ authenticated INTEGER NOT NULL DEFAULT 0,++ PRIMARY KEY (service, hour, vhash)++ );++++ CREATE TABLE IF NOT EXISTS analytics_day_salts (++ service TEXT NOT NULL,++ day TEXT NOT NULL,++ salt TEXT NOT NULL,++ PRIMARY KEY (service, day)++ );++++ CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);++ CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);++ CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);++ CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);++++ -- Indexes over the retired personal columns.++ DROP INDEX IF EXISTS idx_analytics_events_ip;++ DROP INDEX IF EXISTS idx_analytics_events_user;++`;++++/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */++function ensureSchema(db) {++ db.exec(ANALYTICS_SCHEMA);++ const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);++ if (!cols.includes('authenticated')) {++ db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');++ }++}++++module.exports = { ANALYTICS_SCHEMA, ensureSchema };nnew file mode 100644@@@ -0,0 +1,753 @@++'use strict';++/**++ * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the pre-ADR++ * openvibe-shared AnalyticsTracker (same constructor, tables and getStats/getOverview/getBotAnalysis++ * shapes), minus the personal data:++ *++ * raw row service, event_type, route template, method, status, response time, rotating session++ * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,++ * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.++ * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after++ * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.++ * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped++ * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).++ * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.++ * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as++ * the day is over); hashes go to analytics_visitor_days only, never to raw events,++ * and are deleted by the first aggregation after their day ends, right after that day's++ * final hourly/daily rollup — so no hash or salt outlives its day by more than about++ * one aggregation interval (1 h). Rollups keep only the counts.++ *++ * opt-out a request with `Sec-GPC: 1` or `DNT: 1` is not recorded at all (privacy.optedOut):++ * no row, no visitor hash, no session id, no rate counter.++ *++ * Retention of raw events (30 days) is ./retention.js; this tracker writes, rolls up and (with++ * opts.retention) schedules the prune. The row shape is analytics/event.v1 (./event.js).++ */++++const crypto = require('crypto');++const { ensureSchema } = require('./schema');++const privacy = require('./privacy');++const retention = require('./retention');++const { allowed } = require('./event');++++const SESSION_IDLE_MS = 30 * 60 * 1000;++const MAX_TRACKED_CLIENTS = 200000;++const MAX_BUFFER = 5000;++const SESSION_ID_RE = /^[0-9a-f]{16}$/;++const METHOD_RE = /^[A-Z][A-Z-]{0,15}$/;++const EVENT_NAME_RE = /^[A-Za-z][A-Za-z0-9_.:-]{0,63}$/;++const ALLOWED = { device: allowed('device'), browser: allowed('browser'), os: allowed('os') };++++const pad = (n) => String(n).padStart(2, '0');++/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */++function sqlTime(ms) {++ const d = new Date(ms);++ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;++}++const dayOf = (ms) => sqlTime(ms).slice(0, 10);++const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';++const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');++++function isStaticOrHealth(path) {++ return path === '/api/health' || path === '/health' ||++ /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);++}++++class AnalyticsTracker {++ /**++ * @param {object} db better-sqlite3 handle (its own analytics database)++ * @param {string} service service name ('live', 'openvibe-yt', …)++ * @param {object} [opts]++ * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter++ * (added to privacy.DEFAULT_PARAM_PREFIXES), e.g. Network's ['avatar', 'anon', 'projects']++ * @param {Array<[RegExp, string]>} [opts.pathRules] rewrites applied to a raw path before it is++ * templated, for parameters the segment rules cannot see (privacy.normalisePath)++ * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:++ * the raw-event prune on a timer (retention.schedulePrune; the job is this.pruneJob). On by++ * default (ADR-021: raw events at most 30 days); pass false only when the service schedules++ * retention.pruneRawEvents itself.++ * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)++ * @param {function} [opts.now] clock (tests)++ */++ constructor(db, service, opts = {}) {++ this.db = db;++ this.service = service;++ this._now = opts.now || Date.now;++ this._pathOpts = privacy.pathOptions({ paramPrefixes: opts.paramPrefixes, pathRules: opts.pathRules });++ this.pruneJob = null;++ this._buffer = [];++ this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }++ this._sessions = new Map(); // vhash -> { sid, last }++ this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)++ this._saltDay = null;++ this._salt = null;++ this._timers = [];++++ ensureSchema(db);++ try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }++ try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }++ db.exec('DELETE FROM analytics_rate_tracking');++++ this._insertEvent = db.prepare(`++ INSERT INTO analytics_events++ (service, event_type, path, method, status_code, response_time_ms,++ user_id, session_id, ip, country, city, user_agent, referer,++ is_bot, bot_type, device_type, browser, os, authenticated, created_at)++ VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ `);++ this._upsertVisitor = db.prepare(`++ INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)++ ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)++ `);++ this._write = db.transaction((events, visitors) => {++ for (const e of events) {++ this._insertEvent.run(++ e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,++ e.session_id, e.country, e.user_agent, e.referer,++ e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,++ );++ }++ for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);++ });++++ if (opts.timers !== false) {++ const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };++ const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };++ every(() => { this.flush(); this._sweep(); }, 5000);++ every(() => this.aggregate(), 60 * 60 * 1000);++ once(() => this.aggregate(), 10_000);++ if (opts.retention !== false) {++ const r = opts.retention && typeof opts.retention === 'object' ? opts.retention : {};++ this.pruneJob = retention.schedulePrune(db, { label: `Analytics:${service}`, now: this._now, ...r });++ }++ }++ }++++ // ── Per-request reducers ──────────────────────────────────++++ _daySalt(day) {++ if (this._saltDay === day) return this._salt;++ let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);++ if (!row) {++ this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')++ .run(this.service, day, crypto.randomBytes(32).toString('hex'));++ row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);++ }++ // Earlier days' salts are never needed again: their hashes are already written.++ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);++ this._saltDay = day;++ this._salt = row.salt;++ this._sessions.clear();++ return this._salt;++ }++++ _visitorHash(ip, ua, day) {++ return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);++ }++++ _sessionId(vhash, nowMs) {++ let s = this._sessions.get(vhash);++ if (!s || nowMs - s.last > SESSION_IDLE_MS) {++ s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };++ this._sessions.set(vhash, s);++ }++ s.last = nowMs;++ return s.sid;++ }++++ /** Hits from this client in the current and previous minute (memory only). */++ _rateHit(ip, nowMs) {++ const minute = Math.floor(nowMs / 60000);++ let r = this._rates.get(ip);++ if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }++ if (r.minute !== minute) {++ r.prev = r.minute === minute - 1 ? r.count : 0;++ r.count = 0;++ r.minute = minute;++ }++ r.count++;++ return r.count + r.prev;++ }++++ _sweep() {++ const nowMs = this._now();++ const minute = Math.floor(nowMs / 60000);++ for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);++ for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);++ if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();++ if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();++ }++++ _push(event, visitor) {++ if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();++ this._buffer.push(event);++ if (visitor) {++ const key = `${visitor.hour}|${visitor.vhash}`;++ const prev = this._visitors.get(key);++ if (!prev) this._visitors.set(key, visitor);++ else if (visitor.authenticated) prev.authenticated = true;++ }++ if (this._buffer.length >= 100) this.flush();++ }++++ /**++ * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted++ * router is relative by the time the response finishes). Returns false, recording nothing, for an++ * opted-out request (Sec-GPC: 1 / DNT: 1); true otherwise.++ */++ record(req, res, entryPath, responseTimeMs) {++ const headers = req.headers || {};++ if (privacy.optedOut(headers)) return false;++ entryPath = String(entryPath || '');++ const nowMs = this._now();++ const ua = String(headers['user-agent'] || '');++ const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');++ const bot = privacy.classifyRequest({ headers, path: entryPath });++ if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {++ bot.isBot = true;++ bot.botType = 'rate_limit';++ bot.confidence = 0.80;++ }++ const parsed = privacy.parseUserAgent(ua);++ const day = dayOf(nowMs);++ const vhash = this._visitorHash(ip, ua, day);++ const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));++ const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');++ this._push({++ service: this.service,++ event_type: isApi ? 'api_call' : 'pageview',++ path: privacy.routeTemplate(req, entryPath, this._pathOpts),++ method: METHOD_RE.test(String(req.method || '')) ? req.method : null,++ status_code: res.statusCode,++ response_time_ms: responseTimeMs,++ session_id: this._sessionId(vhash, nowMs),++ country: privacy.countryCode(headers['cf-ipcountry']),++ user_agent: privacy.uaClass(ua),++ referer: privacy.refererOrigin(headers.referer || headers.referrer),++ is_bot: bot.isBot,++ bot_type: bot.botType,++ device_type: parsed.device,++ browser: parsed.browser,++ os: parsed.os,++ authenticated,++ created_at: sqlTime(nowMs),++ }, { day, hour: hourOf(nowMs), vhash, authenticated });++ return true;++ }++++ /**++ * Express middleware: one row per finished request (health checks, static assets and opted-out++ * requests skipped).++ */++ middleware() {++ return (req, res, next) => {++ const entryPath = req.path || '';++ if (isStaticOrHealth(entryPath) || privacy.optedOut(req.headers)) return next();++ const started = Date.now();++ let done = false;++ const finish = () => {++ if (done) return;++ done = true;++ try { this.record(req, res, entryPath, Date.now() - started); }++ catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }++ };++ res.once('finish', finish);++ res.once('close', finish);++ next();++ };++ }++++ /**++ * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,++ * referer and user_agent are reduced like a request's; session_id is kept only if it is one of++ * this tracker's rotating ids; any other value outside analytics/event.v1 is stored as NULL, and++ * an event name outside it throws a TypeError. Pass the triggering request's headers as++ * `data.headers` when there is one: an opted-out request (Sec-GPC / DNT) records nothing.++ * Returns whether it recorded.++ */++ trackEvent(eventType, data = {}) {++ const name = String(eventType == null ? 'event' : eventType);++ if (!EVENT_NAME_RE.test(name)) throw new TypeError(`analytics event name must match ${EVENT_NAME_RE} (analytics/event.v1)`);++ if (privacy.optedOut(data.headers)) return false;++ const nowMs = this._now();++ const ua = data.user_agent ? String(data.user_agent) : '';++ const parsed = ua ? privacy.parseUserAgent(ua) : {};++ const method = data.method ? String(data.method).toUpperCase() : null;++ const status = Number(data.status_code);++ const ms = Number(data.response_time_ms);++ const botType = data.bot_type ? String(data.bot_type) : null;++ const pick = (field, given, fallback) => (ALLOWED[field].includes(given) ? given : (fallback || null));++ this._push({++ service: this.service,++ event_type: name,++ path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,++ method: method && METHOD_RE.test(method) ? method : null,++ status_code: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,++ response_time_ms: Number.isFinite(ms) && ms >= 0 ? Math.round(ms) : null,++ session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,++ country: privacy.countryCode(data.country),++ user_agent: ua ? privacy.uaClass(ua) : null,++ referer: privacy.refererOrigin(data.referer),++ is_bot: !!data.is_bot,++ bot_type: botType && /^[a-z0-9_]{1,32}$/.test(botType) ? botType : null,++ device_type: pick('device', data.device_type, parsed.device),++ browser: pick('browser', data.browser, parsed.browser),++ os: pick('os', data.os, parsed.os),++ authenticated: !!data.authenticated,++ created_at: sqlTime(nowMs),++ }, null);++ return true;++ }++++ /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */++ flush() {++ if (!this._buffer.length && !this._visitors.size) return;++ const events = this._buffer.splice(0);++ const visitors = [...this._visitors.values()];++ this._visitors.clear();++ try {++ this._write(events, visitors);++ } catch (err) {++ console.error(`[Analytics:${this.service}] Flush error:`, err.message);++ this._buffer = events.slice(-MAX_BUFFER);++ for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);++ }++ }++++ // ── Rollups ───────────────────────────────────────────────++++ /**++ * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has++ * visitor hashes (finalised here), after which those hashes are deleted.++ */++ aggregate() {++ try {++ this.flush();++ const nowMs = this._now();++ const today = dayOf(nowMs);++ const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);++ const days = new Set([today, dayOf(nowMs - 86400000)]);++ const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);++ for (const p of pending) { hours.add(p.hour); days.add(p.day); }++ for (const h of hours) this._aggregateHour(h);++ for (const d of days) this._aggregateDay(d);++ this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);++ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);++ } catch (err) {++ console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);++ }++ }++++ _aggregateHour(hourStart) {++ const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);++ const svc = this.service;++ const stats = this.db.prepare(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count++ FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ?++ `).get(svc, hourStart, nextHour);++ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;++++ // Uniques only grow within a period, so never lower a stored count (restarts, or rows++ // counted before this tracker when the old one still stored IPs).++ const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);++ const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};++++ const topPaths = this.db.prepare(`++ SELECT path, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0++ GROUP BY path ORDER BY cnt DESC LIMIT 10++ `).all(svc, hourStart, nextHour);++ const topReferers = this.db.prepare(`++ SELECT referer, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0++ GROUP BY referer ORDER BY cnt DESC LIMIT 10++ `).all(svc, hourStart, nextHour);++++ this.db.prepare(`++ INSERT INTO analytics_hourly++ (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)++ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ ON CONFLICT(service, hour) DO UPDATE SET++ pageviews = excluded.pageviews, api_calls = excluded.api_calls,++ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,++ bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,++ error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers++ `).run(++ svc, hourStart, stats.pageviews, stats.api_calls,++ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),++ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,++ JSON.stringify(topPaths), JSON.stringify(topReferers),++ );++ }++++ _aggregateDay(date) {++ const dayStart = date + ' 00:00:00';++ const dayEnd = date + ' 23:59:59';++ const svc = this.service;++ const q = (sql) => this.db.prepare(sql);++ const stats = q(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count++ FROM analytics_events++ WHERE service = ? AND created_at >= ? AND created_at <= ?++ `).get(svc, dayStart, dayEnd);++ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;++++ const v = q(`++ SELECT COUNT(DISTINCT vhash) AS visitors,++ COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users++ FROM analytics_visitor_days WHERE service = ? AND day = ?++ `).get(svc, date);++ const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};++++ const range = [svc, dayStart, dayEnd];++ const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);++ const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);++ const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);++ const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);++ const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);++++ q(`++ INSERT INTO analytics_daily++ (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,++ avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)++ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)++ ON CONFLICT(service, date) DO UPDATE SET++ pageviews = excluded.pageviews, api_calls = excluded.api_calls,++ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,++ new_users = excluded.new_users, bot_hits = excluded.bot_hits,++ avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,++ top_paths = excluded.top_paths, top_referers = excluded.top_referers,++ top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,++ browser_breakdown = excluded.browser_breakdown++ `).run(++ svc, date, stats.pageviews, stats.api_calls,++ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),++ // New users needs a user id history, which analytics no longer has: keep what was++ // recorded before, NULL (not measured) from now on.++ prev.new_users != null ? prev.new_users : null,++ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,++ JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),++ JSON.stringify(devices), JSON.stringify(browsers),++ );++ }++++ // ── Reads (admin dashboards) ─────────────────────────────++ // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,++ // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based++ // numbers (daily/hourly unique_visitors) are true unique visitors per period.++++ getStats(options = {}) {++ const { days = 30, hours, service } = options;++ const svc = service || this.service;++ const nowMs = this._now();++ const rangeMs = hours ? hours * 3600000 : days * 86400000;++ const rangeCutoff = sqlTime(nowMs - rangeMs);++ const isSubDay = hours && hours < 24;++ const effectiveDays = hours ? hours / 24 : days;++ const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;++ const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;++ const q = (sql) => this.db.prepare(sql);++ const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };++++ let daily = [];++ if (!isSubDay) {++ daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));++ }++++ const hourlyLimit = hours ? Math.min(hours, 168) : 24;++ const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);++ const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);++++ let timeBuckets = [];++ if (isSubDay) {++ timeBuckets = q(`++ SELECT ${bucketExpr} as bucket,++ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,++ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms++ FROM analytics_events++ WHERE service = ? AND created_at >= ?++ GROUP BY bucket ORDER BY bucket ASC++ `).all(svc, rangeCutoff);++ }++++ let summary;++ if (hours && hours < 48) {++ summary = q(`++ SELECT++ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,++ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,++ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,++ COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,++ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms++ FROM analytics_events++ WHERE service = ? AND created_at >= ?++ `).get(svc, rangeCutoff);++ } else {++ summary = q(`++ SELECT++ SUM(pageviews) as total_pageviews,++ SUM(api_calls) as total_api_calls,++ SUM(unique_visitors) as total_unique_visitors,++ SUM(unique_users) as total_unique_users,++ SUM(bot_hits) as total_bot_hits,++ SUM(error_count) as total_errors,++ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms++ FROM analytics_daily++ WHERE service = ? AND date >= ?++ `).get(svc, dayOf(nowMs - rangeMs));++ }++++ const realtime = q(`++ SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots++ FROM analytics_events WHERE service = ? AND created_at >= ?++ `).get(svc, sqlTime(nowMs - 300000));++++ const human = [svc, rangeCutoff];++ const topPages = q(`++ SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY path ORDER BY hits DESC LIMIT 20++ `).all(...human);++ const topReferers = q(`++ SELECT referer, COUNT(*) as hits FROM analytics_events++ WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0++ GROUP BY referer ORDER BY hits DESC LIMIT 15++ `).all(...human);++ const botBreakdown = q(`++ SELECT bot_type, COUNT(*) as hits FROM analytics_events++ WHERE service = ? AND created_at >= ? AND is_bot = 1++ GROUP BY bot_type ORDER BY hits DESC LIMIT 15++ `).all(...human);++ const statusCodes = q(`++ SELECT CASE++ WHEN status_code >= 200 AND status_code < 300 THEN '2xx'++ WHEN status_code >= 300 AND status_code < 400 THEN '3xx'++ WHEN status_code >= 400 AND status_code < 500 THEN '4xx'++ WHEN status_code >= 500 THEN '5xx'++ ELSE 'other' END as group_code,++ COUNT(*) as cnt++ FROM analytics_events WHERE service = ? AND created_at >= ?++ GROUP BY group_code ORDER BY cnt DESC++ `).all(...human);++ const breakdown = (col) => q(`++ SELECT ${col}, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY ${col} ORDER BY cnt DESC++ `).all(...human);++ const deviceBreakdown = breakdown('device_type');++ const browserBreakdown = breakdown('browser');++ const osBreakdown = breakdown('os');++ const countryBreakdown = q(`++ SELECT country, COUNT(*) as cnt FROM analytics_events++ WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0++ GROUP BY country ORDER BY cnt DESC LIMIT 20++ `).all(...human);++++ const responsePercentiles = safe(() => {++ const vals = q(`++ SELECT response_time_ms FROM analytics_events++ WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0++ ORDER BY response_time_ms ASC++ `).pluck().all(...human);++ if (!vals.length) return {};++ const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];++ return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };++ }, {});++++ const bandwidth = safe(() => {++ const bw = q(`++ SELECT COUNT(*) as total_requests,++ COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,++ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,++ SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ `).get(...human);++ return bw && {++ total_requests: bw.total_requests || 0,++ page_requests: bw.page_requests || 0,++ api_requests: bw.api_requests || 0,++ estimated_bytes: bw.estimated_bytes || 0,++ };++ }, {});++++ const authBreakdown = safe(() => q(`++ SELECT++ COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,++ COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,++ COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ `).get(...human), {});++++ const errorTrend = safe(() => (isSubDay++ ? q(`++ SELECT ${bucketExpr} as bucket, COUNT(*) AS total,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,++ COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY bucket ORDER BY bucket ASC++ `).all(...human)++ : q(`++ SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors++ FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC++ `).all(svc, hourlyCutoff)), []);++++ const peakHours = safe(() => q(`++ SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,++ COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY hour_of_day ORDER BY hour_of_day ASC++ `).all(...human), []);++++ const topApiEndpoints = safe(() => q(`++ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,++ COUNT(*) FILTER (WHERE status_code >= 400) AS errors++ FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0++ GROUP BY path, method ORDER BY hits DESC LIMIT 20++ `).all(...human), []);++++ const sessionCount = safe(() => q(`++ SELECT COUNT(DISTINCT session_id) FROM analytics_events++ WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0++ `).pluck().get(...human), 0);++++ // New vs returning needs a visitor identity that outlives a day; analytics keeps none.++ const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };++++ const slowestEndpoints = safe(() => q(`++ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms++ FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0++ GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10++ `).all(...human), []);++++ const authTrend = safe(() => q(`++ SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,++ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,++ COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,++ COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits++ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0++ GROUP BY bucket ORDER BY bucket ASC++ `).all(...human), []);++++ return {++ service: svc,++ period_days: effectiveDays,++ period_hours: hours || null,++ summary: summary || {},++ realtime: realtime || {},++ daily,++ hourly,++ timeBuckets,++ topPages,++ topReferers,++ botBreakdown,++ statusCodes,++ deviceBreakdown,++ browserBreakdown,++ osBreakdown,++ countryBreakdown,++ responsePercentiles,++ bandwidth,++ authBreakdown,++ errorTrend,++ peakHours,++ topApiEndpoints,++ sessionCount,++ visitorTypes,++ slowestEndpoints,++ authTrend,++ };++ }++++ getOverview(days = 30) {++ const cutoff = dayOf(this._now() - days * 86400000);++ const services = this.db.prepare(`++ SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,++ SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,++ SUM(bot_hits) as bot_hits, SUM(error_count) as errors,++ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms++ FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC++ `).all(cutoff);++ const totals = this.db.prepare(`++ SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,++ SUM(bot_hits) as bot_hits, SUM(error_count) as errors++ FROM analytics_daily WHERE date >= ?++ `).get(cutoff);++ const dailyTrend = this.db.prepare(`++ SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,++ SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits++ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC++ `).all(cutoff);++ const realtime = this.db.prepare(`++ SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots++ FROM analytics_events WHERE created_at >= ? GROUP BY service++ `).all(sqlTime(this._now() - 300000));++ return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };++ }++++ /**++ * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes++ * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is++ * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.++ */++ getBotAnalysis(days = 30) {++ const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);++ const topBotIPs = this.db.prepare(`++ SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,++ COUNT(DISTINCT session_id) AS sessions,++ MIN(created_at) as first_seen, MAX(created_at) as last_seen++ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?++ GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25++ `).all(since);++ const botTrend = this.db.prepare(`++ SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits++ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC++ `).all(since.slice(0, 10));++ const botTypes = this.db.prepare(`++ SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions++ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?++ GROUP BY bot_type ORDER BY hits DESC++ `).all(since);++ const suspiciousIPs = this.db.prepare(`++ SELECT NULL AS ip, session_id, COUNT(*) as total_hits,++ COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,++ COUNT(DISTINCT path) as unique_paths,++ MIN(created_at) as first_seen, MAX(created_at) as last_seen++ FROM analytics_events++ WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL++ GROUP BY session_id HAVING total_hits > 500++ ORDER BY total_hits DESC LIMIT 20++ `).all(sqlTime(this._now() - 7 * 86400000));++ return { topBotIPs, botTrend, botTypes, suspiciousIPs };++ }++++ destroy() {++ this.flush();++ for (const t of this._timers) { clearInterval(t); clearTimeout(t); }++ this._timers = [];++ if (this.pruneJob) { this.pruneJob.stop(); this.pruneJob = null; }++ this._rates.clear();++ this._sessions.clear();++ }++}++++module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };nnew file mode 100644@@@ -0,0 +1,92 @@++{++ "$schema": "https://json-schema.org/draft/2020-12/schema",++ "$id": "https://openvibe.network/contracts/analytics/event.v1.json",++ "title": "AnalyticsEvent",++ "description": "analytics/event.v1: one raw analytics event (ADR-021). Kept by the owning service for at most 30 days, then only its rollups (daily and hourly counts) remain. It never carries an IP address, a user or subject id, a precise location (region or city), a raw user agent, a full referer URL, or a URL with ids or a query string. Requests sent with `Sec-GPC: 1` or `DNT: 1` produce no event. openvibe-shared/analytics stores it as an analytics_events row. For compatibility that row keeps the retired columns ip, user_id and city, always NULL; event.toEvent(row) maps a row to this shape.",++ "type": "object",++ "required": ["event", "service", "route", "session_id", "country", "ua_class", "is_bot", "authenticated", "timestamp"],++ "additionalProperties": false,++ "properties": {++ "event": {++ "description": "Event name: 'pageview' and 'api_call' for HTTP requests, or a custom name given to trackEvent. Row column: event_type.",++ "type": "string",++ "pattern": "^[A-Za-z][A-Za-z0-9_.:-]{0,63}$"++ },++ "service": {++ "description": "The recording service, for example 'live', 'openvibe-network' or 'openvibe-yt'.",++ "type": "string",++ "pattern": "^[a-z][a-z0-9_-]{0,63}$"++ },++ "route": {++ "description": "The route template, such as '/api/vods/:id/comments' or '/@:user'. It is the matched Express route, or else the request path with ids, usernames and slugs replaced by parameters. It has no query or fragment, and it is null for a custom event without a path. Row column: path.",++ "type": ["string", "null"],++ "maxLength": 200,++ "pattern": "^/(([A-Za-z0-9._~-]+|@?:[A-Za-z_][A-Za-z0-9_]*\\??|\\*)(/([A-Za-z0-9._~-]+|@?:[A-Za-z_][A-Za-z0-9_]*\\??|\\*))*)?$"++ },++ "method": {++ "description": "HTTP method, or null for a custom event.",++ "type": ["string", "null"],++ "pattern": "^[A-Z][A-Z-]{0,15}$"++ },++ "status": {++ "description": "HTTP status code. Row column: status_code.",++ "type": ["integer", "null"],++ "minimum": 100,++ "maximum": 599++ },++ "duration_ms": {++ "description": "Time from the request's arrival to the finished response, in milliseconds. Row column: response_time_ms.",++ "type": ["integer", "null"],++ "minimum": 0++ },++ "session_id": {++ "description": "A rotating session id: 16 random hex characters. It is renewed after 30 minutes idle and at every UTC midnight, and it is never derived from, or equal to, a user id. It is null when no session applies.",++ "type": ["string", "null"],++ "pattern": "^[0-9a-f]{16}$"++ },++ "country": {++ "description": "ISO 3166-1 alpha-2 country code taken from the CDN header, or null. This is the only geography kept.",++ "type": ["string", "null"],++ "pattern": "^[A-Z]{2}$"++ },++ "ua_class": {++ "description": "User-agent class: '<browser>/<os>/<device>', 'bot:<family>' for a known crawler, or 'none' when there is no user agent. The raw user agent is never stored. Row column: user_agent.",++ "type": ["string", "null"],++ "pattern": "^(none|bot:[a-z0-9]+|[a-z]+/[a-z]+/[a-z]+)$"++ },++ "device": {++ "description": "Row column: device_type.",++ "enum": ["desktop", "mobile", "tablet", "unknown", null]++ },++ "browser": {++ "enum": ["chrome", "firefox", "safari", "edge", "opera", "ie", "other", "unknown", null]++ },++ "os": {++ "enum": ["windows", "macos", "linux", "android", "ios", "other", "unknown", null]++ },++ "referer_origin": {++ "description": "The scheme://host[:port] of an http(s) referer, or null. Never its path or query. Row column: referer.",++ "type": ["string", "null"],++ "maxLength": 300,++ "pattern": "^https?://[^/?#\\s]+$"++ },++ "is_bot": {++ "description": "True for a known crawler, a missing user agent, a honeypot path, or a client over the request-rate threshold.",++ "type": "boolean"++ },++ "bot_type": {++ "description": "Why the event is a bot hit: known_crawler, no_useragent, honeypot_hit or rate_limit. A custom event may give its own value.",++ "type": ["string", "null"],++ "pattern": "^[a-z0-9_]{1,32}$"++ },++ "authenticated": {++ "description": "Whether the request was signed in. This is a flag and never says who.",++ "type": "boolean"++ },++ "timestamp": {++ "description": "UTC time in RFC 3339 form with second precision. Row column: created_at, stored as 'YYYY-MM-DD HH:MM:SS' UTC.",++ "type": "string",++ "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?Z$"++ }++ }++}@@@ -12,6 +12,7 @@"jsonwebtoken": "^9.0.0"},"devDependencies": {++ "better-sqlite3": "^11.10.0","express": "^4.22.3"}},@@@ -36,6 +37,61 @@"dev": true,"license": "MIT"},++ "node_modules/base64-js": {++ "version": "1.5.1",++ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",++ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",++ "dev": true,++ "funding": [++ {++ "type": "github",++ "url": "https://github.com/sponsors/feross"++ },++ {++ "type": "patreon",++ "url": "https://www.patreon.com/feross"++ },++ {++ "type": "consulting",++ "url": "https://feross.org/support"++ }++ ],++ "license": "MIT"++ },++ "node_modules/better-sqlite3": {++ "version": "11.10.0",++ "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz",++ "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==",++ "dev": true,++ "hasInstallScript": true,++ "license": "MIT",++ "dependencies": {++ "bindings": "^1.5.0",++ "prebuild-install": "^7.1.1"++ }++ },++ "node_modules/bindings": {++ "version": "1.5.0",++ "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",++ "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "file-uri-to-path": "1.0.0"++ }++ },++ "node_modules/bl": {++ "version": "4.1.0",++ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",++ "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "buffer": "^5.5.0",++ "inherits": "^2.0.4",++ "readable-stream": "^3.4.0"++ }++ },"node_modules/body-parser": {"version": "1.20.8","resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.8.tgz",@@@ -61,6 +117,31 @@"npm": "1.2.8000 || >= 1.4.16"}},++ "node_modules/buffer": {++ "version": "5.7.1",++ "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",++ "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",++ "dev": true,++ "funding": [++ {++ "type": "github",++ "url": "https://github.com/sponsors/feross"++ },++ {++ "type": "patreon",++ "url": "https://www.patreon.com/feross"++ },++ {++ "type": "consulting",++ "url": "https://feross.org/support"++ }++ ],++ "license": "MIT",++ "dependencies": {++ "base64-js": "^1.3.1",++ "ieee754": "^1.1.13"++ }++ },"node_modules/buffer-equal-constant-time": {"version": "1.0.1","resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",@@@ -108,6 +189,13 @@"url": "https://github.com/sponsors/ljharb"}},++ "node_modules/chownr": {++ "version": "1.1.4",++ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",++ "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",++ "dev": true,++ "license": "ISC"++ },"node_modules/content-disposition": {"version": "0.5.4","resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",@@@ -165,6 +253,32 @@"dev": true,"license": "MIT"},++ "node_modules/decompress-response": {++ "version": "6.0.0",++ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",++ "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "mimic-response": "^3.1.0"++ },++ "engines": {++ "node": ">=10"++ },++ "funding": {++ "url": "https://github.com/sponsors/sindresorhus"++ }++ },++ "node_modules/deep-extend": {++ "version": "0.6.0",++ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",++ "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",++ "dev": true,++ "license": "MIT",++ "engines": {++ "node": ">=4.0.0"++ }++ },"node_modules/depd": {"version": "2.0.0","resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",@@@ -186,6 +300,16 @@"npm": "1.2.8000 || >= 1.4.16"}},++ "node_modules/detect-libc": {++ "version": "2.1.2",++ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",++ "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",++ "dev": true,++ "license": "Apache-2.0",++ "engines": {++ "node": ">=8"++ }++ },"node_modules/dunder-proto": {"version": "1.0.1","resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",@@@ -227,6 +351,16 @@"node": ">= 0.8"}},++ "node_modules/end-of-stream": {++ "version": "1.4.5",++ "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",++ "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "once": "^1.4.0"++ }++ },"node_modules/es-define-property": {"version": "1.0.1","resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",@@@ -277,6 +411,16 @@"node": ">= 0.6"}},++ "node_modules/expand-template": {++ "version": "2.0.3",++ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",++ "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",++ "dev": true,++ "license": "(MIT OR WTFPL)",++ "engines": {++ "node": ">=6"++ }++ },"node_modules/express": {"version": "4.22.3","resolved": "https://registry.npmjs.org/express/-/express-4.22.3.tgz",@@@ -324,6 +468,13 @@"url": "https://opencollective.com/express"}},++ "node_modules/file-uri-to-path": {++ "version": "1.0.0",++ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",++ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",++ "dev": true,++ "license": "MIT"++ },"node_modules/finalhandler": {"version": "1.3.2","resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz",@@@ -363,6 +514,13 @@"node": ">= 0.6"}},++ "node_modules/fs-constants": {++ "version": "1.0.0",++ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",++ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",++ "dev": true,++ "license": "MIT"++ },"node_modules/function-bind": {"version": "1.1.2","resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",@@@ -412,6 +570,13 @@"node": ">= 0.4"}},++ "node_modules/github-from-package": {++ "version": "0.0.0",++ "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",++ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",++ "dev": true,++ "license": "MIT"++ },"node_modules/gopd": {"version": "1.2.0","resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",@@@ -485,6 +650,27 @@"node": ">=0.10.0"}},++ "node_modules/ieee754": {++ "version": "1.2.1",++ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",++ "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",++ "dev": true,++ "funding": [++ {++ "type": "github",++ "url": "https://github.com/sponsors/feross"++ },++ {++ "type": "patreon",++ "url": "https://www.patreon.com/feross"++ },++ {++ "type": "consulting",++ "url": "https://feross.org/support"++ }++ ],++ "license": "BSD-3-Clause"++ },"node_modules/inherits": {"version": "2.0.4","resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",@@@ -492,6 +678,13 @@"dev": true,"license": "ISC"},++ "node_modules/ini": {++ "version": "1.3.8",++ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",++ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",++ "dev": true,++ "license": "ISC"++ },"node_modules/ipaddr.js": {"version": "1.9.1","resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",@@@ -663,12 +856,49 @@"node": ">= 0.6"}},++ "node_modules/mimic-response": {++ "version": "3.1.0",++ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",++ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",++ "dev": true,++ "license": "MIT",++ "engines": {++ "node": ">=10"++ },++ "funding": {++ "url": "https://github.com/sponsors/sindresorhus"++ }++ },++ "node_modules/minimist": {++ "version": "1.2.8",++ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",++ "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",++ "dev": true,++ "license": "MIT",++ "funding": {++ "url": "https://github.com/sponsors/ljharb"++ }++ },++ "node_modules/mkdirp-classic": {++ "version": "0.5.3",++ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",++ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",++ "dev": true,++ "license": "MIT"++ },"node_modules/ms": {"version": "2.1.3","resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz","integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==","license": "MIT"},++ "node_modules/napi-build-utils": {++ "version": "2.0.0",++ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",++ "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==",++ "dev": true,++ "license": "MIT"++ },"node_modules/negotiator": {"version": "0.6.3","resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",@@@ -679,6 +909,19 @@"node": ">= 0.6"}},++ "node_modules/node-abi": {++ "version": "3.96.0",++ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz",++ "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "semver": "^7.3.5"++ },++ "engines": {++ "node": ">=10"++ }++ },"node_modules/object-inspect": {"version": "1.13.4","resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",@@@ -705,6 +948,16 @@"node": ">= 0.8"}},++ "node_modules/once": {++ "version": "1.4.0",++ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",++ "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",++ "dev": true,++ "license": "ISC",++ "dependencies": {++ "wrappy": "1"++ }++ },"node_modules/parseurl": {"version": "1.3.3","resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",@@@ -722,6 +975,34 @@"dev": true,"license": "MIT"},++ "node_modules/prebuild-install": {++ "version": "7.1.3",++ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",++ "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==",++ "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "detect-libc": "^2.0.0",++ "expand-template": "^2.0.3",++ "github-from-package": "0.0.0",++ "minimist": "^1.2.3",++ "mkdirp-classic": "^0.5.3",++ "napi-build-utils": "^2.0.0",++ "node-abi": "^3.3.0",++ "pump": "^3.0.0",++ "rc": "^1.2.7",++ "simple-get": "^4.0.0",++ "tar-fs": "^2.0.0",++ "tunnel-agent": "^0.6.0"++ },++ "bin": {++ "prebuild-install": "bin.js"++ },++ "engines": {++ "node": ">=10"++ }++ },"node_modules/proxy-addr": {"version": "2.0.8","resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz",@@@ -740,6 +1021,17 @@"url": "https://opencollective.com/express"}},++ "node_modules/pump": {++ "version": "3.0.4",++ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",++ "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "end-of-stream": "^1.1.0",++ "once": "^1.3.1"++ }++ },"node_modules/qs": {"version": "6.16.0","resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz",@@@ -783,6 +1075,37 @@"node": ">= 0.8"}},++ "node_modules/rc": {++ "version": "1.2.8",++ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",++ "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",++ "dev": true,++ "license": "(BSD-2-Clause OR MIT OR Apache-2.0)",++ "dependencies": {++ "deep-extend": "^0.6.0",++ "ini": "~1.3.0",++ "minimist": "^1.2.0",++ "strip-json-comments": "~2.0.1"++ },++ "bin": {++ "rc": "cli.js"++ }++ },++ "node_modules/readable-stream": {++ "version": "3.6.2",++ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",++ "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "inherits": "^2.0.3",++ "string_decoder": "^1.1.1",++ "util-deprecate": "^1.0.1"++ },++ "engines": {++ "node": ">= 6"++ }++ },"node_modules/safe-buffer": {"version": "5.2.1","resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",@@@ -946,6 +1269,53 @@"url": "https://github.com/sponsors/ljharb"}},++ "node_modules/simple-concat": {++ "version": "1.0.1",++ "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",++ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",++ "dev": true,++ "funding": [++ {++ "type": "github",++ "url": "https://github.com/sponsors/feross"++ },++ {++ "type": "patreon",++ "url": "https://www.patreon.com/feross"++ },++ {++ "type": "consulting",++ "url": "https://feross.org/support"++ }++ ],++ "license": "MIT"++ },++ "node_modules/simple-get": {++ "version": "4.0.1",++ "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",++ "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",++ "dev": true,++ "funding": [++ {++ "type": "github",++ "url": "https://github.com/sponsors/feross"++ },++ {++ "type": "patreon",++ "url": "https://www.patreon.com/feross"++ },++ {++ "type": "consulting",++ "url": "https://feross.org/support"++ }++ ],++ "license": "MIT",++ "dependencies": {++ "decompress-response": "^6.0.0",++ "once": "^1.3.1",++ "simple-concat": "^1.0.0"++ }++ },"node_modules/statuses": {"version": "2.0.2","resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",@@@ -956,6 +1326,56 @@"node": ">= 0.8"}},++ "node_modules/string_decoder": {++ "version": "1.3.0",++ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",++ "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "safe-buffer": "~5.2.0"++ }++ },++ "node_modules/strip-json-comments": {++ "version": "2.0.1",++ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",++ "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",++ "dev": true,++ "license": "MIT",++ "engines": {++ "node": ">=0.10.0"++ }++ },++ "node_modules/tar-fs": {++ "version": "2.1.5",++ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz",++ "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "chownr": "^1.1.1",++ "mkdirp-classic": "^0.5.2",++ "pump": "^3.0.0",++ "tar-stream": "^2.1.4"++ }++ },++ "node_modules/tar-stream": {++ "version": "2.2.0",++ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",++ "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",++ "dev": true,++ "license": "MIT",++ "dependencies": {++ "bl": "^4.0.3",++ "end-of-stream": "^1.4.1",++ "fs-constants": "^1.0.0",++ "inherits": "^2.0.3",++ "readable-stream": "^3.1.1"++ },++ "engines": {++ "node": ">=6"++ }++ },"node_modules/toidentifier": {"version": "1.0.1","resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",@@@ -966,6 +1386,19 @@"node": ">=0.6"}},++ "node_modules/tunnel-agent": {++ "version": "0.6.0",++ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",++ "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",++ "dev": true,++ "license": "Apache-2.0",++ "dependencies": {++ "safe-buffer": "^5.0.1"++ },++ "engines": {++ "node": "*"++ }++ },"node_modules/type-is": {"version": "1.6.18","resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",@@@ -990,6 +1423,13 @@"node": ">= 0.8"}},++ "node_modules/util-deprecate": {++ "version": "1.0.2",++ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",++ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",++ "dev": true,++ "license": "MIT"++ },"node_modules/utils-merge": {"version": "1.0.1","resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",@@@ -1009,6 +1449,13 @@"engines": {"node": ">= 0.8"}++ },++ "node_modules/wrappy": {++ "version": "1.0.2",++ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",++ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",++ "dev": true,++ "license": "ISC"}}}@@@ -7,7 +7,14 @@".": "./index.js","./package.json": "./package.json","./account-switcher": "./account-switcher.js",-- "./analytics": "./analytics.js",++ "./analytics": "./analytics/index.js",++ "./analytics/event": "./analytics/event.js",++ "./analytics/privacy": "./analytics/privacy.js",++ "./analytics/prune-cli": "./analytics/prune-cli.js",++ "./analytics/retention": "./analytics/retention.js",++ "./analytics/schema": "./analytics/schema.js",++ "./analytics/tracker": "./analytics/tracker.js",++ "./analytics/event.v1.json": "./docs/schemas/analytics-event.v1.json","./app-icon": "./app-icon.js","./auth-client": "./auth-client.js","./brand": "./brand.js",@@@ -43,6 +50,8 @@},"files": ["*.js",++ "analytics/",++ "docs/schemas/","scripts/","CHANGELOG.md"],@@@ -62,6 +71,7 @@"jsonwebtoken": "^9.0.0"},"devDependencies": {++ "better-sqlite3": "^11.10.0","express": "^4.22.3"}}nnew file mode 100644@@@ -0,0 +1,166 @@++'use strict';++/**++ * The analytics-prune command (analytics/prune-cli.js), as Live, Tools and Network wire it:++ * - a dry run (with and without --scrub) changes nothing;++ * - --apply needs --backup <new file|dir> or --no-backup, refuses an existing target and days > 30;++ * - the backup is verified and owner-only (it can hold pre-ADR IPs), then prune + scrub run and the++ * rollup totals are unchanged;++ * - several databases (Tools' apps) back up into a directory; a service's defaults, extra options and++ * path options (Network) are honoured.++ *++ * node test/analytics-cli.test.js++ */++const assert = require('assert');++const crypto = require('crypto');++const fs = require('fs');++const os = require('os');++const path = require('path');++const Database = require('better-sqlite3');++const { AnalyticsTracker, retention } = require('openvibe-shared/analytics');++const { sqlTime } = require('openvibe-shared/analytics/tracker');++const cli = require('openvibe-shared/analytics/prune-cli');++++let failures = 0;++async function check(name, fn) {++ try { await fn(); console.log(' ✓', name); }++ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }++}++++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-shared-cli-'));++const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';++const NETWORK = { paramPrefixes: ['avatar', 'anon', 'projects'], pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']] };++const out = [];++const log = (l) => out.push(l);++const main = (argv, extra = {}) => cli.main(argv, { Database, log, ...extra });++const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');++++function dumpAll(file) {++ const db = new Database(file, { readonly: true });++ try {++ return db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all()++ .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');++ } finally { db.close(); }++}++++/** A legacy database: 10 rows past the cutoff, 3 inside it (edge rows 2 min clear of the CLI's own clock). */++function legacyDb(file) {++ fs.mkdirSync(path.dirname(file), { recursive: true });++ const db = new Database(file);++ db.pragma('journal_mode = WAL');++ db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT); INSERT INTO users VALUES (7, 'keepme');");++ new AnalyticsTracker(db, 'live', { timers: false }).destroy();++ const nowMs = Date.now() + 120000;++ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)++ VALUES ('live', 'pageview', ?, 'GET', 200, ?, ?, ?, ?, ?, ?, ?)`);++ for (let i = 0; i < 10; i++) ins.run(`/@old${i}?x=${i}`, 7, 'tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/x?y=1', sqlTime(nowMs - (40 + i) * 86400000));++ ins.run('/internal/users/by-username/keepme', null, null, '127.0.0.1', null, 'node', '', sqlTime(nowMs - 3 * 86400000));++ ins.run('/avatar/keepme?s=96', 9, 'abcdef0123456789', '198.51.100.52', 'Oslo', CHROME, 'https://www.reddit.com/r/a', sqlTime(nowMs - 2 * 86400000));++ ins.run('/vod/12345?t=9', null, null, '198.51.100.53', null, CHROME, '', sqlTime(nowMs - 1 * 86400000));++ db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')++ .run('live', '2026-07-01', 100, 40, 30, 10, 2, JSON.stringify([{ path: '/@keepme', cnt: 3 }, { path: '/internal/users/by-username/keepme', cnt: 2 }]), JSON.stringify([{ referer: 'https://www.google.com/search?q=keepme', cnt: 4 }]));++ db.pragma('wal_checkpoint(TRUNCATE)');++ db.close();++ return file;++}++++(async () => {++ await check('argument parsing: defaults, repeatable --db, extra options, refusals', () => {++ const a = cli.parseArgs(['--db', 'a.db', '--db', 'b.db', '--app', 'yt', '--scrub'], { '--app': 'apps' });++ assert.deepStrictEqual([a.dbs, a.db, a.apps, a.scrub, a.apply, a.days, a.batch], [['a.db', 'b.db'], 'a.db', ['yt'], true, false, 30, 5000]);++ assert.throws(() => cli.parseArgs(['--days', '31']), /1 to 30/);++ assert.throws(() => cli.parseArgs(['--batch', '0']), /batch/);++ assert.throws(() => cli.parseArgs(['--backup', 'x', '--no-backup']), /exclusive/);++ assert.throws(() => cli.parseArgs(['--app', 'yt']), /unknown option --app/);++ assert.throws(() => cli.parseArgs(['--db']), /needs a value/);++ assert.deepStrictEqual(cli.namedTargets(['/srv/tools/apps/yt/data/analytics.db', '/x/analytics.db', '/y/analytics.db']).map((t) => t.name), ['yt', 'analytics', 'analytics-2']);++ });++++ await check('dry run (with and without --scrub) changes nothing; the default database is the service\'s', async () => {++ const file = legacyDb(path.join(tmp, 'dry', 'analytics.db'));++ const before = { hash: sha(file), dump: dumpAll(file) };++ assert.strictEqual(await main([], { defaultDb: file }), 0);++ assert.strictEqual(await main(['--scrub', '--days', '7'], { defaultDb: (args, env) => env.TEST_ANALYTICS_DB, env: { TEST_ANALYTICS_DB: file } }), 0);++ assert.strictEqual(sha(file), before.hash);++ assert.strictEqual(dumpAll(file), before.dump);++ assert.ok(out.some((l) => /dry run: nothing changed/.test(l)));++ assert.ok(out.some((l) => /prune\s+10 rows/.test(l)), out.join('\n'));++ assert.strictEqual(await main([]), 2, 'no database at all');++ assert.strictEqual(await main(['--db', path.join(tmp, 'missing.db')]), 2);++ assert.strictEqual(await main(['--help'], { usage: 'custom usage' }), 0);++ assert.strictEqual(out[out.length - 1], 'custom usage');++ await assert.rejects(cli.main([], {}), /Database/);++ });++++ await check('--apply refuses without a backup choice, an existing target, or days > 30', async () => {++ const file = legacyDb(path.join(tmp, 'refuse.db'));++ const before = dumpAll(file);++ assert.strictEqual(await main(['--db', file, '--apply']), 2);++ const existing = path.join(tmp, 'exists.bak');++ fs.writeFileSync(existing, 'x');++ assert.strictEqual(await main(['--db', file, '--apply', '--backup', existing]), 2);++ assert.strictEqual(await main(['--db', file, '--apply', '--no-backup', '--days', '31']), 2);++ assert.strictEqual(await main(['--db', file, '--apply', '--no-backup', '--backup', existing]), 2);++ assert.strictEqual(dumpAll(file), before);++ assert.strictEqual(fs.readFileSync(existing, 'utf8'), 'x');++ });++++ await check('--apply --scrub --backup: owner-only verified backup, prune + scrub, rollups equal, other tables kept', async () => {++ const file = legacyDb(path.join(tmp, 'apply.db'));++ const src = new Database(file, { readonly: true });++ const totals = retention.rollupTotals(src);++ src.close();++ const bak = path.join(tmp, 'apply.backup.db');++ assert.strictEqual(await main(['--db', file, '--apply', '--scrub', '--backup', bak, '--batch', '2'], NETWORK), 0, out.join('\n'));++ assert.strictEqual(fs.statSync(bak).mode & 0o777, 0o600);++ const b = new Database(bak, { readonly: true });++ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13, 'backup has every row');++ b.close();++ const db = new Database(file, { readonly: true });++ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all(),++ ['/internal/users/:param/:username', '/avatar/:param', '/vod/:param']);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events WHERE ip IS NOT NULL OR user_id IS NOT NULL OR city IS NOT NULL').pluck().get(), 0);++ assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()),++ [{ path: '/@:user', cnt: 3 }, { path: '/internal/users/:param/:username', cnt: 2 }]);++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ assert.strictEqual(db.prepare('SELECT username FROM users WHERE id = 7').pluck().get(), 'keepme');++ db.close();++ assert.ok(out.some((l) => /mode 0600, quick_check ok/.test(l)));++ // Prune-only with an explicit --no-backup works too (nothing left to prune).++ assert.strictEqual(await main(['--db', file, '--apply', '--no-backup', '--no-vacuum']), 0);++ });++++ await check('several databases (Tools apps): discovery hook, extra option, backups into a directory', async () => {++ const apps = path.join(tmp, 'apps');++ const yt = legacyDb(path.join(apps, 'yt', 'data', 'analytics.db'));++ const food = legacyDb(path.join(apps, 'food', 'data', 'analytics.db'));++ const tools = {++ options: { '--app': 'apps' },++ targets: (args) => (args.apps.length ? args.apps : ['food', 'yt']).map((name) => ({ name, file: path.join(apps, name, 'data', 'analytics.db') })),++ };++ const hashes = [sha(yt), sha(food)];++ assert.strictEqual(await main(['--scrub'], tools), 0);++ assert.strictEqual(await main(['--app', 'yt'], tools), 0);++ assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'dry run changed a file');++ const bdir = path.join(tmp, 'backups');++ assert.strictEqual(await main(['--apply', '--scrub', '--backup', bdir], tools), 0, out.join('\n'));++ for (const [name, file] of [['yt', yt], ['food', food]]) {++ const bfile = path.join(bdir, `${name}.analytics.db`);++ assert.strictEqual(fs.statSync(bfile).mode & 0o777, 0o600);++ const b = new Database(bfile, { readonly: true });++ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13);++ b.close();++ const db = new Database(file, { readonly: true });++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 3);++ assert.strictEqual(db.prepare('SELECT SUM(pageviews) FROM analytics_daily').pluck().get(), 100);++ db.close();++ }++ assert.strictEqual(await main(['--apply', '--backup', bdir], tools), 2, 'existing backup targets are refused');++ const afile = path.join(tmp, 'a-file');++ fs.writeFileSync(afile, 'x');++ assert.strictEqual(await main(['--apply', '--backup', afile], tools), 2, 'several databases need a directory');++ });++++ fs.rmSync(tmp, { recursive: true, force: true });++ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }++ console.log('\nanalytics prune CLI: all passed');++})();nnew file mode 100644@@@ -0,0 +1,402 @@++'use strict';++/**++ * ADR-021 analytics bounds (analytics/): what a tracked request may leave behind.++ * - no IP, user id, city, raw user agent, raw referer or query string anywhere in the database;++ * the session id is a rotating id, never the user id;++ * - paths become route templates (the Express route when matched, else the normaliser), with a++ * service's extra parameter words and path rules (Network's, as options);++ * - unique visitors come from the day's salted hashes, which are deleted once the day is rolled up;++ * - `Sec-GPC: 1` / `DNT: 1` requests are not recorded at all: no row, hash, session or counter;++ * - every row the tracker writes is a valid analytics/event.v1 event; a pre-ADR row is not.++ *++ * node test/analytics-privacy.test.js++ */++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const express = require('express');++const Database = require('better-sqlite3');++const analytics = require('openvibe-shared/analytics');++const { sqlTime } = require('openvibe-shared/analytics/tracker');++const EVENT_V1 = require('openvibe-shared/analytics/event.v1.json');++++const { AnalyticsTracker, privacy, event } = analytics;++++let failures = 0;++async function check(name, fn) {++ try { await fn(); console.log(' ✓', name); }++ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }++}++++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-shared-analytics-'));++const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };++const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';++const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';++const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';++// Network's route shapes, given as options (they were hard-coded in its server/analytics/network.js).++const NETWORK = { paramPrefixes: ['avatar', 'anon', 'projects'], pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']] };++++/** Every value in every table, for "is this anywhere in the file" checks. */++function dumpAll(db) {++ const tables = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all();++ return tables.map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');++}++++async function serve(app, fn) {++ const server = app.listen(0, '127.0.0.1');++ await new Promise((r) => server.once('listening', r));++ try { return await fn(`http://127.0.0.1:${server.address().port}`); }++ finally { await new Promise((r) => server.close(r)); }++}++const settle = () => new Promise((r) => setTimeout(r, 50));++++(async () => {++ // ── Reducers ─────────────────────────────────────────────++ await check('normalisePath strips queries and replaces ids, slugs and usernames', () => {++ const n = privacy.normalisePath;++ const cases = {++ '/': '/',++ '': '/',++ '/vods': '/vods',++ '/vod/8a7c2f10-1b2c-4d5e-8f90-123456789abc?t=30': '/vod/:param',++ '/api/vods/123/comments?page=2&token=abc': '/api/vods/:id/comments',++ '/@JapaneseOldGuy': '/@:user',++ '/@alex/main-stage?stream=77': '/@:user/:param',++ '/p/k3yF00bar': '/p/:param',++ '/recap/some-words': '/recap/:param',++ '/u/alex/settings': '/u/:param/settings',++ '/api/users/alex': '/api/users/:param',++ '/watch/dQw4w9WgXcQ': '/watch/:param',++ '/files/3f786850e387550fdab836ed7e6dc881de23001b': '/files/:param',++ '/api/things/usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W': '/api/things/:id',++ '/x/01J8ZQ4K7M2N3P4Q5R6S7T8V9W': '/x/:id',++ '/verify/alex%40example.com': '/verify/:param',++ '/share/[email protected]/x': '/share/:param/x',++ '/maps/@40.7128,-74.0060,12z': '/maps/@:user',++ '/dashboard/settings': '/dashboard/settings',++ '/wp-login.php': '/wp-login.php',++ '/stream-2024-09-23-highlights': '/:id',++ '/hello world': '/:id',++ 'https://openvibe.live/@alex?x=1#y': '/@:user',++ '/one/two/three/four/five/six/seven/eight/nine': '/one/two/three/four/five/six/seven/eight/*',++ // Tools' shapes.++ '/api/info?url=https://youtube.com/watch?v=abc': '/api/info',++ '/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events': '/api/jobs/:param/events',++ '/recipe/chicken-tikka': '/recipe/:param',++ '/place/@52.37,4.89,14z': '/place/@:user',++ };++ for (const [raw, want] of Object.entries(cases)) assert.strictEqual(n(raw), want, raw);++ for (const t of ['/api/vods/:id/comments', '/@:user/:param', '/api/analytics/channel/:username', '/p/:id']) {++ assert.strictEqual(n(t), t);++ assert.strictEqual(n(n(t)), n(t));++ }++ assert.strictEqual(n('/dishes/tacos', { paramPrefixes: ['dishes'] }), '/dishes/:param');++ });++++ await check('an over-long template drops whole segments, stays ≤ 200 chars and idempotent', () => {++ const long = '/' + Array.from({ length: 8 }, (_, i) => 'abcdefghijklmnopqrstuvwxyzabcd' + 'xy'[i % 2]).join('/');++ const t = privacy.normalisePath(long);++ assert.ok(t.length <= privacy.MAX_TEMPLATE_LENGTH, t.length);++ assert.ok(t.endsWith('/*'), t);++ assert.ok(t.split('/').slice(1, -1).every((s) => /^abcdefghijklmnopqrstuvwxyzabcd[xy]$/.test(s)), t);++ assert.strictEqual(privacy.normalisePath(t), t);++ assert.ok(new RegExp(EVENT_V1.properties.route.pattern, 'u').test(t));++ });++++ await check('service options: Network paramPrefixes and pathRules become parameters', () => {++ const opts = privacy.pathOptions(NETWORK);++ const t = (p) => privacy.normalisePath(p, opts);++ const cases = {++ '/avatar/alex?s=96': '/avatar/:param',++ '/api/auth/anon/k3yF00barBaz?x=1': '/api/auth/anon/:param',++ '/internal/users/by-username/alex': '/internal/users/:param/:username',++ '/api/v1/projects/my-app/apps': '/api/v1/projects/:param/apps',++ '/oauth/authorize?client_id=live&state=abc': '/oauth/authorize',++ '/reset-password?token=abc': '/reset-password',++ '/api/admin/users/42/role': '/api/admin/users/:param/role',++ };++ for (const [raw, want] of Object.entries(cases)) assert.strictEqual(t(raw), want, raw);++ for (const want of Object.values(cases)) assert.strictEqual(t(want), want, `idempotent: ${want}`);++ // Without the rule the shared normaliser keeps the name; the defaults stay in the merged set.++ assert.strictEqual(privacy.normalisePath('/internal/users/by-username/alex'), '/internal/users/:param/alex');++ assert.ok(opts.paramPrefixes.has('vod') && opts.paramPrefixes.has('avatar'));++ assert.strictEqual(privacy.pathOptions({}), undefined);++ assert.throws(() => privacy.pathOptions({ pathRules: [['by-username', 'x']] }), /RegExp/);++ });++++ await check('referer → origin, user agent → class, country → ISO code', () => {++ assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=who+is+alex'), 'https://www.google.com');++ assert.strictEqual(privacy.refererOrigin('http://localhost:3000/@alex'), 'http://localhost:3000');++ assert.strictEqual(privacy.refererOrigin('android-app://com.foo/'), null);++ assert.strictEqual(privacy.refererOrigin('not a url'), null);++ assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');++ assert.strictEqual(privacy.uaClass(FIREFOX), 'firefox/linux/desktop');++ assert.strictEqual(privacy.uaClass('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'), 'bot:googlebot');++ assert.strictEqual(privacy.uaClass(''), 'none');++ assert.strictEqual(privacy.uaClass('chrome/windows/desktop'), 'chrome/windows/desktop');++ assert.strictEqual(privacy.uaClass('bot:googlebot'), 'bot:googlebot');++ assert.strictEqual(privacy.countryCode('de'), 'DE');++ assert.strictEqual(privacy.countryCode('Berlin'), null);++ });++++ await check('optedOut: Sec-GPC: 1 or DNT: 1, nothing else', () => {++ assert.strictEqual(privacy.optedOut({ 'sec-gpc': '1' }), true);++ assert.strictEqual(privacy.optedOut({ dnt: '1' }), true);++ assert.strictEqual(privacy.optedOut({ dnt: ' 1 ' }), true);++ assert.strictEqual(privacy.optedOut({ 'sec-gpc': ['0', '1'] }), true);++ for (const h of [{}, { dnt: '0' }, { 'sec-gpc': '0' }, { dnt: 'yes' }, { 'sec-gpc': '' }, null, undefined]) {++ assert.strictEqual(privacy.optedOut(h), false, JSON.stringify(h));++ }++ assert.strictEqual(analytics.optedOut, privacy.optedOut);++ });++++ // ── A tracked request ────────────────────────────────────++ await check('a tracked request stores no IP, user id, city, raw UA, raw referer or query', async () => {++ const db = newDb('track.db');++ let clock = Date.parse('2026-09-23T10:15:00Z');++ const tracker = new AnalyticsTracker(db, 'live', { timers: false, now: () => clock });++ const app = express();++ app.set('trust proxy', true);++ app.use(tracker.middleware());++ app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT, username: 'alex' }; next(); });++ const router = express.Router();++ router.get('/things/:thingId', (req, res) => res.json({ ok: true }));++ app.use('/api', router);++ app.get('*', (req, res) => res.send('<html></html>'));++ const hdr = (extra) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', 'cf-ipcity': 'Amsterdam', ...extra });++ await serve(app, async (base) => {++ await (await fetch(`${base}/api/things/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();++ await (await fetch(`${base}/@alex/main?stream=12`, { headers: hdr() })).text();++ await (await fetch(`${base}/vod/424242`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();++ await settle();++ });++ tracker.flush();++++ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();++ assert.strictEqual(rows.length, 3);++ for (const r of rows) {++ assert.strictEqual(r.ip, null);++ assert.strictEqual(r.user_id, null);++ assert.strictEqual(r.city, null);++ assert.ok(!/[?#]/.test(r.path), r.path);++ assert.strictEqual(r.referer, 'https://www.google.com');++ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id), r.session_id);++ assert.strictEqual(r.country, 'NL');++ assert.deepStrictEqual(event.checkRow(r), [], JSON.stringify(r));++ }++ assert.deepStrictEqual(rows.map((r) => r.path), ['/api/things/:thingId', '/@:user/:param', '/vod/:param']);++ assert.deepStrictEqual(rows.map((r) => r.user_agent), ['chrome/windows/desktop', 'chrome/windows/desktop', 'firefox/linux/desktop']);++ assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0, 0]);++ assert.deepStrictEqual(rows.map((r) => r.event_type), ['api_call', 'pageview', 'pageview']);++ assert.strictEqual(rows[0].session_id, rows[1].session_id, 'same visitor, same session');++ assert.notStrictEqual(rows[0].session_id, rows[2].session_id);++++ const everything = dumpAll(db);++ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'Amsterdam', 'supersecret', 'secret-query', '98765', '424242', 'Mozilla/5.0', '"alex"']) {++ assert.ok(!everything.includes(needle), `found ${needle} in analytics.db`);++ }++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);++ assert.ok(!db.prepare("SELECT name FROM sqlite_master WHERE name IN ('idx_analytics_events_ip', 'idx_analytics_events_user')").get());++++ // Rollups: two distinct (ip, ua) visitors, one of them signed in.++ tracker.aggregate();++ const hourly = db.prepare("SELECT * FROM analytics_hourly WHERE hour = '2026-09-23 10:00:00'").get();++ assert.strictEqual(hourly.pageviews, 2);++ assert.strictEqual(hourly.api_calls, 1);++ assert.strictEqual(hourly.unique_visitors, 2);++ assert.strictEqual(hourly.unique_users, 1);++ const daily = db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();++ assert.strictEqual(daily.unique_visitors, 2);++ assert.strictEqual(daily.unique_users, 1);++ assert.strictEqual(daily.new_users, null);++ assert.ok(JSON.parse(daily.top_paths).every((p) => !/\d{3,}/.test(p.path)));++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 2);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 1);++++ // Next day: the finished day is rolled up for the last time, then its hashes and salt go.++ clock = Date.parse('2026-09-24T00:20:00Z');++ tracker.aggregate();++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 0);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 0);++ assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2, 'uniques survive the hashes');++ clock = Date.parse('2026-09-24T01:20:00Z');++ tracker.aggregate();++ assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2, 'a recompute never lowers uniques');++++ // The same visitor gets a new session id (and hash) on a new day.++ assert.strictEqual(tracker.record({ headers: { 'user-agent': CHROME }, ip: '203.0.113.77', method: 'GET' }, { statusCode: 200 }, '/', 3), true);++ tracker.flush();++ const last = db.prepare('SELECT session_id FROM analytics_events ORDER BY id DESC LIMIT 1').pluck().get();++ assert.notStrictEqual(last, rows[0].session_id);++++ // Dashboards keep the pre-ADR shapes.++ const st = tracker.getStats({ days: 30 });++ for (const k of ['summary', 'realtime', 'daily', 'hourly', 'topPages', 'authBreakdown', 'visitorTypes', 'authTrend']) assert.ok(k in st, k);++ assert.ok(tracker.getStats({ hours: 6 }).timeBuckets);++ const bots = tracker.getBotAnalysis(30);++ for (const k of ['topBotIPs', 'botTrend', 'botTypes', 'suspiciousIPs']) assert.ok(Array.isArray(bots[k]), k);++ assert.ok(tracker.getOverview(30).services.length >= 1);++ tracker.destroy();++ db.close();++ });++++ await check('Sec-GPC: 1 and DNT: 1 requests leave nothing: no row, hash, session or rate counter', async () => {++ const db = newDb('optout.db');++ const clock = Date.parse('2026-09-23T10:15:00Z');++ const tracker = new AnalyticsTracker(db, 'live', { timers: false, now: () => clock });++ const app = express();++ app.set('trust proxy', true);++ app.use(tracker.middleware());++ app.get('*', (req, res) => res.send('ok'));++ const hdr = (extra) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.5', 'cf-ipcountry': 'FR', ...extra });++ await serve(app, async (base) => {++ await (await fetch(`${base}/@gpc-user`, { headers: hdr({ 'sec-gpc': '1' }) })).text();++ await (await fetch(`${base}/@dnt-user`, { headers: hdr({ dnt: '1' }) })).text();++ for (let i = 0; i < 70; i++) await (await fetch(`${base}/flood`, { headers: hdr({ 'sec-gpc': '1', 'x-forwarded-for': '203.0.113.6' }) })).text();++ await (await fetch(`${base}/counted`, { headers: hdr({ dnt: '0', 'sec-gpc': '0' }) })).text();++ await settle();++ });++ tracker.flush();++ tracker.aggregate();++ const rows = db.prepare('SELECT * FROM analytics_events').all();++ assert.deepStrictEqual(rows.map((r) => r.path), ['/counted'], 'only the request without an opt-out is recorded');++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 1);++ assert.strictEqual(tracker._sessions.size, 1, 'no session for an opted-out visitor');++ assert.ok(!tracker._rates.has('203.0.113.6'), 'no rate counter for an opted-out client');++ const hourly = db.prepare('SELECT pageviews, unique_visitors FROM analytics_hourly').get();++ assert.deepStrictEqual({ ...hourly }, { pageviews: 1, unique_visitors: 1 });++ // Direct calls honour it too.++ assert.strictEqual(tracker.record({ headers: { 'user-agent': CHROME, dnt: '1' }, ip: '192.0.2.9', method: 'GET' }, { statusCode: 200 }, '/x', 1), false);++ assert.strictEqual(tracker.trackEvent('share_clicked', { headers: { 'sec-gpc': '1' }, path: '/@bob' }), false);++ assert.strictEqual(tracker.trackEvent('share_clicked', { headers: { 'user-agent': CHROME }, path: '/@bob' }), true);++ tracker.flush();++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 2);++ const all = dumpAll(db);++ for (const needle of ['gpc-user', 'dnt-user', 'flood', '203.0.113.', '192.0.2.9']) assert.ok(!all.includes(needle), needle);++ tracker.destroy();++ db.close();++ });++++ await check('Network options on the tracker: unmatched paths use its rules', async () => {++ const db = newDb('network.db');++ const tracker = new AnalyticsTracker(db, 'openvibe-network', { timers: false, ...NETWORK });++ const app = express();++ app.use(tracker.middleware());++ app.get('/avatar/:username', (req, res) => res.send('img'));++ app.use('/internal', (req, res) => res.status(403).end()); // refused before routing++ app.use('/api/auth/anon', (req, res) => res.status(404).end());++ await serve(app, async (base) => {++ for (const p of ['/avatar/alexsecretname?s=96', '/internal/users/by-username/carolsecretname', '/api/auth/anon/0123456789abcdefanon']) {++ await (await fetch(base + p, { headers: { 'user-agent': CHROME } })).arrayBuffer();++ }++ await settle();++ });++ tracker.flush();++ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY id').pluck().all(),++ ['/avatar/:username', '/internal/users/:param/:username', '/api/auth/anon/:param']);++ const all = dumpAll(db);++ for (const needle of ['alexsecretname', 'carolsecretname', '0123456789abcdefanon']) assert.ok(!all.includes(needle), needle);++ tracker.destroy();++ db.close();++ });++++ await check('rate check keeps IPs in memory only and still flags floods; custom events are reduced', () => {++ const db = newDb('rate.db');++ const clock = Date.parse('2026-09-23T12:00:30Z');++ const tracker = new AnalyticsTracker(db, 'live', { timers: false, now: () => clock });++ for (let i = 0; i < 70; i++) tracker.record({ headers: { 'user-agent': CHROME }, ip: '192.0.2.1', method: 'GET' }, { statusCode: 200 }, '/api/x', 1);++ tracker.flush();++ assert.ok(db.prepare("SELECT COUNT(*) FROM analytics_events WHERE bot_type = 'rate_limit'").pluck().get() >= 10);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);++ assert.ok(!dumpAll(db).includes('192.0.2.1'));++ tracker.trackEvent('custom', {++ ip: '192.0.2.2', user_id: 7, city: 'Paris', path: '/@bob?x=1', referer: 'https://a.example/x?y=1', user_agent: CHROME,++ session_id: 'user-7', method: 'post', status_code: 999, response_time_ms: -4, browser: 'MyBrowser 1.0', bot_type: 'Weird Type',++ });++ tracker.flush();++ const e = db.prepare("SELECT * FROM analytics_events WHERE event_type = 'custom'").get();++ assert.strictEqual(e.path, '/@:user');++ assert.strictEqual(e.referer, 'https://a.example');++ assert.strictEqual(e.session_id, null);++ assert.strictEqual(e.user_agent, 'chrome/windows/desktop');++ assert.strictEqual(e.method, 'POST');++ assert.strictEqual(e.status_code, null);++ assert.strictEqual(e.response_time_ms, null);++ assert.strictEqual(e.browser, 'chrome');++ assert.strictEqual(e.bot_type, null);++ assert.deepStrictEqual(event.checkRow(e), []);++ assert.ok(!dumpAll(db).includes('Paris') && !dumpAll(db).includes('192.0.2.2'));++ assert.throws(() => tracker.trackEvent('has spaces'), TypeError);++ assert.throws(() => tracker.trackEvent('x'.repeat(65)), TypeError);++ for (const r of db.prepare('SELECT * FROM analytics_events').all()) assert.deepStrictEqual(event.checkRow(r), []);++ tracker.destroy();++ db.close();++ });++++ // ── analytics/event.v1 ───────────────────────────────────++ await check('event.v1: the schema file is what the package exports, and a pre-ADR row fails it', () => {++ assert.strictEqual(event.EVENT_V1, EVENT_V1);++ assert.strictEqual(EVENT_V1.$id, 'https://openvibe.network/contracts/analytics/event.v1.json');++ assert.strictEqual(EVENT_V1.additionalProperties, false);++ for (const banned of ['ip', 'user_id', 'city', 'subject_id', 'user_agent', 'referer', 'url', 'region']) {++ assert.ok(!(banned in EVENT_V1.properties), `${banned} is not an event.v1 field`);++ }++ for (const f of ['event', 'service', 'route', 'session_id', 'country', 'ua_class', 'timestamp']) assert.ok(EVENT_V1.required.includes(f), f);++ const good = {++ event: 'pageview', service: 'live', route: '/@:user/:param', method: 'GET', status: 200, duration_ms: 12,++ session_id: '0123456789abcdef', country: 'NL', ua_class: 'chrome/windows/desktop', device: 'desktop', browser: 'chrome',++ os: 'windows', referer_origin: 'https://www.google.com', is_bot: false, bot_type: null, authenticated: true, timestamp: '2026-09-23T10:15:00Z',++ };++ assert.deepStrictEqual(event.validateEvent(good), []);++ const bad = (patch) => event.validateEvent({ ...good, ...patch });++ assert.deepStrictEqual(bad({ ip: '203.0.113.1' }), ['ip: not allowed']);++ assert.deepStrictEqual(bad({ user_id: 42 }), ['user_id: not allowed']);++ assert.ok(bad({ route: '/vod/12345?t=1' }).length);++ assert.ok(bad({ route: '/@alex smith' }).length);++ assert.ok(bad({ referer_origin: 'https://www.google.com/search?q=x' }).length);++ assert.ok(bad({ ua_class: CHROME }).length);++ assert.ok(bad({ session_id: '42' }).length);++ assert.ok(bad({ country: 'Amsterdam' }).length);++ assert.ok(bad({ timestamp: '2026-09-23 10:15:00' }).length);++ assert.ok(bad({ status: 99 }).length);++ const { session_id, ...noSession } = good; // eslint-disable-line no-unused-vars++ assert.deepStrictEqual(event.validateEvent(noSession), ['session_id: required']);++++ const legacy = {++ service: 'live', event_type: 'pageview', path: '/@alex?x=1', method: 'GET', status_code: 200, response_time_ms: 5,++ user_id: 42, session_id: 'user-42', ip: '198.51.100.7', country: null, city: 'Lyon', user_agent: CHROME,++ referer: 'https://t.co/abc?x=1', is_bot: 0, bot_type: null, device_type: 'desktop', browser: 'chrome', os: 'windows',++ authenticated: 0, created_at: sqlTime(Date.parse('2026-09-23T10:15:00Z')),++ };++ const problems = event.checkRow(legacy).map((p) => p.split(':')[0]);++ for (const f of ['ip', 'user_id', 'city', 'route', 'session_id', 'ua_class', 'referer_origin']) assert.ok(problems.includes(f), `${f} flagged: ${problems}`);++ assert.strictEqual(event.toEvent(legacy).timestamp, '2026-09-23T10:15:00Z');++ });++++ await check('the pre-ADR export names are still there (1.x), backed by the ADR-021 code', () => {++ const root = require('openvibe-shared');++ assert.strictEqual(root.AnalyticsTracker, AnalyticsTracker);++ assert.strictEqual(root.classifyRequest, privacy.classifyRequest);++ assert.strictEqual(root.parseUserAgent, privacy.parseUserAgent);++ assert.strictEqual(root.BOT_USER_AGENTS, privacy.BOT_USER_AGENTS);++ assert.strictEqual(root.ANALYTICS_SCHEMA, analytics.schema.ANALYTICS_SCHEMA);++ assert.strictEqual(root.SUSPICIOUS_PATTERNS.highRequestRate, privacy.HIGH_REQUEST_RATE);++ assert.ok(Object.isFrozen(root.SUSPICIOUS_PATTERNS));++ assert.deepStrictEqual(privacy.classifyRequest({ headers: { 'user-agent': 'curl/8.5.0' } }).botType, 'known_crawler');++ // The old constructor call (db, service) still works; the prune timer is on by default.++ const db = newDb('compat.db');++ const t = new AnalyticsTracker(db, 'live');++ assert.ok(t.pruneJob && typeof t.pruneJob.run === 'function');++ t.destroy();++ assert.strictEqual(t.pruneJob, null);++ const off = new AnalyticsTracker(db, 'live', { retention: false });++ assert.strictEqual(off.pruneJob, null);++ off.destroy();++ db.close();++ });++++ fs.rmSync(tmp, { recursive: true, force: true });++ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }++ console.log('\nanalytics privacy: all passed');++})();nnew file mode 100644@@@ -0,0 +1,160 @@++'use strict';++/**++ * ADR-021 retention (analytics/retention.js):++ * - prune deletes strictly-older raw rows in bounded batches and never touches rollups;++ * - schedulePrune (the tracker's timer, or a service's own job) prunes and reports;++ * - scrub rewrites legacy rows and rollup top lists without changing any rollup counter, with a++ * service's path options (Network's by-username rule), and touches no other table.++ *++ * node test/analytics-retention.test.js++ */++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const Database = require('better-sqlite3');++const { AnalyticsTracker, retention, event } = require('openvibe-shared/analytics');++const { sqlTime } = require('openvibe-shared/analytics/tracker');++++let failures = 0;++async function check(name, fn) {++ try { await fn(); console.log(' ✓', name); }++ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }++}++++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-shared-retention-'));++const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };++const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';++const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';++const NETWORK = { paramPrefixes: ['avatar', 'anon', 'projects'], pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']] };++++function dumpAll(db) {++ const tables = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all();++ return tables.map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');++}++++/** Rows as the pre-ADR tracker wrote them, around a 30-day cutoff from nowMs. */++function seedLegacy(db, nowMs) {++ new AnalyticsTracker(db, 'live', { timers: false }).destroy(); // schema++ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, response_time_ms, user_id, session_id, ip, city, user_agent, referer, is_bot, created_at)++ VALUES ('live', 'pageview', ?, 'GET', 200, 5, ?, ?, ?, ?, ?, ?, 0, ?)`);++ const at = (days, s = 0) => sqlTime(nowMs - days * 86400000 + s * 1000);++ const cutoffMs = nowMs - 30 * 86400000;++ const rows = [];++ for (let i = 0; i < 10; i++) rows.push(['/@old' + i, 5, 'eyJhbGciOi.tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/abc?x=1', at(45 + i)]);++ rows.push(['/@edge-older', 5, null, '198.51.100.50', null, CHROME, '', sqlTime(cutoffMs - 1000)]);++ rows.push(['/@edge-exact', 5, null, '198.51.100.51', null, CHROME, '', sqlTime(cutoffMs)]);++ rows.push(['/@edge-newer', null, null, '198.51.100.52', null, FIREFOX, '', sqlTime(cutoffMs + 1000)]);++ rows.push(['/vod/12345?t=9', 9, 'abcdef0123456789', '198.51.100.53', 'Oslo', FIREFOX, 'https://www.reddit.com/r/x/comments/1', at(2)]);++ for (const r of rows) ins.run(...r);++ const hr = db.prepare('INSERT INTO analytics_hourly (service, hour, pageviews, api_calls, unique_visitors, unique_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?)');++ const dy = db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');++ const tops = JSON.stringify([{ path: '/@alex', cnt: 3 }, { path: '/@bob', cnt: 2 }, { path: '/vods', cnt: 1 }]);++ const refs = JSON.stringify([{ referer: 'https://www.google.com/search?q=alex', cnt: 4 }, { referer: 'https://www.google.com/', cnt: 1 }]);++ hr.run('live', at(60).slice(0, 13) + ':00:00', 10, 4, 3, 1, tops, refs);++ dy.run('live', at(60).slice(0, 10), 100, 40, 30, 10, 2, tops, refs);++ dy.run('live', at(2).slice(0, 10), 50, 20, 15, 5, 1, tops, refs);++ db.prepare("INSERT INTO analytics_rate_tracking (ip, window_start, hit_count) VALUES ('198.51.100.99', 1, 1)").run();++}++++(async () => {++ await check('prune: strictly older than the cutoff, bounded batches, rollups untouched', async () => {++ const db = newDb('prune.db');++ const nowMs = Date.parse('2026-09-23T12:00:00Z');++ seedLegacy(db, nowMs);++ const totals = retention.rollupTotals(db);++ const bounded = await retention.pruneRawEvents(db, { days: 30, batchSize: 3, maxBatches: 2, now: () => nowMs });++ assert.strictEqual(bounded.deleted, 6);++ assert.strictEqual(bounded.complete, false);++ const out = await retention.pruneRawEvents(db, { days: 30, batchSize: 3, now: () => nowMs });++ assert.strictEqual(out.deleted, 5);++ assert.strictEqual(out.cutoff, '2026-08-24 12:00:00');++ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all(), ['/@edge-exact', '/@edge-newer', '/vod/12345?t=9']);++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_daily').pluck().get(), 2, 'old rollups stay');++ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);++ assert.strictEqual((await retention.pruneRawEvents(db, { days: 30, now: () => nowMs })).deleted, 0);++ assert.throws(() => retention.cutoffFor(31), /1 to 30/);++ assert.throws(() => retention.cutoffFor(0), /1 to 30/);++ db.close();++ });++++ await check('schedulePrune: run() prunes and logs, overlapping runs are skipped, days > 30 refused', async () => {++ const db = newDb('job.db');++ const nowMs = Date.parse('2026-09-23T12:00:00Z');++ seedLegacy(db, nowMs);++ const totals = retention.rollupTotals(db);++ const logs = [];++ const job = retention.schedulePrune(db, { initialDelayMs: 1e9, intervalMs: 1e9, label: 'Analytics:test', log: (l) => logs.push(l), now: () => nowMs });++ try {++ const [a, b] = await Promise.all([job.run(), job.run()]);++ assert.strictEqual(a.deleted, 11);++ assert.strictEqual(b, null, 'overlap skipped');++ assert.ok(/^\[Analytics:test\] pruned 11 raw events older than 2026-08-24 12:00:00$/.test(logs[0]), logs.join('\n'));++ } finally { job.stop(); }++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ assert.throws(() => retention.schedulePrune(db, { days: 90 }), /1 to 30/);++ // The tracker's own timer is the same job.++ const t = new AnalyticsTracker(db, 'live', { retention: { initialDelayMs: 1e9, intervalMs: 1e9, log: () => {} }, now: () => nowMs });++ assert.strictEqual((await t.pruneJob.run()).deleted, 0);++ t.destroy();++ db.close();++ });++++ await check('scrub: legacy rows and rollup top lists reduced, counters unchanged, rows now event.v1', async () => {++ const db = newDb('scrub.db');++ const nowMs = Date.parse('2026-09-23T12:00:00Z');++ seedLegacy(db, nowMs);++ await retention.pruneRawEvents(db, { days: 30, now: () => nowMs });++ const totals = retention.rollupTotals(db);++ const s = await retention.scrubEvents(db, { batchSize: 2 });++ assert.strictEqual(s.rows, 3);++ assert.deepStrictEqual(retention.scrubRollups(db), { hourly: 1, daily: 2 });++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY created_at').all();++ assert.deepStrictEqual(rows.map((x) => x.path), ['/@:user', '/@:user', '/vod/:param']);++ assert.deepStrictEqual(rows.map((x) => x.authenticated), [1, 0, 1]);++ assert.deepStrictEqual(rows.map((x) => x.session_id), [null, null, 'abcdef0123456789']);++ assert.deepStrictEqual(rows.map((x) => x.referer), ['', '', 'https://www.reddit.com']);++ assert.deepStrictEqual(rows.map((x) => x.user_agent), ['chrome/windows/desktop', 'firefox/linux/desktop', 'firefox/linux/desktop']);++ for (const x of rows) assert.deepStrictEqual(event.checkRow(x), [], JSON.stringify(x));++ const daily = db.prepare('SELECT top_paths, top_referers FROM analytics_daily ORDER BY date LIMIT 1').get();++ assert.deepStrictEqual(JSON.parse(daily.top_paths), [{ path: '/@:user', cnt: 5 }, { path: '/vods', cnt: 1 }]);++ assert.deepStrictEqual(JSON.parse(daily.top_referers), [{ referer: 'https://www.google.com', cnt: 5 }]);++ const all = dumpAll(db);++ for (const needle of ['198.51.100.', 'Lyon', 'Oslo', 'eyJhbGciOi', 'q=alex', '@alex', '/vod/12345', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);++ await retention.scrubEvents(db);++ assert.deepStrictEqual(retention.scrubRollups(db), { hourly: 0, daily: 0 }, 'idempotent');++ assert.deepStrictEqual(retention.rollupTotals(db), totals);++ db.close();++ });++++ await check('scrub with Network path options; other tables in the same database untouched', async () => {++ const db = newDb('network-scrub.db');++ db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT); INSERT INTO users VALUES (7, 'keepme');");++ new AnalyticsTracker(db, 'openvibe-network', { timers: false }).destroy();++ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, user_id, ip, user_agent, created_at) VALUES ('openvibe-network', 'pageview', ?, 7, '127.0.0.1', 'node', ?)`);++ const now = sqlTime(Date.now());++ for (const p of ['/avatar/keepme?s=1', '/api/auth/anon/tok0X9zz99', '/internal/users/by-username/keepme']) ins.run(p, now);++ db.prepare("INSERT INTO analytics_daily (service, date, pageviews, top_paths) VALUES ('openvibe-network', '2026-07-01', 5, ?)")++ .run(JSON.stringify([{ path: '/internal/users/by-username/keepme', cnt: 2 }, { path: '/avatar/keepme', cnt: 3 }]));++ const before = retention.inspect(db, NETWORK);++ assert.strictEqual(before.toScrub.paths, 3);++ await retention.scrubEvents(db, NETWORK);++ retention.scrubRollups(db, NETWORK);++ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY id').pluck().all(),++ ['/avatar/:param', '/api/auth/anon/:param', '/internal/users/:param/:username']);++ assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()),++ [{ path: '/avatar/:param', cnt: 3 }, { path: '/internal/users/:param/:username', cnt: 2 }]);++ assert.strictEqual(retention.inspect(db, NETWORK).toScrub.paths, 0);++ assert.strictEqual(db.prepare('SELECT username FROM users WHERE id = 7').pluck().get(), 'keepme');++ const analyticsOnly = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'analytics_%'").pluck().all()++ .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');++ for (const needle of ['keepme', 'X9zz99', '127.0.0.1']) assert.ok(!analyticsOnly.includes(needle), needle);++ db.close();++ });++++ fs.rmSync(tmp, { recursive: true, force: true });++ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }++ console.log('\nanalytics retention: all passed');++})();@@@ -7,7 +7,8 @@ const path = require('path');const pkg = require('../package.json');const ROOT = path.join(__dirname, '..');--const modules = fs.readdirSync(ROOT).filter((f) => f.endsWith('.js'));++const modules = fs.readdirSync(ROOT).filter((f) => f.endsWith('.js'))++ .concat(fs.readdirSync(path.join(ROOT, 'analytics')).filter((f) => f.endsWith('.js')).map((f) => `analytics/${f}`));const targets = new Set(Object.values(pkg.exports).map((t) => t.replace(/^\.\//, '')));for (const f of modules) assert.ok(targets.has(f), `${f} is exported`);for (const [sub, target] of Object.entries(pkg.exports)) {@@@ -20,6 +21,14 @@ for (const sub of ['analytics', 'app-icon', 'auth-client', 'brand', 'chrome-ssr'}// Server modules load in Node; the navbar module loads too (UMD, no DOM touched at require time).for (const sub of ['analytics', 'app-icon', 'brand', 'chrome-ssr', 'footer', 'icons', 'legal', 'notifications', 'seo', 'url-resolver', 'files', 'nav-icons', 'metrics', 'ready']) require(`openvibe-shared/${sub}`);++// analytics is dependency-free: its parts require only Node built-ins and each other.++for (const f of fs.readdirSync(path.join(ROOT, 'analytics'))) {++ const src = fs.readFileSync(path.join(ROOT, 'analytics', f), 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');++ for (const [, dep] of src.matchAll(/require\((['"])([^'"]+)\1\)/g).map((m) => [m[1], m[2]])) {++ assert.ok(dep.startsWith('.') || ['fs', 'path', 'crypto'].includes(dep), `analytics/${f} requires ${dep}`);++ }++}++assert.strictEqual(require('openvibe-shared/analytics/event.v1.json').$id, 'https://openvibe.network/contracts/analytics/event.v1.json');assert.strictEqual(typeof require('openvibe-shared/navbar').init, 'function');assert.ok(/^\d+\.\d+\.\d+$/.test(pkg.version), 'semver version');assert.ok(fs.readFileSync(path.join(ROOT, 'CHANGELOG.md'), 'utf8').includes(`## ${pkg.version}`), 'CHANGELOG has an entry for this version');
@@ -24,7 +24,7 @@ jobs:
@@ -24,7 +24,7 @@ jobs:
git archive --format=tar.gz --prefix=OpenVibe.Shared-ci/ -o "$RUNNER_TEMP/openvibe-shared.tgz" HEAD
git archive --format=tar.gz --prefix=OpenVibe.Shared-ci/ -o "$RUNNER_TEMP/openvibe-shared.tgz" HEAD
mkdir "$RUNNER_TEMP/consumer" && cd "$RUNNER_TEMP/consumer" && npm init -y >/dev/null
mkdir "$RUNNER_TEMP/consumer" && cd "$RUNNER_TEMP/consumer" && npm init -y >/dev/null
npm install --no-audit --no-fund "$RUNNER_TEMP/openvibe-shared.tgz"
npm install --no-audit --no-fund "$RUNNER_TEMP/openvibe-shared.tgz"
- node -e "require('openvibe-shared'); require('openvibe-shared/navbar'); require('openvibe-shared/legal'); require('openvibe-shared/metrics'); require('openvibe-shared/ready'); const f = require('openvibe-shared/files'); for (const n of f.BROWSER) require('fs').accessSync(f.path(n)); console.log('ok', require('openvibe-shared/package.json').version)"
+ node -e "require('openvibe-shared'); require('openvibe-shared/navbar'); require('openvibe-shared/legal'); require('openvibe-shared/metrics'); require('openvibe-shared/ready'); require('openvibe-shared/analytics'); require('openvibe-shared/analytics/prune-cli'); require('openvibe-shared/analytics/event.v1.json'); const f = require('openvibe-shared/files'); for (const n of f.BROWSER) require('fs').accessSync(f.path(n)); console.log('ok', require('openvibe-shared/package.json').version)"
# Secret scan (gitleaks) and dependency audit, the workflow every OpenVibers repo calls (Track Q).
# Secret scan (gitleaks) and dependency audit, the workflow every OpenVibers repo calls (Track Q).
security:
security:
uses: ./.github/workflows/security.yml
uses: ./.github/workflows/security.yml
@@ -23,7 +23,8 @@ were reconciled first.
@@ -23,7 +23,8 @@ were reconciled first.
| Kind | Files |
| Kind | Files |
|---|---|
|---|---|
| Browser scripts, served at `/shared/<file>` (listed in `files.js`) | `navbar.js`, `nav-icons.js`, `theme-loader.js`, `footer.js`, `notification-ui.js`, `account-switcher.js`, `user-card.js`, `ov-mark.js`, `ov-icons.js`, `history.js`, `sso-client.js`, `panels.js`, `ui.js`, `island.js`, `tooltip.js`, `openvibe-sw.js` |
| Browser scripts, served at `/shared/<file>` (listed in `files.js`) | `navbar.js`, `nav-icons.js`, `theme-loader.js`, `footer.js`, `notification-ui.js`, `account-switcher.js`, `user-card.js`, `ov-mark.js`, `ov-icons.js`, `history.js`, `sso-client.js`, `panels.js`, `ui.js`, `island.js`, `tooltip.js`, `openvibe-sw.js` |
-| Node modules (`require('openvibe-shared/<name>')`) | `index` (`.`), `analytics`, `app-icon`, `auth-client`, `brand`, `builtin-themes`, `chrome-ssr`, `legal`, `middleware`, `notifications`, `seo`, `theme-sync`, `url-resolver`, `files`, `release`, `metrics`, `ready`; `footer` and `icons` (= `ov-icons.js`) work on both sides |
+| Node modules (`require('openvibe-shared/<name>')`) | `index` (`.`), `analytics` (+ `analytics/{privacy,tracker,retention,schema,event,prune-cli}`), `app-icon`, `auth-client`, `brand`, `builtin-themes`, `chrome-ssr`, `legal`, `middleware`, `notifications`, `seo`, `theme-sync`, `url-resolver`, `files`, `release`, `metrics`, `ready`; `footer` and `icons` (= `ov-icons.js`) work on both sides |
+| Schemas | `docs/schemas/analytics-event.v1.json` (`analytics/event.v1`, exported as `openvibe-shared/analytics/event.v1.json`) |
| Generators | `scripts/build-nav-icons.py` (Font Awesome glyphs → `nav-icons.js`, `ov-icons.js`), `scripts/build-navbar-icons.js` (navbar.js's built-in glyphs), `scripts/build-theme-loader.js`, `scripts/build-app-icons.js` |
| Generators | `scripts/build-nav-icons.py` (Font Awesome glyphs → `nav-icons.js`, `ov-icons.js`), `scripts/build-navbar-icons.js` (navbar.js's built-in glyphs), `scripts/build-theme-loader.js`, `scripts/build-app-icons.js` |
Every module file has an `exports` entry, so `require('openvibe-shared/navbar')`,
Every module file has an `exports` entry, so `require('openvibe-shared/navbar')`,
@@ -72,6 +73,29 @@ A check fails when it throws, times out, returns `false`, a string (the reason)
@@ -72,6 +73,29 @@ A check fails when it throws, times out, returns `false`, a string (the reason)
`{ ok: false, error }`. Name a dependency required only when the service really cannot serve
`{ ok: false, error }`. Name a dependency required only when the service really cannot serve
without it.
without it.
+### Analytics (server, ADR-021)
+
+```js
+const Database = require('better-sqlite3'); // the service's own; Shared has no native dependency
+const { AnalyticsTracker } = require('openvibe-shared/analytics');
+const db = new Database('data/analytics.db');
+db.pragma('journal_mode = WAL');
+const analytics = new AnalyticsTracker(db, 'live', {
+ retention: { days: 30 }, // nightly prune (the default); false if you schedule it yourself
+ paramPrefixes: ['avatar'], // extra words whose next segment is a parameter
+ pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']], // parameters the segment rules can't see
+});
+app.use(analytics.middleware());
+```
+
+A raw event has no IP, user or subject id, or city. It has a route template (never a raw URL), a
+referer origin, a user-agent class, a rotating session id and a country. Raw events are kept for
+30 days, and the rollups for longer. A request with `Sec-GPC: 1` or `DNT: 1` is not recorded at
+all. The row shape is `analytics/event.v1` (`docs/schemas/analytics-event.v1.json`), and
+`require('openvibe-shared/analytics/event').checkRow(row)` checks a stored row against it. The
+prune and one-time scrub command is `openvibe-shared/analytics/prune-cli`: a service wraps it in
+its own `scripts/analytics-prune.js` and passes in its `Database` and default paths.
+
### Browser side: two options
### Browser side: two options
1. **From the Network** (what most pages do today):
1. **From the Network** (what most pages do today):
@@ -131,7 +155,12 @@ Tests are plain Node with stubbed browser globals. Some tests guard the release:
@@ -131,7 +155,12 @@ Tests are plain Node with stubbed browser globals. Some tests guard the release:
- **Generated blocks**: the generated navbar and theme-loader blocks must be up to date.
- **Generated blocks**: the generated navbar and theme-loader blocks must be up to date.
- **Browser bundles**: browser files must be free of `require()` and Node globals
- **Browser bundles**: browser files must be free of `require()` and Node globals
(`test/browser-bundles.test.js`).
(`test/browser-bundles.test.js`).
-- **Exports**: every export must resolve.
+- **Exports**: every export must resolve, and every module file (including `analytics/*.js`) must
+ have one.
+- **Analytics (ADR-021)**: no personal data in any table after real requests, opt-out requests
+ leave nothing, every row is a valid `analytics/event.v1`, prune/scrub keep rollups intact, and
+ the prune CLI's dry run changes nothing (`test/analytics-*.test.js`, with `better-sqlite3` as a
+ dev dependency).
CI runs on Node 22.22.1 and installs a tag-style tarball into an empty project.
CI runs on Node 22.22.1 and installs a tag-style tarball into an empty project.
deleted file mode 100644
deleted file mode 100644
@@ -1,1066 +0,0 @@
@@ -1,1066 +0,0 @@
-'use strict';
-
-// ═══════════════════════════════════════════════════════════════
-// OpenVibe — Unified Analytics Module
-// Shared analytics tracking for all OpenVibe services.
-// Each service stores events in its own SQLite DB using the
-// same schema. The admin panel aggregates via internal API.
-// ═══════════════════════════════════════════════════════════════
-
-// ── Bot Detection ────────────────────────────────────────────
-
-const BOT_USER_AGENTS = [
- /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,
- /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,
- /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,
- /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,
- /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,
- /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,
- /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,
- /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,
- /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,
- /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,
- /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,
- /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,
- /statuscake/i, /monitoring/i, /health.?check/i,
-];
-
-const SUSPICIOUS_PATTERNS = {
- // Hits per minute thresholds
- highRequestRate: 60,
- // Pages per session in under a minute
- rapidPageViews: 20,
- // No JS execution (missing beacon)
- noJsExecution: true,
- // Accessing well-known bot traps
- honeypotPaths: ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'],
-};
-
-/**
- * Classify whether a request is from a bot.
- * Returns: { isBot: boolean, botType: string|null, confidence: number }
- */
-function classifyRequest(req) {
- const ua = req.headers['user-agent'] || '';
- const ip = req.ip || req.connection?.remoteAddress || '';
-
- // 1. Known bot user agents
- for (const pattern of BOT_USER_AGENTS) {
- if (pattern.test(ua)) {
- return { isBot: true, botType: 'known_crawler', confidence: 0.95 };
- }
- }
-
- // 2. Missing or empty user agent
- if (!ua || ua.length < 10) {
- return { isBot: true, botType: 'no_useragent', confidence: 0.85 };
- }
-
- // 3. Honeypot paths
- const path = req.path || req.url || '';
- if (SUSPICIOUS_PATTERNS.honeypotPaths.some(p => path.toLowerCase().startsWith(p))) {
- return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };
- }
-
- // 4. Looks human (default)
- return { isBot: false, botType: null, confidence: 0.1 };
-}
-
-
-// ── Schema ───────────────────────────────────────────────────
-
-const ANALYTICS_SCHEMA = `
- -- Page views / API hits
- CREATE TABLE IF NOT EXISTS analytics_events (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- service TEXT NOT NULL,
- event_type TEXT NOT NULL DEFAULT 'pageview',
- path TEXT,
- method TEXT DEFAULT 'GET',
- status_code INTEGER,
- response_time_ms INTEGER,
- user_id INTEGER,
- session_id TEXT,
- ip TEXT,
- country TEXT,
- city TEXT,
- user_agent TEXT,
- referer TEXT,
- is_bot INTEGER DEFAULT 0,
- bot_type TEXT,
- device_type TEXT,
- browser TEXT,
- os TEXT,
- created_at DATETIME DEFAULT CURRENT_TIMESTAMP
- );
-
- -- Aggregated hourly stats (materialized by cron/timer)
- CREATE TABLE IF NOT EXISTS analytics_hourly (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- service TEXT NOT NULL,
- hour TEXT NOT NULL,
- pageviews INTEGER DEFAULT 0,
- api_calls INTEGER DEFAULT 0,
- unique_visitors INTEGER DEFAULT 0,
- unique_users INTEGER DEFAULT 0,
- bot_hits INTEGER DEFAULT 0,
- avg_response_ms INTEGER DEFAULT 0,
- error_count INTEGER DEFAULT 0,
- bandwidth_bytes INTEGER DEFAULT 0,
- top_paths TEXT,
- top_referers TEXT,
- UNIQUE(service, hour)
- );
-
- -- Daily aggregates
- CREATE TABLE IF NOT EXISTS analytics_daily (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- service TEXT NOT NULL,
- date TEXT NOT NULL,
- pageviews INTEGER DEFAULT 0,
- api_calls INTEGER DEFAULT 0,
- unique_visitors INTEGER DEFAULT 0,
- unique_users INTEGER DEFAULT 0,
- new_users INTEGER DEFAULT 0,
- bot_hits INTEGER DEFAULT 0,
- avg_response_ms INTEGER DEFAULT 0,
- error_count INTEGER DEFAULT 0,
- top_paths TEXT,
- top_referers TEXT,
- top_countries TEXT,
- device_breakdown TEXT,
- browser_breakdown TEXT,
- UNIQUE(service, date)
- );
-
- -- Rate tracking for bot detection
- CREATE TABLE IF NOT EXISTS analytics_rate_tracking (
- ip TEXT NOT NULL,
- window_start INTEGER NOT NULL,
- hit_count INTEGER DEFAULT 1,
- PRIMARY KEY (ip, window_start)
- );
-
- -- Indexes
- CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_ip ON analytics_events(ip, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_events_user ON analytics_events(user_id, created_at);
- CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);
- CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);
-`;
-
-
-// ── Device/Browser Parsing (lightweight) ─────────────────────
-
-function parseUserAgent(ua) {
- if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };
-
- // Device
- let device = 'desktop';
- if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';
- else if (/ipad|tablet/i.test(ua)) device = 'tablet';
-
- // Browser
- let browser = 'other';
- if (/edg\//i.test(ua)) browser = 'edge';
- else if (/opr\/|opera/i.test(ua)) browser = 'opera';
- else if (/firefox\//i.test(ua)) browser = 'firefox';
- else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';
- else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';
- else if (/trident|msie/i.test(ua)) browser = 'ie';
-
- // OS
- let os = 'other';
- if (/windows/i.test(ua)) os = 'windows';
- else if (/macintosh|mac os/i.test(ua)) os = 'macos';
- else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';
- else if (/android/i.test(ua)) os = 'android';
- else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';
-
- return { device, browser, os };
-}
-
-
-// ── Analytics Tracker Class ──────────────────────────────────
-
-class AnalyticsTracker {
- /**
- * @param {object} db - better-sqlite3 database instance
- * @param {string} service - service name (e.g., 'live', 'openvibe-network', 'tools')
- */
- constructor(db, service) {
- this.db = db;
- this.service = service;
- this._buffer = [];
- this._flushInterval = null;
- this._aggregateInterval = null;
-
- // Initialize schema
- this.db.exec(ANALYTICS_SCHEMA);
-
- // Prepared statements
- this._insertEvent = this.db.prepare(`
- INSERT INTO analytics_events
- (service, event_type, path, method, status_code, response_time_ms,
- user_id, session_id, ip, country, city, user_agent, referer,
- is_bot, bot_type, device_type, browser, os)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `);
-
- this._insertMany = this.db.transaction((events) => {
- for (const e of events) {
- this._insertEvent.run(
- e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,
- e.user_id, e.session_id, e.ip, e.country, e.city, e.user_agent, e.referer,
- e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os
- );
- }
- });
-
- this._updateRate = this.db.prepare(`
- INSERT INTO analytics_rate_tracking (ip, window_start, hit_count)
- VALUES (?, ?, 1)
- ON CONFLICT(ip, window_start) DO UPDATE SET hit_count = hit_count + 1
- `);
-
- this._getRate = this.db.prepare(
- 'SELECT SUM(hit_count) as total FROM analytics_rate_tracking WHERE ip = ? AND window_start >= ?'
- );
-
- this._cleanRates = this.db.prepare(
- 'DELETE FROM analytics_rate_tracking WHERE window_start < ?'
- );
-
- // Start flush timer (write buffered events every 5s)
- this._flushInterval = setInterval(() => this.flush(), 5000);
-
- // Hourly aggregation timer
- this._aggregateInterval = setInterval(() => this.aggregate(), 60 * 60 * 1000);
-
- // Initial aggregation on startup (catch up)
- setTimeout(() => this.aggregate(), 10_000);
-
- // Clean old rate tracking every 10 min
- setInterval(() => {
- const cutoff = Math.floor(Date.now() / 1000) - 300;
- try { this._cleanRates.run(cutoff); } catch {}
- }, 10 * 60 * 1000);
- }
-
- /**
- * Express middleware for automatic request tracking.
- * Tracks page views and API calls with response times.
- */
- middleware() {
- return (req, res, next) => {
- // Skip health checks and static assets
- const path = req.path || '';
- if (path === '/api/health' || path === '/health' ||
- /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path)) {
- return next();
- }
-
- const startTime = Date.now();
-
- // Hook into response finish
- const originalEnd = res.end;
- res.end = (...args) => {
- res.end = originalEnd;
- res.end(...args);
-
- const responseTime = Date.now() - startTime;
- const botCheck = classifyRequest(req);
- const parsed = parseUserAgent(req.headers['user-agent'] || '');
-
- const isApi = path.startsWith('/api/') || path.startsWith('/internal/') || path.startsWith('/oauth/');
- const eventType = isApi ? 'api_call' : 'pageview';
-
- // Rate-based bot detection
- const ip = req.ip || '';
- const windowStart = Math.floor(Date.now() / 60000); // 1-minute windows
- try {
- this._updateRate.run(ip, windowStart);
- if (!botCheck.isBot) {
- const rateData = this._getRate.get(ip, windowStart - 1);
- if (rateData && rateData.total > SUSPICIOUS_PATTERNS.highRequestRate) {
- botCheck.isBot = true;
- botCheck.botType = 'rate_limit';
- botCheck.confidence = 0.80;
- }
- }
- } catch {}
-
- // Get session ID from cookie or generate from IP+UA
- const sessionId = req.cookies?.openvibe_session ||
- req.cookies?.ov_token?.slice(-16) ||
- null;
-
- this._buffer.push({
- service: this.service,
- event_type: eventType,
- path: path.substring(0, 500),
- method: req.method,
- status_code: res.statusCode,
- response_time_ms: responseTime,
- user_id: req.user?.id || req.user?.sub || null,
- session_id: sessionId,
- ip: ip,
- country: req.headers['cf-ipcountry'] || null,
- city: null,
- user_agent: (req.headers['user-agent'] || '').substring(0, 500),
- referer: (req.headers['referer'] || '').substring(0, 500),
- is_bot: botCheck.isBot,
- bot_type: botCheck.botType,
- device_type: parsed.device,
- browser: parsed.browser,
- os: parsed.os,
- });
-
- // Auto-flush if buffer gets large
- if (this._buffer.length >= 100) {
- this.flush();
- }
- };
-
- next();
- };
- }
-
- /**
- * Track a custom event (not from HTTP request).
- */
- trackEvent(eventType, data = {}) {
- this._buffer.push({
- service: this.service,
- event_type: eventType,
- path: data.path || null,
- method: data.method || null,
- status_code: data.status_code || null,
- response_time_ms: data.response_time_ms || null,
- user_id: data.user_id || null,
- session_id: data.session_id || null,
- ip: data.ip || null,
- country: data.country || null,
- city: data.city || null,
- user_agent: data.user_agent || null,
- referer: data.referer || null,
- is_bot: data.is_bot || false,
- bot_type: data.bot_type || null,
- device_type: data.device_type || null,
- browser: data.browser || null,
- os: data.os || null,
- });
- }
-
- /**
- * Flush buffered events to database.
- */
- flush() {
- if (this._buffer.length === 0) return;
- const events = this._buffer.splice(0);
- try {
- this._insertMany(events);
- } catch (err) {
- console.error(`[Analytics:${this.service}] Flush error:`, err.message);
- }
- }
-
- /**
- * Run hourly + daily aggregation.
- */
- aggregate() {
- try {
- this._aggregateHourly();
- this._aggregateDaily();
- // Prune raw events older than 90 days
- this.db.prepare("DELETE FROM analytics_events WHERE created_at < datetime('now', '-90 days')").run();
- } catch (err) {
- console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);
- }
- }
-
- _aggregateHourly() {
- // Aggregate last 2 hours (to catch stragglers)
- const hours = [];
- const now = new Date();
- for (let i = 0; i < 2; i++) {
- const d = new Date(now.getTime() - i * 3600000);
- hours.push(d.toISOString().slice(0, 13) + ':00:00');
- }
-
- const upsert = this.db.prepare(`
- INSERT OR REPLACE INTO analytics_hourly
- (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `);
-
- for (const hour of hours) {
- const nextHour = new Date(new Date(hour).getTime() + 3600000).toISOString().slice(0, 19).replace('T', ' ');
- const hourStart = hour.replace('T', ' ');
-
- const stats = this.db.prepare(`
- SELECT
- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
- COUNT(DISTINCT ip) AS unique_visitors,
- COUNT(DISTINCT user_id) AS unique_users,
- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at < ?
- `).get(this.service, hourStart, nextHour);
-
- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) continue;
-
- const topPaths = this.db.prepare(`
- SELECT path, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0
- GROUP BY path ORDER BY cnt DESC LIMIT 10
- `).all(this.service, hourStart, nextHour);
-
- const topReferers = this.db.prepare(`
- SELECT referer, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0
- GROUP BY referer ORDER BY cnt DESC LIMIT 10
- `).all(this.service, hourStart, nextHour);
-
- upsert.run(
- this.service, hourStart,
- stats.pageviews, stats.api_calls, stats.unique_visitors, stats.unique_users,
- stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
- JSON.stringify(topPaths), JSON.stringify(topReferers)
- );
- }
- }
-
- _aggregateDaily() {
- // Aggregate last 2 days
- const days = [];
- const now = new Date();
- for (let i = 0; i < 2; i++) {
- const d = new Date(now.getTime() - i * 86400000);
- days.push(d.toISOString().slice(0, 10));
- }
-
- const upsert = this.db.prepare(`
- INSERT OR REPLACE INTO analytics_daily
- (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,
- avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `);
-
- for (const date of days) {
- const dayStart = date + ' 00:00:00';
- const dayEnd = date + ' 23:59:59';
-
- const stats = this.db.prepare(`
- SELECT
- COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
- COUNT(DISTINCT ip) AS unique_visitors,
- COUNT(DISTINCT user_id) AS unique_users,
- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
- COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ?
- `).get(this.service, dayStart, dayEnd);
-
- if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) continue;
-
- const topPaths = this.db.prepare(`
- SELECT path, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0
- GROUP BY path ORDER BY cnt DESC LIMIT 15
- `).all(this.service, dayStart, dayEnd);
-
- const topReferers = this.db.prepare(`
- SELECT referer, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0
- GROUP BY referer ORDER BY cnt DESC LIMIT 10
- `).all(this.service, dayStart, dayEnd);
-
- const topCountries = this.db.prepare(`
- SELECT country, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0
- GROUP BY country ORDER BY cnt DESC LIMIT 10
- `).all(this.service, dayStart, dayEnd);
-
- const devices = this.db.prepare(`
- SELECT device_type, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0
- GROUP BY device_type ORDER BY cnt DESC
- `).all(this.service, dayStart, dayEnd);
-
- const browsers = this.db.prepare(`
- SELECT browser, COUNT(*) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0
- GROUP BY browser ORDER BY cnt DESC
- `).all(this.service, dayStart, dayEnd);
-
- // New users (first seen this day)
- const newUsers = this.db.prepare(`
- SELECT COUNT(DISTINCT user_id) as cnt FROM analytics_events
- WHERE service = ? AND created_at >= ? AND created_at <= ? AND user_id IS NOT NULL
- AND user_id NOT IN (
- SELECT DISTINCT user_id FROM analytics_events
- WHERE service = ? AND created_at < ? AND user_id IS NOT NULL
- )
- `).get(this.service, dayStart, dayEnd, this.service, dayStart);
-
- upsert.run(
- this.service, date, stats.pageviews, stats.api_calls,
- stats.unique_visitors, stats.unique_users,
- newUsers?.cnt || 0, stats.bot_hits,
- stats.avg_response_ms || 0, stats.error_count,
- JSON.stringify(topPaths), JSON.stringify(topReferers),
- JSON.stringify(topCountries), JSON.stringify(devices),
- JSON.stringify(browsers)
- );
- }
- }
-
- /**
- * Get analytics data for the admin dashboard.
- * @param {object} options
- * @param {number} options.days - Number of days to look back (default 30)
- * @param {number} options.hours - If set, overrides days with an hour-based range (e.g. 1, 12, 24, 72)
- * @param {string} options.service - Service name override
- */
- getStats(options = {}) {
- const { days = 30, hours, service } = options;
- const svc = service || this.service;
-
- // Compute cutoff timestamp for raw event queries
- const rangeMs = hours ? hours * 3600000 : days * 86400000;
- const rangeCutoff = new Date(Date.now() - rangeMs).toISOString().slice(0, 19).replace('T', ' ');
- const isSubDay = hours && hours < 24;
- const effectiveDays = hours ? hours / 24 : days;
-
- // Daily trend (use daily aggregates for multi-day, raw events for sub-day)
- let daily = [];
- if (!isSubDay) {
- const dateCutoff = new Date(Date.now() - rangeMs).toISOString().slice(0, 10);
- daily = this.db.prepare(`
- SELECT * FROM analytics_daily
- WHERE service = ? AND date >= ?
- ORDER BY date ASC
- `).all(svc, dateCutoff);
- }
-
- // Hourly trend (always useful — scope to the requested range)
- const hourlyLimit = hours ? Math.min(hours, 168) : 24;
- const hourlyCutoff = new Date(Date.now() - hourlyLimit * 3600000).toISOString().slice(0, 19).replace('T', ' ');
- const hourly = this.db.prepare(`
- SELECT * FROM analytics_hourly
- WHERE service = ? AND hour >= ?
- ORDER BY hour ASC
- `).all(svc, hourlyCutoff);
-
- // For sub-day ranges, generate fine-grained time buckets from raw events
- let timeBuckets = [];
- if (isSubDay) {
- // 5-minute buckets for ≤1h, 15-min for ≤12h, 30-min for ≤24h
- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;
- timeBuckets = this.db.prepare(`
- SELECT
- strftime('%Y-%m-%d %H:', created_at) ||
- CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT) as bucket,
- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,
- COUNT(DISTINCT ip) FILTER (WHERE is_bot = 0) AS unique_visitors,
- COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- GROUP BY bucket
- ORDER BY bucket ASC
- `).all(svc, rangeCutoff);
- }
-
- // Summary totals — for sub-day ranges, compute from raw events
- let summary;
- if (hours && hours < 48) {
- summary = this.db.prepare(`
- SELECT
- COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,
- COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,
- COUNT(DISTINCT ip) FILTER (WHERE is_bot = 0) AS total_unique_visitors,
- COUNT(DISTINCT user_id) FILTER (WHERE is_bot = 0 AND user_id IS NOT NULL) AS total_unique_users,
- COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,
- COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- `).get(svc, rangeCutoff);
- } else {
- const dateCutoff = new Date(Date.now() - rangeMs).toISOString().slice(0, 10);
- summary = this.db.prepare(`
- SELECT
- SUM(pageviews) as total_pageviews,
- SUM(api_calls) as total_api_calls,
- SUM(unique_visitors) as total_unique_visitors,
- SUM(unique_users) as total_unique_users,
- SUM(bot_hits) as total_bot_hits,
- SUM(error_count) as total_errors,
- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
- FROM analytics_daily
- WHERE service = ? AND date >= ?
- `).get(svc, dateCutoff);
- }
-
- // Real-time (last 5 min)
- const fiveMinAgo = new Date(Date.now() - 300000).toISOString().slice(0, 19).replace('T', ' ');
- const realtime = this.db.prepare(`
- SELECT
- COUNT(*) as requests,
- COUNT(DISTINCT ip) as visitors,
- COUNT(*) FILTER (WHERE is_bot = 1) as bots
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- `).get(svc, fiveMinAgo);
-
- // Top pages (human only)
- const topPages = this.db.prepare(`
- SELECT path, COUNT(*) as hits, COUNT(DISTINCT ip) as visitors
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY path ORDER BY hits DESC LIMIT 20
- `).all(svc, rangeCutoff);
-
- // Top referers
- const topReferers = this.db.prepare(`
- SELECT referer, COUNT(*) as hits
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- AND referer IS NOT NULL AND referer != '' AND is_bot = 0
- GROUP BY referer ORDER BY hits DESC LIMIT 15
- `).all(svc, rangeCutoff);
-
- // Bot breakdown
- const botBreakdown = this.db.prepare(`
- SELECT bot_type, COUNT(*) as hits
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 1
- GROUP BY bot_type ORDER BY hits DESC LIMIT 15
- `).all(svc, rangeCutoff);
-
- // Status code distribution
- const statusCodes = this.db.prepare(`
- SELECT
- CASE
- WHEN status_code >= 200 AND status_code < 300 THEN '2xx'
- WHEN status_code >= 300 AND status_code < 400 THEN '3xx'
- WHEN status_code >= 400 AND status_code < 500 THEN '4xx'
- WHEN status_code >= 500 THEN '5xx'
- ELSE 'other'
- END as group_code,
- COUNT(*) as cnt
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- GROUP BY group_code ORDER BY cnt DESC
- `).all(svc, rangeCutoff);
-
- // Device/browser/OS breakdown
- const deviceBreakdown = this.db.prepare(`
- SELECT device_type, COUNT(*) as cnt
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY device_type ORDER BY cnt DESC
- `).all(svc, rangeCutoff);
-
- const browserBreakdown = this.db.prepare(`
- SELECT browser, COUNT(*) as cnt
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY browser ORDER BY cnt DESC
- `).all(svc, rangeCutoff);
-
- const osBreakdown = this.db.prepare(`
- SELECT os, COUNT(*) as cnt
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY os ORDER BY cnt DESC
- `).all(svc, rangeCutoff);
-
- // Country breakdown
- const countryBreakdown = this.db.prepare(`
- SELECT country, COUNT(*) as cnt
- FROM analytics_events
- WHERE service = ? AND created_at >= ?
- AND country IS NOT NULL AND is_bot = 0
- GROUP BY country ORDER BY cnt DESC LIMIT 20
- `).all(svc, rangeCutoff);
-
- // ── Enhanced Metrics ─────────────────────────────────
-
- // Response time percentiles (p50, p90, p95, p99)
- let responsePercentiles = {};
- try {
- const rtRows = this.db.prepare(`
- SELECT response_time_ms FROM analytics_events
- WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
- ORDER BY response_time_ms ASC
- `).all(svc, rangeCutoff);
- if (rtRows.length > 0) {
- const vals = rtRows.map(r => r.response_time_ms);
- const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];
- responsePercentiles = { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };
- }
- } catch {}
-
- // Bandwidth estimation (response_time_ms * rough factor isn't great,
- // but we can count requests and estimate from content-length if tracked;
- // for now, count total requests * avg page weight heuristic)
- let bandwidth = {};
- try {
- const bwData = this.db.prepare(`
- SELECT
- COUNT(*) as total_requests,
- COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,
- COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,
- SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- `).get(svc, rangeCutoff);
- if (bwData) {
- bandwidth = {
- total_requests: bwData.total_requests || 0,
- page_requests: bwData.page_requests || 0,
- api_requests: bwData.api_requests || 0,
- estimated_bytes: bwData.estimated_bytes || 0,
- };
- }
- } catch {}
-
- // Authenticated vs anonymous sessions
- let authBreakdown = {};
- try {
- authBreakdown = this.db.prepare(`
- SELECT
- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS authenticated,
- COUNT(*) FILTER (WHERE user_id IS NULL) AS anonymous,
- COUNT(DISTINCT user_id) FILTER (WHERE user_id IS NOT NULL) AS unique_authenticated_users,
- COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- `).get(svc, rangeCutoff) || {};
- } catch {}
-
- // Error rate over time (hourly for multi-day, bucketed for sub-day)
- let errorTrend = [];
- try {
- if (isSubDay) {
- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;
- errorTrend = this.db.prepare(`
- SELECT
- strftime('%Y-%m-%d %H:', created_at) ||
- CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT) as bucket,
- COUNT(*) AS total,
- COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
- COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY bucket ORDER BY bucket ASC
- `).all(svc, rangeCutoff);
- } else {
- errorTrend = this.db.prepare(`
- SELECT hour as bucket,
- (pageviews + api_calls) as total,
- error_count as errors
- FROM analytics_hourly
- WHERE service = ? AND hour >= ?
- ORDER BY hour ASC
- `).all(svc, hourlyCutoff);
- }
- } catch {}
-
- // Peak hours (which hours of day get most traffic)
- let peakHours = [];
- try {
- peakHours = this.db.prepare(`
- SELECT
- CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day,
- COUNT(*) AS hits,
- COUNT(DISTINCT ip) AS visitors,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY hour_of_day
- ORDER BY hour_of_day ASC
- `).all(svc, rangeCutoff);
- } catch {}
-
- // Top API endpoints (separate from top pages)
- let topApiEndpoints = [];
- try {
- topApiEndpoints = this.db.prepare(`
- SELECT path, method, COUNT(*) as hits,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,
- COUNT(*) FILTER (WHERE status_code >= 400) AS errors
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0
- GROUP BY path, method ORDER BY hits DESC LIMIT 20
- `).all(svc, rangeCutoff);
- } catch {}
-
- // Unique sessions count
- let sessionCount = 0;
- try {
- const sc = this.db.prepare(`
- SELECT COUNT(DISTINCT session_id) as cnt
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0
- `).get(svc, rangeCutoff);
- sessionCount = sc?.cnt || 0;
- } catch {}
-
- // New vs returning visitors (first-time IPs in this period vs seen before)
- let visitorTypes = {};
- try {
- visitorTypes = this.db.prepare(`
- SELECT
- COUNT(DISTINCT ip) FILTER (
- WHERE ip NOT IN (
- SELECT DISTINCT ip FROM analytics_events
- WHERE service = ? AND created_at < ? AND is_bot = 0
- )
- ) AS new_visitors,
- COUNT(DISTINCT ip) FILTER (
- WHERE ip IN (
- SELECT DISTINCT ip FROM analytics_events
- WHERE service = ? AND created_at < ? AND is_bot = 0
- )
- ) AS returning_visitors
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- `).get(svc, rangeCutoff, svc, rangeCutoff, svc, rangeCutoff) || {};
- } catch {}
-
- // Slowest endpoints
- let slowestEndpoints = [];
- try {
- slowestEndpoints = this.db.prepare(`
- SELECT path, method,
- COUNT(*) as hits,
- CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,
- MAX(response_time_ms) AS max_ms
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
- GROUP BY path, method
- HAVING hits >= 3
- ORDER BY avg_ms DESC LIMIT 10
- `).all(svc, rangeCutoff);
- } catch {}
-
- // Authenticated vs anonymous time trend
- let authTrend = [];
- try {
- if (isSubDay) {
- const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;
- authTrend = this.db.prepare(`
- SELECT
- strftime('%Y-%m-%d %H:', created_at) ||
- CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT) as bucket,
- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NOT NULL) AS auth_visitors,
- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NULL) AS anon_visitors,
- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS auth_hits,
- COUNT(*) FILTER (WHERE user_id IS NULL) AS anon_hits
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY bucket ORDER BY bucket ASC
- `).all(svc, rangeCutoff);
- } else {
- authTrend = this.db.prepare(`
- SELECT
- strftime('%Y-%m-%d', created_at) AS bucket,
- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NOT NULL) AS auth_visitors,
- COUNT(DISTINCT ip) FILTER (WHERE user_id IS NULL) AS anon_visitors,
- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS auth_hits,
- COUNT(*) FILTER (WHERE user_id IS NULL) AS anon_hits
- FROM analytics_events
- WHERE service = ? AND created_at >= ? AND is_bot = 0
- GROUP BY bucket ORDER BY bucket ASC
- `).all(svc, rangeCutoff);
- }
- } catch {}
-
- return {
- service: svc,
- period_days: effectiveDays,
- period_hours: hours || null,
- summary: summary || {},
- realtime: realtime || {},
- daily,
- hourly,
- timeBuckets,
- topPages,
- topReferers,
- botBreakdown,
- statusCodes,
- deviceBreakdown,
- browserBreakdown,
- osBreakdown,
- countryBreakdown,
- // Enhanced metrics
- responsePercentiles,
- bandwidth,
- authBreakdown,
- errorTrend,
- peakHours,
- topApiEndpoints,
- sessionCount,
- visitorTypes,
- slowestEndpoints,
- authTrend,
- };
- }
-
- /**
- * Get a cross-service overview (for the admin panel).
- */
- getOverview(days = 30) {
- const cutoff = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);
-
- const services = this.db.prepare(`
- SELECT
- service,
- SUM(pageviews) as pageviews,
- SUM(api_calls) as api_calls,
- SUM(unique_visitors) as unique_visitors,
- SUM(unique_users) as unique_users,
- SUM(bot_hits) as bot_hits,
- SUM(error_count) as errors,
- CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
- FROM analytics_daily
- WHERE date >= ?
- GROUP BY service
- ORDER BY pageviews DESC
- `).all(cutoff);
-
- // Totals across all services
- const totals = this.db.prepare(`
- SELECT
- SUM(pageviews) as pageviews,
- SUM(api_calls) as api_calls,
- SUM(unique_visitors) as unique_visitors,
- SUM(bot_hits) as bot_hits,
- SUM(error_count) as errors
- FROM analytics_daily
- WHERE date >= ?
- `).get(cutoff);
-
- // Daily trend (all services combined)
- const dailyTrend = this.db.prepare(`
- SELECT date,
- SUM(pageviews) as pageviews,
- SUM(api_calls) as api_calls,
- SUM(unique_visitors) as unique_visitors,
- SUM(bot_hits) as bot_hits
- FROM analytics_daily
- WHERE date >= ?
- GROUP BY date
- ORDER BY date ASC
- `).all(cutoff);
-
- // Real-time all services
- const fiveMinAgo = new Date(Date.now() - 300000).toISOString().slice(0, 19).replace('T', ' ');
- const realtime = this.db.prepare(`
- SELECT
- service,
- COUNT(*) as requests,
- COUNT(DISTINCT ip) as visitors,
- COUNT(*) FILTER (WHERE is_bot = 1) as bots
- FROM analytics_events
- WHERE created_at >= ?
- GROUP BY service
- `).all(fiveMinAgo);
-
- return {
- period_days: days,
- totals: totals || {},
- services,
- dailyTrend,
- realtime,
- };
- }
-
- /**
- * Get bot analysis data.
- */
- getBotAnalysis(days = 30) {
- const daysCast = Math.min(Math.max(1, days), 365);
-
- // Top bot IPs
- const topBotIPs = this.db.prepare(`
- SELECT ip, COUNT(*) as hits, bot_type, user_agent,
- MIN(created_at) as first_seen, MAX(created_at) as last_seen
- FROM analytics_events
- WHERE is_bot = 1 AND created_at >= datetime('now', '-' || ? || ' days')
- GROUP BY ip ORDER BY hits DESC LIMIT 25
- `).all(daysCast);
-
- // Bot vs human ratio over time
- const botTrend = this.db.prepare(`
- SELECT date,
- SUM(pageviews) + SUM(api_calls) as human_hits,
- SUM(bot_hits) as bot_hits
- FROM analytics_daily
- WHERE date >= date('now', '-' || ? || ' days')
- GROUP BY date ORDER BY date ASC
- `).all(daysCast);
-
- // Bot type breakdown
- const botTypes = this.db.prepare(`
- SELECT bot_type, COUNT(*) as hits, COUNT(DISTINCT ip) as unique_ips
- FROM analytics_events
- WHERE is_bot = 1 AND created_at >= datetime('now', '-' || ? || ' days')
- GROUP BY bot_type ORDER BY hits DESC
- `).all(daysCast);
-
- // Suspicious behavior patterns
- const suspiciousIPs = this.db.prepare(`
- SELECT ip, COUNT(*) as total_hits,
- COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,
- COUNT(DISTINCT path) as unique_paths,
- MIN(created_at) as first_seen,
- MAX(created_at) as last_seen
- FROM analytics_events
- WHERE created_at >= datetime('now', '-7 days') AND is_bot = 0
- GROUP BY ip
- HAVING total_hits > 500
- ORDER BY total_hits DESC LIMIT 20
- `).all();
-
- return {
- topBotIPs,
- botTrend,
- botTypes,
- suspiciousIPs,
- };
- }
-
- /**
- * Destroy tracker (clean up timers).
- */
- destroy() {
- this.flush();
- if (this._flushInterval) clearInterval(this._flushInterval);
- if (this._aggregateInterval) clearInterval(this._aggregateInterval);
- }
-}
-
-
-module.exports = {
- AnalyticsTracker,
- classifyRequest,
- parseUserAgent,
- ANALYTICS_SCHEMA,
- BOT_USER_AGENTS,
- SUSPICIOUS_PATTERNS,
-};
new file mode 100644
new file mode 100644
@@ -0,0 +1,130 @@
@@ -0,0 +1,130 @@
+'use strict';
+/**
+ * analytics/event.v1: the raw analytics event shape (ADR-021). The JSON schema is
+ * docs/schemas/analytics-event.v1.json, also exported as 'openvibe-shared/analytics/event.v1.json'.
+ *
+ * EVENT_V1 the schema object
+ * toEvent(row) an analytics_events row -> an event.v1 object (column renames, 0/1 -> boolean,
+ * created_at -> RFC 3339). The retired columns ip/user_id/city are not part
+ * of the event; checkRow() reports them.
+ * validateEvent(ev) [] when ev conforms to event.v1, else a list of 'field: problem' strings
+ * checkRow(row) validateEvent(toEvent(row)) plus any non-NULL ip / user_id / city
+ *
+ * The validator interprets the schema file itself, for the keywords it uses (type, enum, pattern,
+ * maxLength, minimum, maximum, required, properties, additionalProperties: false), and refuses a
+ * schema with any other keyword, so the file and this code cannot drift apart. No dependencies.
+ */
+
+const EVENT_V1 = require('../docs/schemas/analytics-event.v1.json');
+
+const RETIRED_COLUMNS = ['ip', 'user_id', 'city'];
+const ANNOTATIONS = new Set(['$schema', '$id', 'title', 'description']);
+const OBJECT_KEYWORDS = new Set(['type', 'required', 'additionalProperties', 'properties']);
+const VALUE_KEYWORDS = new Set(['type', 'enum', 'pattern', 'maxLength', 'minimum', 'maximum']);
+
+function checkKeywords(node, allowed, where) {
+ for (const k of Object.keys(node)) {
+ if (!ANNOTATIONS.has(k) && !allowed.has(k)) throw new Error(`analytics event schema: unsupported keyword ${k} at ${where}`);
+ }
+}
+
+/** Compile the schema into per-field checks, once. */
+function compile(schema) {
+ checkKeywords(schema, OBJECT_KEYWORDS, '/');
+ if (schema.type !== 'object' || schema.additionalProperties !== false) throw new Error('analytics event schema: expected a closed object');
+ const fields = {};
+ for (const [name, spec] of Object.entries(schema.properties)) {
+ checkKeywords(spec, VALUE_KEYWORDS, `/properties/${name}`);
+ const types = spec.type == null ? null : [].concat(spec.type);
+ const re = spec.pattern ? new RegExp(spec.pattern, 'u') : null;
+ fields[name] = (v) => {
+ if (spec.enum && !spec.enum.some((e) => e === v)) return `not one of ${JSON.stringify(spec.enum)}`;
+ if (types && !types.some((t) => typeOk(t, v))) return `expected ${types.join(' or ')}`;
+ if (v === null) return null;
+ if (typeof v === 'string') {
+ if (spec.maxLength != null && [...v].length > spec.maxLength) return `longer than ${spec.maxLength}`;
+ if (re && !re.test(v)) return `does not match ${spec.pattern}`;
+ }
+ if (typeof v === 'number') {
+ if (spec.minimum != null && v < spec.minimum) return `below ${spec.minimum}`;
+ if (spec.maximum != null && v > spec.maximum) return `above ${spec.maximum}`;
+ }
+ return null;
+ };
+ }
+ return { fields, required: schema.required || [] };
+}
+
+function typeOk(t, v) {
+ switch (t) {
+ case 'null': return v === null;
+ case 'string': return typeof v === 'string';
+ case 'boolean': return typeof v === 'boolean';
+ case 'integer': return Number.isInteger(v);
+ case 'number': return typeof v === 'number' && Number.isFinite(v);
+ default: throw new Error(`analytics event schema: unsupported type ${t}`);
+ }
+}
+
+const COMPILED = compile(EVENT_V1);
+
+/** [] when `ev` is a valid analytics/event.v1 object; otherwise one 'field: problem' string per issue. */
+function validateEvent(ev) {
+ if (!ev || typeof ev !== 'object' || Array.isArray(ev)) return ['event: expected an object'];
+ const errors = [];
+ for (const k of COMPILED.required) if (!(k in ev)) errors.push(`${k}: required`);
+ for (const [k, v] of Object.entries(ev)) {
+ const check = COMPILED.fields[k];
+ if (!check) { errors.push(`${k}: not allowed`); continue; }
+ const problem = check(v);
+ if (problem) errors.push(`${k}: ${problem}`);
+ }
+ return errors;
+}
+
+const nullable = (v) => (v === undefined || v === '' ? null : v);
+const bool = (v) => v === true || v === 1 || v === '1';
+
+/** 'YYYY-MM-DD HH:MM:SS' (SQLite CURRENT_TIMESTAMP, UTC) -> 'YYYY-MM-DDTHH:MM:SSZ'; ISO input passes. */
+function isoTime(v) {
+ if (v == null) return v;
+ const s = String(v);
+ if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(s)) return s.replace(' ', 'T') + 'Z';
+ return s;
+}
+
+/** An analytics_events row as an analytics/event.v1 object. */
+function toEvent(row) {
+ return {
+ event: row.event_type,
+ service: row.service,
+ route: nullable(row.path),
+ method: nullable(row.method),
+ status: nullable(row.status_code),
+ duration_ms: nullable(row.response_time_ms),
+ session_id: nullable(row.session_id),
+ country: nullable(row.country),
+ ua_class: nullable(row.user_agent),
+ device: nullable(row.device_type),
+ browser: nullable(row.browser),
+ os: nullable(row.os),
+ referer_origin: nullable(row.referer),
+ is_bot: bool(row.is_bot),
+ bot_type: nullable(row.bot_type),
+ authenticated: bool(row.authenticated),
+ timestamp: isoTime(row.created_at),
+ };
+}
+
+/** Problems with a stored row: retired personal columns that are set, then the event.v1 checks. */
+function checkRow(row) {
+ const errors = RETIRED_COLUMNS.filter((c) => row[c] != null).map((c) => `${c}: must be NULL (ADR-021)`);
+ return errors.concat(validateEvent(toEvent(row)));
+}
+
+/** The enum of an event.v1 field, without null (for reducers that must stay inside the schema). */
+function allowed(field) {
+ return EVENT_V1.properties[field].enum.filter((v) => v !== null);
+}
+
+module.exports = { EVENT_V1, RETIRED_COLUMNS, toEvent, validateEvent, checkRow, allowed };
new file mode 100644
new file mode 100644
@@ -0,0 +1,49 @@
@@ -0,0 +1,49 @@
+'use strict';
+/**
+ * openvibe-shared/analytics: request analytics within ADR-021 (OpenVibe.Contracts
+ * docs/adr/ADR-021-analytics.md). A raw event has no IP, user or subject id, or city. Paths are route
+ * templates and referers are reduced to origins. A user agent is stored as a class, sessions are
+ * rotating ids, and raw events are pruned after 30 days. A request with `Sec-GPC: 1` or `DNT: 1` is
+ * not recorded. The event shape is analytics/event.v1 (docs/schemas/analytics-event.v1.json).
+ *
+ * const { AnalyticsTracker } = require('openvibe-shared/analytics');
+ * const tracker = new AnalyticsTracker(db, 'live', { retention: { days: 30 } }); // db: better-sqlite3
+ * app.use(tracker.middleware());
+ *
+ * Parts (also available as subpaths): tracker, privacy, retention, schema, event, prune-cli.
+ * No dependencies: the caller passes its own better-sqlite3 handle (and Database, for the CLI).
+ */
+
+const { AnalyticsTracker } = require('./tracker');
+const privacy = require('./privacy');
+const retention = require('./retention');
+const schema = require('./schema');
+const event = require('./event');
+
+/**
+ * @deprecated The pre-ADR-021 module's tuning object, kept so the 1.x export list stays complete
+ * (register item C-80). It is frozen and read-only, and the tracker does not read it. It will be
+ * removed in 2.0.0.
+ */
+const SUSPICIOUS_PATTERNS = Object.freeze({
+ highRequestRate: privacy.HIGH_REQUEST_RATE,
+ rapidPageViews: 20,
+ noJsExecution: true,
+ honeypotPaths: Object.freeze([...privacy.HONEYPOT_PATHS]),
+});
+
+module.exports = {
+ AnalyticsTracker,
+ privacy,
+ retention,
+ schema,
+ event,
+ ensureSchema: schema.ensureSchema,
+ optedOut: privacy.optedOut,
+ // Names the pre-ADR module exported. They are kept for 1.x and now use the ADR-021 behaviour.
+ classifyRequest: privacy.classifyRequest,
+ parseUserAgent: privacy.parseUserAgent,
+ ANALYTICS_SCHEMA: schema.ANALYTICS_SCHEMA,
+ BOT_USER_AGENTS: privacy.BOT_USER_AGENTS,
+ SUSPICIOUS_PATTERNS,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,276 @@
@@ -0,0 +1,276 @@
+'use strict';
+/**
+ * What a raw analytics row may carry (ADR-021, OpenVibe.Contracts docs/adr/ADR-021-analytics.md):
+ * the event name, the service, a ROUTE TEMPLATE (never a raw URL with ids or a query string), a
+ * rotating session id, the country, a user-agent CLASS and the timestamp. Never an IP address, a
+ * precise location or a user/subject id.
+ *
+ * This file holds the pure reducers that turn a request into those fields, and the opt-out check
+ * (Sec-GPC / DNT). No dependencies. The raw row shape is analytics/event.v1
+ * (docs/schemas/analytics-event.v1.json, ./event.js).
+ */
+
+// ── Opt-out (Global Privacy Control, Do Not Track) ───────────
+
+/** True when a header value is exactly "1" (either header may arrive repeated as an array). */
+function headerIsOne(v) {
+ if (Array.isArray(v)) return v.some(headerIsOne);
+ return v != null && String(v).trim() === '1';
+}
+
+/**
+ * True when the request asks not to be tracked: `Sec-GPC: 1` (Global Privacy Control) or `DNT: 1`.
+ * Such a request is not recorded at all — no raw row, no visitor hash, no session, no rate counter —
+ * so it is also missing from every rollup (ADR-021 allows no per-request trace of an opted-out
+ * visitor, and does not provide for a separate opted-out count).
+ */
+function optedOut(headers) {
+ if (!headers) return false;
+ return headerIsOne(headers['sec-gpc']) || headerIsOne(headers.dnt);
+}
+
+// ── Bots ──────────────────────────────────────────────────────
+
+const BOT_USER_AGENTS = [
+ /googlebot/i, /bingbot/i, /slurp/i, /duckduckbot/i, /baiduspider/i,
+ /yandexbot/i, /facebot/i, /ia_archiver/i, /semrushbot/i, /ahrefsbot/i,
+ /mj12bot/i, /dotbot/i, /petalbot/i, /rogerbot/i, /screaming frog/i,
+ /seznambot/i, /sogou/i, /exabot/i, /archive\.org_bot/i, /crawler/i,
+ /spider/i, /python-requests/i, /python-urllib/i, /httpx/i,
+ /go-http-client/i, /java\//i, /libwww-perl/i, /wget/i, /curl/i,
+ /headlesschrome/i, /phantomjs/i, /scrapy/i, /node-fetch/i,
+ /axios/i, /postman/i, /insomnia/i, /lighthouse/i, /pagespeed/i,
+ /gptbot/i, /chatgpt-user/i, /claudebot/i, /anthropic/i,
+ /bytespider/i, /amazonbot/i, /applebot/i, /twitterbot/i,
+ /facebookexternalhit/i, /linkedinbot/i, /whatsapp/i, /telegrambot/i,
+ /discordbot/i, /slackbot/i, /uptimerobot/i, /pingdom/i,
+ /statuscake/i, /monitoring/i, /health.?check/i,
+];
+
+const HONEYPOT_PATHS = ['/wp-login.php', '/xmlrpc.php', '/wp-admin', '/.env', '/admin.php', '/phpmyadmin'];
+
+/** Requests per client in the current + previous minute above which a "human" is reclassified as a bot. */
+const HIGH_REQUEST_RATE = 60;
+
+/** The family name of the first bot pattern the user agent matches, or null. */
+function botName(ua) {
+ if (!ua) return null;
+ for (const re of BOT_USER_AGENTS) {
+ if (re.test(ua)) return re.source.toLowerCase().replace(/[^a-z0-9]+/g, '') || 'bot';
+ }
+ return null;
+}
+
+/** { isBot, botType, confidence } from the user agent and the path (the rate check is the tracker's). */
+function classifyRequest(req) {
+ const ua = (req.headers && req.headers['user-agent']) || '';
+ if (botName(ua)) return { isBot: true, botType: 'known_crawler', confidence: 0.95 };
+ if (!ua || ua.length < 10) return { isBot: true, botType: 'no_useragent', confidence: 0.85 };
+ const path = String(req.path || req.url || '').toLowerCase();
+ if (HONEYPOT_PATHS.some((p) => path.startsWith(p))) return { isBot: true, botType: 'honeypot_hit', confidence: 0.90 };
+ return { isBot: false, botType: null, confidence: 0.1 };
+}
+
+// ── User agent → class ───────────────────────────────────────
+
+function parseUserAgent(ua) {
+ if (!ua) return { device: 'unknown', browser: 'unknown', os: 'unknown' };
+ let device = 'desktop';
+ if (/mobile|android|iphone|ipod/i.test(ua)) device = 'mobile';
+ else if (/ipad|tablet/i.test(ua)) device = 'tablet';
+
+ let browser = 'other';
+ if (/edg\//i.test(ua)) browser = 'edge';
+ else if (/opr\/|opera/i.test(ua)) browser = 'opera';
+ else if (/firefox\//i.test(ua)) browser = 'firefox';
+ else if (/chrome\//i.test(ua) && !/chromium/i.test(ua)) browser = 'chrome';
+ else if (/safari\//i.test(ua) && !/chrome/i.test(ua)) browser = 'safari';
+ else if (/trident|msie/i.test(ua)) browser = 'ie';
+
+ let os = 'other';
+ if (/windows/i.test(ua)) os = 'windows';
+ else if (/macintosh|mac os/i.test(ua)) os = 'macos';
+ else if (/linux/i.test(ua) && !/android/i.test(ua)) os = 'linux';
+ else if (/android/i.test(ua)) os = 'android';
+ else if (/iphone|ipad|ipod/i.test(ua)) os = 'ios';
+
+ return { device, browser, os };
+}
+
+const UA_CLASS_RE = /^(none|bot:[a-z0-9]+|[a-z]+\/[a-z]+\/[a-z]+)$/;
+
+/**
+ * The user-agent class stored instead of the user-agent string: `bot:<family>` for a known crawler,
+ * `<browser>/<os>/<device>` otherwise, `none` when there is no user agent. Idempotent (a class maps
+ * to itself), so a scrub can run over rows that are already reduced.
+ */
+function uaClass(ua) {
+ if (ua == null || ua === '') return 'none';
+ const s = String(ua);
+ if (UA_CLASS_RE.test(s)) return s;
+ const bot = botName(s);
+ if (bot) return `bot:${bot}`;
+ const p = parseUserAgent(s);
+ return `${p.browser}/${p.os}/${p.device}`;
+}
+
+// ── Referer → origin ─────────────────────────────────────────
+
+/** Only the scheme://host[:port] of an http(s) referer; anything else is dropped. */
+function refererOrigin(ref) {
+ if (!ref) return null;
+ try {
+ const u = new URL(String(ref));
+ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
+ return u.origin;
+ } catch {
+ return null;
+ }
+}
+
+// ── Country ──────────────────────────────────────────────────
+
+/** An ISO 3166-1 alpha-2 code (CDN country header) or null. Never a region or city. */
+function countryCode(v) {
+ const s = String(v || '').trim().toUpperCase();
+ return /^[A-Z]{2}$/.test(s) ? s : null;
+}
+
+// ── Paths → route templates ──────────────────────────────────
+
+/**
+ * Words after which the next path segment is a parameter even when it looks like a word
+ * (usernames, slugs). Over-redacting is the safe failure: `/u/alex` → `/u/:param`.
+ */
+const DEFAULT_PARAM_PREFIXES = [
+ 'u', 'user', 'users', 'profile', 'profiles', 'member', 'members', 'channel', 'channels', 'c',
+ 'vod', 'clip', 'p', 'paste', 'recap', 'stream', 'watch', 'v', 'embed', 'video', 'videos', 'shorts',
+ 'playlist', 'category', 'tag', 'tags', 'search', 'q', 's', 'invite', 'reset', 'verify', 'confirm',
+ 'token', 'tokens', 'share', 'r', 'job', 'jobs', 'download', 'downloads', 'file', 'files', 'doc',
+ 'recipe', 'recipes', 'place', 'places', 'room', 'rooms', 'dm', 'dms', 'thread', 'threads',
+];
+
+const MAX_SEGMENTS = 8;
+const MAX_TEMPLATE_LENGTH = 200;
+const SAFE_SEGMENT_RE = /^[A-Za-z0-9._~-]+$/;
+const TEMPLATE_SEGMENT_RE = /^@?:[A-Za-z_][A-Za-z0-9_]*\??$|^\*$/;
+
+/** True for a segment that is (or may be) an identifier rather than a fixed word of the route. */
+function looksLikeId(s) {
+ if (!SAFE_SEGMENT_RE.test(s)) return true; // spaces, @, +, =, commas, %…
+ if (/^\d+$/.test(s)) return true; // 123
+ if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s)) return true; // uuid
+ if (/^[0-9a-f]{8,}$/i.test(s)) return true; // hashes
+ if (/^[A-Za-z]{2,6}_[A-Za-z0-9]{6,}$/.test(s)) return true; // usr_…, hbt_…
+ if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(s)) return true; // ulid
+ if (/\d/.test(s) && s.length >= 6) return true; // slugs/tokens carrying numbers
+ if (/[a-z]/.test(s) && /[A-Z]/.test(s) && s.length >= 8) return true; // base64-ish ids, CamelCase names
+ if (s.length > 32) return true;
+ return false;
+}
+
+/**
+ * Path options with a service's extra parameter words merged into the defaults:
+ * `{ paramPrefixes: Set, pathRules }`, or undefined when there is nothing to add.
+ */
+function pathOptions({ paramPrefixes, pathRules } = {}) {
+ const extra = paramPrefixes ? [...paramPrefixes] : [];
+ const rules = pathRules && pathRules.length ? checkRules(pathRules) : null;
+ if (!extra.length && !rules) return undefined;
+ const out = { paramPrefixes: new Set([...DEFAULT_PARAM_PREFIXES, ...extra]) };
+ if (rules) out.pathRules = rules;
+ return out;
+}
+
+function checkRules(rules) {
+ for (const r of rules) {
+ if (!Array.isArray(r) || !(r[0] instanceof RegExp) || typeof r[1] !== 'string') {
+ throw new TypeError('pathRules entries are [RegExp, replacement string] pairs');
+ }
+ }
+ return rules;
+}
+
+/** Apply a service's pathRules ([RegExp, replacement] pairs) to a raw path, in order. */
+function applyPathRules(p, rules) {
+ let s = String(p);
+ if (rules) for (const [re, to] of rules) { re.lastIndex = 0; s = s.replace(re, to); }
+ return s;
+}
+
+/**
+ * Reduce a path (or a full URL) to a route template: no query, no fragment, identifiers replaced by
+ * `:id`, the segment after a parameter-taking word by `:param`, `/@name` by `/@:user`. Already-templated
+ * segments (`:id`, `@:user`, Express `:name`) pass through, so the function is idempotent.
+ *
+ * Options (a service's own route shapes):
+ * paramPrefixes words whose next segment is a parameter (default DEFAULT_PARAM_PREFIXES; pass the
+ * merged set from pathOptions() to add to the defaults rather than replace them)
+ * pathRules [RegExp, replacement] pairs applied to the raw path first, for a parameter the
+ * segment rules cannot see (e.g. [/\/by-username\/[^/?#]+/gi, '/by-username/:username']).
+ * A rule's output must be stable under the same rule, or the result is not idempotent.
+ */
+function normalisePath(raw, { paramPrefixes = DEFAULT_PARAM_PREFIXES, pathRules } = {}) {
+ if (raw == null || raw === '') return '/';
+ let p = String(raw);
+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {
+ try { p = new URL(p).pathname; } catch { return '/'; }
+ }
+ p = p.split(/[?#]/)[0];
+ if (pathRules) p = applyPathRules(p, pathRules);
+ const prefixes = paramPrefixes instanceof Set ? paramPrefixes : new Set(paramPrefixes);
+ const segs = p.split('/').filter(Boolean);
+ const out = [];
+ let expectParam = false;
+ for (let i = 0; i < segs.length && i < MAX_SEGMENTS; i++) {
+ let s = segs[i];
+ try { s = decodeURIComponent(s); } catch { /* keep the raw segment */ }
+ if (TEMPLATE_SEGMENT_RE.test(s)) { out.push(s); expectParam = false; continue; }
+ if (s.startsWith('@')) { out.push('@:user'); expectParam = true; continue; }
+ if (expectParam) { out.push(':param'); expectParam = false; continue; }
+ if (looksLikeId(s)) { out.push(':id'); continue; }
+ out.push(s);
+ expectParam = prefixes.has(s.toLowerCase());
+ }
+ if (segs.length > MAX_SEGMENTS) out.push('*');
+ // Too long: drop whole segments (never cut one in half) and end with `*`.
+ if (('/' + out.join('/')).length > MAX_TEMPLATE_LENGTH) {
+ if (out[out.length - 1] === '*') out.pop();
+ while (out.length && ('/' + out.join('/')).length > MAX_TEMPLATE_LENGTH - 2) out.pop();
+ out.push('*');
+ }
+ return '/' + out.join('/');
+}
+
+/**
+ * The route template for a finished request: the matched Express route (mount path + route path)
+ * when there is one, else the normalised original path. Both go through normalisePath, because a
+ * mount path with parameters (`/api/vods/stream/:streamId/chunk`) arrives in req.baseUrl filled in.
+ */
+function routeTemplate(req, fallbackPath, opts) {
+ const r = req && req.route;
+ if (r && typeof r.path === 'string' && r.path && !/[*(]/.test(r.path)) {
+ return normalisePath(`${req.baseUrl || ''}${r.path}`, opts);
+ }
+ return normalisePath(fallbackPath || (req && (req.originalUrl || req.url)) || '/', opts);
+}
+
+module.exports = {
+ BOT_USER_AGENTS,
+ HONEYPOT_PATHS,
+ HIGH_REQUEST_RATE,
+ DEFAULT_PARAM_PREFIXES,
+ MAX_TEMPLATE_LENGTH,
+ optedOut,
+ botName,
+ classifyRequest,
+ parseUserAgent,
+ uaClass,
+ refererOrigin,
+ countryCode,
+ looksLikeId,
+ pathOptions,
+ applyPathRules,
+ normalisePath,
+ routeTemplate,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,235 @@
@@ -0,0 +1,235 @@
+'use strict';
+/**
+ * The analytics-prune command (ADR-021): raw-event retention and the one-time scrub of pre-ADR rows,
+ * for one or more analytics databases. A service keeps a thin scripts/analytics-prune.js that passes
+ * its better-sqlite3 and its defaults:
+ *
+ * const Database = require('better-sqlite3');
+ * const cli = require('openvibe-shared/analytics/prune-cli');
+ * const main = (argv, log) => cli.main(argv, { Database, log, defaultDb: process.env.ANALYTICS_DB || 'data/analytics.db' });
+ * if (require.main === module) cli.run(main);
+ * module.exports = { main };
+ *
+ * Options given by the service (second argument of main):
+ * Database better-sqlite3's constructor (required; this package has no native dependency)
+ * log line logger (default console.log)
+ * defaultDb the database when no --db is given: a path, or (args, env) => path
+ * targets (args) => [{ name, file }]: the databases when no --db is given (several apps)
+ * options extra repeatable value flags, { '--app': 'apps' } -> args.apps = [...]
+ * paramPrefixes the service's extra parameter words } as for the tracker, so legacy rows are
+ * pathRules the service's [RegExp, replacement] } templated like new ones
+ * backupMode file mode of a backup (default 0o600: a pre-scrub backup holds the old IPs)
+ * usage the help text (default: USAGE below)
+ * root base for relative --db / --backup paths (default process.cwd())
+ * env environment (default process.env)
+ *
+ * Exit codes: 0 done (or dry run), 1 a check failed (backup unreadable, rollup totals changed),
+ * 2 refused (bad arguments, no backup choice, existing backup target, missing database).
+ */
+
+const fs = require('fs');
+const path = require('path');
+const retention = require('./retention');
+
+const USAGE = `Raw analytics retention and one-time scrub (ADR-021).
+
+ analytics-prune dry run: counts only, changes nothing
+ analytics-prune --scrub dry run including what the scrub would rewrite
+ analytics-prune --apply --backup <path> online backup, then prune
+ analytics-prune --apply --scrub --backup <path>
+ analytics-prune --apply --no-backup prune without a backup (explicit)
+
+Options:
+ --db <file> an analytics database (repeatable; default: the service's own)
+ --days <n> keep raw events newer than n days, 1..30 (default 30)
+ --scrub after pruning, rewrite the remaining rows: ip/user_id/city -> NULL, path -> route
+ template, referer -> origin, user_agent -> class, legacy session ids -> NULL; same
+ path/referer reduction in the rollups' top lists (their counts are untouched)
+ --backup <path> one database: a new file; several: a directory, each backup <name>.analytics.db
+ in it (sqlite online backup, owner-only, verified with quick_check and a row count)
+ --no-backup explicitly skip the backup
+ --no-vacuum skip the VACUUM that follows an --apply (VACUUM rewrites the file so pruned and
+ scrubbed values do not survive in free pages; it needs ~2x the database size free
+ and holds the database's write lock while it runs)
+ --batch <n> rows per write batch (default 5000)
+
+--apply refuses to run without --backup or --no-backup. Rollup totals are compared before and after
+every database; any difference exits 1. Prune and scrub are safe while the service is up (short
+batches; the tracker retries a busy flush).`;
+
+function parseArgs(argv, extra = {}) {
+ const a = {
+ apply: false, scrub: false, vacuum: true, backup: null, noBackup: false,
+ days: retention.MAX_DAYS, batch: retention.DEFAULT_BATCH, dbs: [], db: null,
+ };
+ for (const key of Object.values(extra)) a[key] = [];
+ for (let i = 0; i < argv.length; i++) {
+ const k = argv[i];
+ const val = () => { if (i + 1 >= argv.length) throw new Error(`${k} needs a value`); return argv[++i]; };
+ if (k === '--apply') a.apply = true;
+ else if (k === '--scrub') a.scrub = true;
+ else if (k === '--no-vacuum') a.vacuum = false;
+ else if (k === '--no-backup') a.noBackup = true;
+ else if (k === '--backup') a.backup = val();
+ else if (k === '--days') a.days = Number(val());
+ else if (k === '--batch') a.batch = Number(val());
+ else if (k === '--db') a.dbs.push(val());
+ else if (Object.prototype.hasOwnProperty.call(extra, k)) a[extra[k]].push(val());
+ else if (k === '-h' || k === '--help') a.help = true;
+ else throw new Error(`unknown option ${k}`);
+ }
+ a.db = a.dbs[0] || null;
+ retention.checkDays(a.days);
+ if (!Number.isInteger(a.batch) || a.batch < 1 || a.batch > 100000) throw new Error('--batch must be 1..100000');
+ if (a.backup && a.noBackup) throw new Error('--backup and --no-backup are exclusive');
+ return a;
+}
+
+/** [{ name, file }] for explicit --db paths: apps/<app>/data/analytics.db -> "<app>", else the file's base name. */
+function namedTargets(files, root = process.cwd()) {
+ const seen = new Map();
+ return files.map((f) => {
+ const file = path.resolve(root, f);
+ let name = path.basename(path.dirname(file)) === 'data'
+ ? path.basename(path.dirname(path.dirname(file)))
+ : path.basename(file, path.extname(file));
+ const n = (seen.get(name) || 0) + 1;
+ seen.set(name, n);
+ if (n > 1) name = `${name}-${n}`;
+ return { name, file };
+ });
+}
+
+function resolveTargets(args, opts) {
+ const root = opts.root || process.cwd();
+ if (args.dbs.length) return namedTargets(args.dbs, root);
+ if (opts.targets) return opts.targets(args);
+ const d = typeof opts.defaultDb === 'function' ? opts.defaultDb(args, opts.env || process.env) : opts.defaultDb;
+ if (!d) return [];
+ return namedTargets([d], root);
+}
+
+const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;
+function fileBytes(f) { try { return fs.statSync(f).size; } catch { return 0; } }
+function freeBytes(dir) { try { const s = fs.statfsSync(dir); return s.bavail * s.bsize; } catch { return null; } }
+const sameTotals = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+async function backupTo(Database, db, target, need, mode, expectRows, log) {
+ if (fs.existsSync(target)) { log(`error: backup target ${target} already exists; choose a new path`); return 2; }
+ const free = freeBytes(path.dirname(target));
+ if (free != null && free < need * 1.1) { log(`error: ${mb(free)} free at ${path.dirname(target)}, the backup needs about ${mb(need)}`); return 2; }
+ // The backup may hold pre-ADR rows (IPs, user ids): owner-only from the first byte.
+ const umask = process.umask(0o077);
+ try { await db.backup(target); } finally { process.umask(umask); }
+ fs.chmodSync(target, mode);
+ const b = new Database(target, { readonly: true });
+ try {
+ const ok = b.pragma('quick_check', { simple: true });
+ const n = b.prepare("SELECT COUNT(*) FROM sqlite_master WHERE name = 'analytics_events'").pluck().get()
+ ? b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get() : 0;
+ if (ok !== 'ok' || n < expectRows) { log(`error: backup check failed (quick_check=${ok}, rows=${n})`); return 1; }
+ } finally { b.close(); }
+ log(`backup ${target} (${mb(fileBytes(target))}, mode ${(mode & 0o777).toString(8).padStart(4, '0')}, quick_check ok)`);
+ return 0;
+}
+
+async function runOne(t, args, backupTarget, opts, log) {
+ const { Database } = opts;
+ const pathOpts = { paramPrefixes: opts.paramPrefixes, pathRules: opts.pathRules };
+ if (!fs.existsSync(t.file)) { log(`error: ${t.file} does not exist`); return 2; }
+ const db = new Database(t.file, args.apply ? { fileMustExist: true } : { readonly: true, fileMustExist: true });
+ try {
+ db.pragma('busy_timeout = 5000');
+ const walBytes = fileBytes(t.file + '-wal');
+ const before = retention.inspect(db, { days: args.days, ...pathOpts });
+ log(`\n[${t.name}] ${t.file}`);
+ log(`size ${mb(before.bytes)} (+ ${mb(walBytes)} WAL); a backup needs about ${mb(before.bytes + walBytes)}; VACUUM about ${mb(2 * before.bytes)} more`);
+ log(`raw events ${before.events} (oldest ${before.oldest || '-'}, newest ${before.newest || '-'})`);
+ log(`prune ${before.older} rows created before ${before.cutoff} UTC (${args.days} days); ${before.remaining} stay`);
+ if (before.toScrub) {
+ const s = before.toScrub;
+ log(`scrub of the rows that stay: ${s.personal} with ip/user_id/city, ${s.paths} paths to template, ${s.referers} referers to origin, ${s.user_agents} user agents to class, ${s.sessions} legacy session ids${args.scrub ? '' : ' (needs --scrub)'}`);
+ }
+ log(`rate rows ${before.rateRows || 0} (IP counters; the tracker keeps them in memory now)`);
+ for (const [k, v] of Object.entries(before.rollups)) log(`rollups ${k}: ${v.rows} rows, ${v.pageviews} pageviews, ${v.api_calls} api calls (kept)`);
+ if (!args.apply) return 0;
+
+ if (backupTarget) {
+ const code = await backupTo(Database, db, backupTarget, before.bytes + walBytes, opts.backupMode, before.events, log);
+ if (code) return code;
+ }
+
+ db.pragma('secure_delete = ON');
+ const totalsBefore = retention.rollupTotals(db);
+ const pruned = await retention.pruneRawEvents(db, { days: args.days, batchSize: args.batch });
+ log(`pruned ${pruned.deleted} rows in ${pruned.batches} batches`);
+ if (args.scrub) {
+ const s = await retention.scrubEvents(db, { batchSize: args.batch, ...pathOpts });
+ const r = retention.scrubRollups(db, pathOpts);
+ log(`scrubbed ${s.rows} raw rows in ${s.batches} batches; rollup top lists rewritten in ${r.hourly} hourly and ${r.daily} daily rows`);
+ }
+ const totalsAfter = retention.rollupTotals(db);
+ if (!sameTotals(totalsBefore, totalsAfter)) {
+ log(`error: rollup totals changed!\n before ${JSON.stringify(totalsBefore)}\n after ${JSON.stringify(totalsAfter)}`);
+ return 1;
+ }
+ log('rollups totals unchanged');
+ if (args.vacuum && (pruned.deleted || args.scrub)) {
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ db.exec('VACUUM');
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ log(`vacuumed ${mb(fileBytes(t.file))}`);
+ }
+ const after = retention.inspect(db, { days: args.days, ...pathOpts });
+ log(`now ${after.events} raw events, ${after.older} older than ${args.days} days${after.toScrub ? `, ${after.toScrub.personal} with ip/user_id/city` : ''}`);
+ return 0;
+ } finally {
+ db.close();
+ }
+}
+
+/** Run the command. Resolves the exit code; never calls process.exit. */
+async function main(argv, opts = {}) {
+ const log = opts.log || console.log;
+ if (typeof opts.Database !== 'function') throw new TypeError('prune-cli: pass { Database } (better-sqlite3)');
+ const o = { backupMode: 0o600, ...opts };
+ let args;
+ try { args = parseArgs(argv, o.options || {}); } catch (e) { log(`error: ${e.message}`); return 2; }
+ if (args.help) { log(o.usage || USAGE); return 0; }
+ const list = resolveTargets(args, o);
+ if (!list.length) { log('no analytics databases found (use --db)'); return 2; }
+ if (args.apply && !args.backup && !args.noBackup) {
+ log('refusing --apply without a backup: pass --backup <new file or directory> (sqlite online backup) or --no-backup');
+ return 2;
+ }
+ let backupFor = () => null;
+ if (args.apply && args.backup) {
+ const target = path.resolve(o.root || process.cwd(), args.backup);
+ if (list.length === 1) {
+ backupFor = () => target;
+ } else {
+ if (fs.existsSync(target) && !fs.statSync(target).isDirectory()) { log(`error: ${target} is a file; several databases need a directory`); return 2; }
+ const clash = list.map((t) => path.join(target, `${t.name}.analytics.db`)).filter((f) => fs.existsSync(f));
+ if (clash.length) { log(`error: backup targets already exist: ${clash.join(', ')}`); return 2; }
+ fs.mkdirSync(target, { recursive: true, mode: 0o700 });
+ backupFor = (t) => path.join(target, `${t.name}.analytics.db`);
+ }
+ }
+ let worst = 0;
+ for (const t of list) {
+ const code = await runOne(t, args, backupFor(t), o, log);
+ worst = Math.max(worst, code);
+ if (code) { if (list.length > 1) log(`[${t.name}] stopped (exit ${code}); later databases not touched`); break; }
+ }
+ if (!args.apply && worst === 0) log('\ndry run: nothing changed. Re-run with --apply --backup <file|dir> (or --no-backup).');
+ return worst;
+}
+
+/** For a service's script: `if (require.main === module) run(main)` — exits with main's code. */
+function run(mainFn, argv = process.argv.slice(2)) {
+ return Promise.resolve()
+ .then(() => mainFn(argv))
+ .then((code) => process.exit(code), (err) => { console.error(err); process.exit(1); });
+}
+
+module.exports = { USAGE, main, run, parseArgs, namedTargets, resolveTargets };
new file mode 100644
new file mode 100644
@@ -0,0 +1,248 @@
@@ -0,0 +1,248 @@
+'use strict';
+/**
+ * Raw analytics retention and the one-time scrub (ADR-021).
+ *
+ * pruneRawEvents(db, { days }) delete analytics_events rows older than `days` (≤ 30) in bounded
+ * batches, yielding to the event loop between batches. Rollups
+ * (analytics_hourly / analytics_daily) are never touched. Also clears
+ * leftovers: visitor hashes older than yesterday, salts of past days,
+ * and analytics_rate_tracking (IP counters live in memory now).
+ * scrubEvents(db) rewrite the rows written before ADR-021: ip / user_id / city →
+ * NULL (authenticated = 1 where a user id was present), path → route
+ * template, referer → origin, user_agent → class, session ids that are
+ * not rotating ids → NULL.
+ * scrubRollups(db) the same path/referer reduction inside the rollups' top_paths /
+ * top_referers JSON (counts merged, never changed).
+ * inspect(db, { days }) read-only counts for a dry run.
+ * rollupTotals(db) sums of every rollup counter, to prove a run left them alone.
+ * schedulePrune(db, opts) the nightly prune on unref'd timers (first run 5 min after boot,
+ * then every 24 h); returns { run, stop }.
+ *
+ * scrubEvents / scrubRollups / inspect take the service's path options ({ paramPrefixes, pathRules },
+ * see privacy.normalisePath) so legacy rows are templated with the same rules as new ones.
+ *
+ * Used by the tracker (opts.retention), a service's own job runner, and ./prune-cli.js.
+ */
+
+const { ensureSchema } = require('./schema');
+const privacy = require('./privacy');
+
+const MAX_DAYS = 30;
+const DEFAULT_BATCH = 5000;
+const ROTATING_SESSION_RE = /^[0-9a-f]{16}$/;
+
+const pad = (n) => String(n).padStart(2, '0');
+function sqlTime(ms) {
+ const d = new Date(ms);
+ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
+}
+
+function checkDays(days) {
+ const n = Number(days);
+ if (!Number.isInteger(n) || n < 1 || n > MAX_DAYS) {
+ throw new Error(`days must be an integer from 1 to ${MAX_DAYS} (ADR-021 keeps raw analytics at most ${MAX_DAYS} days)`);
+ }
+ return n;
+}
+
+/** The created_at bound: rows strictly older than this are pruned. */
+function cutoffFor(days, nowMs = Date.now()) {
+ return sqlTime(nowMs - checkDays(days) * 86400000);
+}
+
+const tableExists = (db, name) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
+const columnExists = (db, table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === col);
+const tick = () => new Promise((r) => setImmediate(r));
+
+/**
+ * Delete raw events older than `days` in batches of `batchSize` (each batch its own short write
+ * transaction). Returns { cutoff, deleted, batches, complete }.
+ */
+async function pruneRawEvents(db, { days = MAX_DAYS, batchSize = DEFAULT_BATCH, maxBatches = Infinity, now = Date.now } = {}) {
+ const cutoff = cutoffFor(days, now());
+ const out = { cutoff, deleted: 0, batches: 0, complete: true };
+ if (!tableExists(db, 'analytics_events')) return out;
+ const del = db.prepare('DELETE FROM analytics_events WHERE id IN (SELECT id FROM analytics_events WHERE created_at < ? LIMIT ?)');
+ for (;;) {
+ if (out.batches >= maxBatches) { out.complete = false; break; }
+ const n = del.run(cutoff, batchSize).changes;
+ out.batches++;
+ out.deleted += n;
+ if (n < batchSize) break;
+ await tick();
+ }
+ const today = sqlTime(now()).slice(0, 10);
+ const yesterday = sqlTime(now() - 86400000).slice(0, 10);
+ if (tableExists(db, 'analytics_visitor_days')) db.prepare('DELETE FROM analytics_visitor_days WHERE day < ?').run(yesterday);
+ if (tableExists(db, 'analytics_day_salts')) db.prepare('DELETE FROM analytics_day_salts WHERE day < ?').run(today);
+ if (tableExists(db, 'analytics_rate_tracking')) db.prepare('DELETE FROM analytics_rate_tracking').run();
+ return out;
+}
+
+/** SQL functions for the scrub and the dry-run counts (connection-scoped, safe on a read-only handle). */
+function registerFunctions(db, { paramPrefixes, pathRules } = {}) {
+ const opts = privacy.pathOptions({ paramPrefixes, pathRules });
+ db.function('ov_route_template', { deterministic: true }, (p) => (p == null ? null : privacy.normalisePath(p, opts)));
+ db.function('ov_referer_origin', { deterministic: true }, (r) => (r == null || r === '' ? r : privacy.refererOrigin(r)));
+ db.function('ov_ua_class', { deterministic: true }, (u) => privacy.uaClass(u));
+ db.function('ov_rotating_session', { deterministic: true }, (s) => (s != null && ROTATING_SESSION_RE.test(String(s)) ? s : null));
+}
+
+/** Rewrite every raw row to the ADR-021 fields, in id-range batches. Returns { rows, batches }. */
+async function scrubEvents(db, { batchSize = DEFAULT_BATCH, paramPrefixes, pathRules } = {}) {
+ ensureSchema(db);
+ registerFunctions(db, { paramPrefixes, pathRules });
+ const { lo, hi } = db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi FROM analytics_events').get();
+ const out = { rows: 0, batches: 0 };
+ if (lo == null) return out;
+ const upd = db.prepare(`
+ UPDATE analytics_events SET
+ authenticated = CASE WHEN user_id IS NOT NULL THEN 1 ELSE authenticated END,
+ ip = NULL, user_id = NULL, city = NULL,
+ session_id = ov_rotating_session(session_id),
+ path = ov_route_template(path),
+ referer = ov_referer_origin(referer),
+ user_agent = ov_ua_class(user_agent)
+ WHERE id >= ? AND id < ?
+ `);
+ for (let from = lo; from <= hi; from += batchSize) {
+ out.rows += upd.run(from, from + batchSize).changes;
+ out.batches++;
+ await tick();
+ }
+ return out;
+}
+
+/** Re-template a rollup's top list ([{ path|referer, cnt }]) — counts are summed, never altered. */
+function reduceTopList(json, key, fn) {
+ let list;
+ try { list = JSON.parse(json); } catch { return null; }
+ if (!Array.isArray(list)) return null;
+ const merged = new Map();
+ for (const item of list) {
+ if (!item || typeof item !== 'object') continue;
+ const k = fn(item[key]);
+ if (k == null) continue;
+ merged.set(k, (merged.get(k) || 0) + (Number(item.cnt) || 0));
+ }
+ const next = [...merged].map(([k, cnt]) => ({ [key]: k, cnt })).sort((a, b) => b.cnt - a.cnt);
+ const text = JSON.stringify(next);
+ return text === json ? null : text;
+}
+
+/** Scrub top_paths / top_referers in both rollup tables. Returns { hourly, daily } rows changed. */
+function scrubRollups(db, { paramPrefixes, pathRules } = {}) {
+ const opts = privacy.pathOptions({ paramPrefixes, pathRules });
+ const tpl = (p) => (p == null ? null : privacy.normalisePath(p, opts));
+ const out = { hourly: 0, daily: 0 };
+ for (const table of ['analytics_hourly', 'analytics_daily']) {
+ if (!tableExists(db, table)) continue;
+ const rows = db.prepare(`SELECT id, top_paths, top_referers FROM ${table}`).all();
+ const upd = db.prepare(`UPDATE ${table} SET top_paths = ?, top_referers = ? WHERE id = ?`);
+ db.transaction(() => {
+ for (const r of rows) {
+ const paths = r.top_paths ? reduceTopList(r.top_paths, 'path', tpl) : null;
+ const refs = r.top_referers ? reduceTopList(r.top_referers, 'referer', privacy.refererOrigin) : null;
+ if (paths == null && refs == null) continue;
+ upd.run(paths != null ? paths : r.top_paths, refs != null ? refs : r.top_referers, r.id);
+ out[table === 'analytics_hourly' ? 'hourly' : 'daily']++;
+ }
+ })();
+ }
+ return out;
+}
+
+/** Sums of every rollup counter; equal before and after a prune or scrub. */
+function rollupTotals(db) {
+ const out = {};
+ if (tableExists(db, 'analytics_hourly')) {
+ out.hourly = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
+ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(bot_hits) AS bot_hits,
+ TOTAL(error_count) AS error_count FROM analytics_hourly`).get();
+ }
+ if (tableExists(db, 'analytics_daily')) {
+ out.daily = db.prepare(`SELECT COUNT(*) AS rows, TOTAL(pageviews) AS pageviews, TOTAL(api_calls) AS api_calls,
+ TOTAL(unique_visitors) AS unique_visitors, TOTAL(unique_users) AS unique_users, TOTAL(new_users) AS new_users,
+ TOTAL(bot_hits) AS bot_hits, TOTAL(error_count) AS error_count FROM analytics_daily`).get();
+ }
+ return out;
+}
+
+/** Read-only counts for a dry run. */
+function inspect(db, { days = MAX_DAYS, now = Date.now, paramPrefixes, pathRules } = {}) {
+ const cutoff = cutoffFor(days, now());
+ const out = { cutoff, events: 0, older: 0, remaining: 0, oldest: null, newest: null };
+ const pageSize = db.pragma('page_size', { simple: true });
+ out.bytes = db.pragma('page_count', { simple: true }) * pageSize;
+ out.freeBytes = db.pragma('freelist_count', { simple: true }) * pageSize;
+ out.rollups = rollupTotals(db);
+ if (!tableExists(db, 'analytics_events')) return out;
+ registerFunctions(db, { paramPrefixes, pathRules });
+ const e = db.prepare('SELECT COUNT(*) AS n, MIN(created_at) AS oldest, MAX(created_at) AS newest FROM analytics_events').get();
+ out.events = e.n;
+ out.oldest = e.oldest;
+ out.newest = e.newest;
+ out.older = db.prepare('SELECT COUNT(*) FROM analytics_events WHERE created_at < ?').pluck().get(cutoff);
+ out.remaining = out.events - out.older;
+ const personal = ['ip', 'user_id', 'city'].filter((c) => columnExists(db, 'analytics_events', c))
+ .map((c) => `${c} IS NOT NULL`).join(' OR ') || '0';
+ const r = db.prepare(`
+ SELECT
+ COUNT(*) FILTER (WHERE ${personal}) AS personal,
+ COUNT(*) FILTER (WHERE path IS NOT ov_route_template(path)) AS paths,
+ COUNT(*) FILTER (WHERE referer IS NOT ov_referer_origin(referer)) AS referers,
+ COUNT(*) FILTER (WHERE user_agent IS NOT ov_ua_class(user_agent)) AS user_agents,
+ COUNT(*) FILTER (WHERE session_id IS NOT ov_rotating_session(session_id)) AS sessions
+ FROM analytics_events WHERE created_at >= ?
+ `).get(cutoff);
+ out.toScrub = r;
+ out.rateRows = tableExists(db, 'analytics_rate_tracking') ? db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get() : 0;
+ return out;
+}
+
+/**
+ * Run pruneRawEvents(db, { days }) on timers: first after `initialDelayMs` (default 5 min), then every
+ * `intervalMs` (default 24 h). Timers are unref'd, so they never hold the process open. An overlapping
+ * run is skipped (resolves null). `label` names the service in log lines.
+ * Returns { run, stop }: run() prunes now (resolves the prune result, or null on error/overlap).
+ */
+function schedulePrune(db, {
+ days = MAX_DAYS, batchSize = DEFAULT_BATCH, initialDelayMs = 5 * 60 * 1000, intervalMs = 24 * 60 * 60 * 1000,
+ label = 'Analytics', log = console.log, logError = console.error, now = Date.now,
+} = {}) {
+ checkDays(days);
+ let running = false;
+ const run = async () => {
+ if (running) return null;
+ running = true;
+ try {
+ const out = await pruneRawEvents(db, { days, batchSize, now });
+ if (out.deleted) log(`[${label}] pruned ${out.deleted} raw events older than ${out.cutoff}`);
+ return out;
+ } catch (err) {
+ logError(`[${label}] analytics-prune error:`, err.message);
+ return null;
+ } finally {
+ running = false;
+ }
+ };
+ const first = setTimeout(run, initialDelayMs);
+ const every = setInterval(run, intervalMs);
+ if (first.unref) first.unref();
+ if (every.unref) every.unref();
+ return { run, stop() { clearTimeout(first); clearInterval(every); } };
+}
+
+module.exports = {
+ MAX_DAYS,
+ DEFAULT_BATCH,
+ checkDays,
+ cutoffFor,
+ pruneRawEvents,
+ scrubEvents,
+ scrubRollups,
+ rollupTotals,
+ inspect,
+ reduceTopList,
+ schedulePrune,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,123 @@
@@ -0,0 +1,123 @@
+'use strict';
+/**
+ * Analytics database schema (ADR-021). Same tables as the openvibe-shared tracker it replaces, so the
+ * existing analytics.db files and the admin dashboards keep working; the columns that used to hold
+ * personal data (ip, user_id, city) stay in the table for compatibility but are always NULL now, and
+ * user_agent / referer / path hold the reduced forms (class, origin, route template).
+ *
+ * Added:
+ * analytics_events.authenticated 0/1 — whether the request was signed in (a flag, never who)
+ * analytics_visitor_days the day's salted visitor hashes, for the unique-visitor rollups;
+ * rows are deleted once the day's rollup is final (≤ ~25 h)
+ * analytics_day_salts the random salt of the current UTC day (so a restart does not
+ * double-count); deleted together with that day's hashes
+ * Emptied: analytics_rate_tracking (the rate check keeps per-IP counters in memory only now)
+ */
+
+const ANALYTICS_SCHEMA = `
+ CREATE TABLE IF NOT EXISTS analytics_events (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ event_type TEXT NOT NULL DEFAULT 'pageview',
+ path TEXT,
+ method TEXT DEFAULT 'GET',
+ status_code INTEGER,
+ response_time_ms INTEGER,
+ user_id INTEGER,
+ session_id TEXT,
+ ip TEXT,
+ country TEXT,
+ city TEXT,
+ user_agent TEXT,
+ referer TEXT,
+ is_bot INTEGER DEFAULT 0,
+ bot_type TEXT,
+ device_type TEXT,
+ browser TEXT,
+ os TEXT,
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_hourly (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ hour TEXT NOT NULL,
+ pageviews INTEGER DEFAULT 0,
+ api_calls INTEGER DEFAULT 0,
+ unique_visitors INTEGER DEFAULT 0,
+ unique_users INTEGER DEFAULT 0,
+ bot_hits INTEGER DEFAULT 0,
+ avg_response_ms INTEGER DEFAULT 0,
+ error_count INTEGER DEFAULT 0,
+ bandwidth_bytes INTEGER DEFAULT 0,
+ top_paths TEXT,
+ top_referers TEXT,
+ UNIQUE(service, hour)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_daily (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ service TEXT NOT NULL,
+ date TEXT NOT NULL,
+ pageviews INTEGER DEFAULT 0,
+ api_calls INTEGER DEFAULT 0,
+ unique_visitors INTEGER DEFAULT 0,
+ unique_users INTEGER DEFAULT 0,
+ new_users INTEGER DEFAULT 0,
+ bot_hits INTEGER DEFAULT 0,
+ avg_response_ms INTEGER DEFAULT 0,
+ error_count INTEGER DEFAULT 0,
+ top_paths TEXT,
+ top_referers TEXT,
+ top_countries TEXT,
+ device_breakdown TEXT,
+ browser_breakdown TEXT,
+ UNIQUE(service, date)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_rate_tracking (
+ ip TEXT NOT NULL,
+ window_start INTEGER NOT NULL,
+ hit_count INTEGER DEFAULT 1,
+ PRIMARY KEY (ip, window_start)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_visitor_days (
+ service TEXT NOT NULL,
+ day TEXT NOT NULL,
+ hour TEXT NOT NULL,
+ vhash TEXT NOT NULL,
+ authenticated INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (service, hour, vhash)
+ );
+
+ CREATE TABLE IF NOT EXISTS analytics_day_salts (
+ service TEXT NOT NULL,
+ day TEXT NOT NULL,
+ salt TEXT NOT NULL,
+ PRIMARY KEY (service, day)
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_service ON analytics_events(service, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_path ON analytics_events(service, path, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_events_bot ON analytics_events(is_bot, created_at);
+ CREATE INDEX IF NOT EXISTS idx_analytics_hourly_lookup ON analytics_hourly(service, hour);
+ CREATE INDEX IF NOT EXISTS idx_analytics_daily_lookup ON analytics_daily(service, date);
+ CREATE INDEX IF NOT EXISTS idx_analytics_visitor_days_day ON analytics_visitor_days(service, day);
+
+ -- Indexes over the retired personal columns.
+ DROP INDEX IF EXISTS idx_analytics_events_ip;
+ DROP INDEX IF EXISTS idx_analytics_events_user;
+`;
+
+/** Create or upgrade the analytics tables in place. Idempotent; never touches rows. */
+function ensureSchema(db) {
+ db.exec(ANALYTICS_SCHEMA);
+ const cols = db.prepare('PRAGMA table_info(analytics_events)').all().map((c) => c.name);
+ if (!cols.includes('authenticated')) {
+ db.exec('ALTER TABLE analytics_events ADD COLUMN authenticated INTEGER NOT NULL DEFAULT 0');
+ }
+}
+
+module.exports = { ANALYTICS_SCHEMA, ensureSchema };
new file mode 100644
new file mode 100644
@@ -0,0 +1,753 @@
@@ -0,0 +1,753 @@
+'use strict';
+/**
+ * Request analytics within the ADR-021 privacy bounds. Drop-in replacement for the pre-ADR
+ * openvibe-shared AnalyticsTracker (same constructor, tables and getStats/getOverview/getBotAnalysis
+ * shapes), minus the personal data:
+ *
+ * raw row service, event_type, route template, method, status, response time, rotating session
+ * id, country (CDN header), user-agent class + browser/os/device, referer ORIGIN,
+ * bot flags, signed-in flag, timestamp. ip / user_id / city are always NULL.
+ * session id random 16 hex chars, held in memory against the day's visitor hash; a new one after
+ * 30 minutes idle and at every UTC midnight. Never derived from, or equal to, a user id.
+ * rate check per-IP hit counters for the current and previous minute, IN MEMORY only (dropped
+ * after 2 minutes idle; analytics_rate_tracking is emptied on boot and no longer written).
+ * uniques visitor hash = HMAC-SHA256(day salt, ip + "\n" + user agent), first 16 hex chars.
+ * The salt is random per UTC day and service (analytics_day_salts, deleted as soon as
+ * the day is over); hashes go to analytics_visitor_days only, never to raw events,
+ * and are deleted by the first aggregation after their day ends, right after that day's
+ * final hourly/daily rollup — so no hash or salt outlives its day by more than about
+ * one aggregation interval (1 h). Rollups keep only the counts.
+ *
+ * opt-out a request with `Sec-GPC: 1` or `DNT: 1` is not recorded at all (privacy.optedOut):
+ * no row, no visitor hash, no session id, no rate counter.
+ *
+ * Retention of raw events (30 days) is ./retention.js; this tracker writes, rolls up and (with
+ * opts.retention) schedules the prune. The row shape is analytics/event.v1 (./event.js).
+ */
+
+const crypto = require('crypto');
+const { ensureSchema } = require('./schema');
+const privacy = require('./privacy');
+const retention = require('./retention');
+const { allowed } = require('./event');
+
+const SESSION_IDLE_MS = 30 * 60 * 1000;
+const MAX_TRACKED_CLIENTS = 200000;
+const MAX_BUFFER = 5000;
+const SESSION_ID_RE = /^[0-9a-f]{16}$/;
+const METHOD_RE = /^[A-Z][A-Z-]{0,15}$/;
+const EVENT_NAME_RE = /^[A-Za-z][A-Za-z0-9_.:-]{0,63}$/;
+const ALLOWED = { device: allowed('device'), browser: allowed('browser'), os: allowed('os') };
+
+const pad = (n) => String(n).padStart(2, '0');
+/** 'YYYY-MM-DD HH:MM:SS' in UTC, the format of CURRENT_TIMESTAMP. */
+function sqlTime(ms) {
+ const d = new Date(ms);
+ return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
+}
+const dayOf = (ms) => sqlTime(ms).slice(0, 10);
+const hourOf = (ms) => sqlTime(ms).slice(0, 13) + ':00:00';
+const parseSqlTime = (s) => Date.parse(s.replace(' ', 'T') + 'Z');
+
+function isStaticOrHealth(path) {
+ return path === '/api/health' || path === '/health' ||
+ /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i.test(path);
+}
+
+class AnalyticsTracker {
+ /**
+ * @param {object} db better-sqlite3 handle (its own analytics database)
+ * @param {string} service service name ('live', 'openvibe-yt', …)
+ * @param {object} [opts]
+ * @param {string[]} [opts.paramPrefixes] extra words whose next path segment is a parameter
+ * (added to privacy.DEFAULT_PARAM_PREFIXES), e.g. Network's ['avatar', 'anon', 'projects']
+ * @param {Array<[RegExp, string]>} [opts.pathRules] rewrites applied to a raw path before it is
+ * templated, for parameters the segment rules cannot see (privacy.normalisePath)
+ * @param {false|object} [opts.retention] { days = 30, intervalMs = 24 h, initialDelayMs = 5 min }:
+ * the raw-event prune on a timer (retention.schedulePrune; the job is this.pruneJob). On by
+ * default (ADR-021: raw events at most 30 days); pass false only when the service schedules
+ * retention.pruneRawEvents itself.
+ * @param {boolean} [opts.timers=true] false: no flush/aggregate timers (tests call flush/aggregate)
+ * @param {function} [opts.now] clock (tests)
+ */
+ constructor(db, service, opts = {}) {
+ this.db = db;
+ this.service = service;
+ this._now = opts.now || Date.now;
+ this._pathOpts = privacy.pathOptions({ paramPrefixes: opts.paramPrefixes, pathRules: opts.pathRules });
+ this.pruneJob = null;
+ this._buffer = [];
+ this._visitors = new Map(); // `${hour}|${vhash}` -> { day, hour, vhash, authenticated }
+ this._sessions = new Map(); // vhash -> { sid, last }
+ this._rates = new Map(); // client ip -> { minute, count, prev } (memory only)
+ this._saltDay = null;
+ this._salt = null;
+ this._timers = [];
+
+ ensureSchema(db);
+ try { db.pragma('secure_delete = ON'); } catch { /* not fatal */ }
+ try { db.pragma('busy_timeout = 250'); } catch { /* not fatal */ }
+ db.exec('DELETE FROM analytics_rate_tracking');
+
+ this._insertEvent = db.prepare(`
+ INSERT INTO analytics_events
+ (service, event_type, path, method, status_code, response_time_ms,
+ user_id, session_id, ip, country, city, user_agent, referer,
+ is_bot, bot_type, device_type, browser, os, authenticated, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `);
+ this._upsertVisitor = db.prepare(`
+ INSERT INTO analytics_visitor_days (service, day, hour, vhash, authenticated) VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(service, hour, vhash) DO UPDATE SET authenticated = MAX(authenticated, excluded.authenticated)
+ `);
+ this._write = db.transaction((events, visitors) => {
+ for (const e of events) {
+ this._insertEvent.run(
+ e.service, e.event_type, e.path, e.method, e.status_code, e.response_time_ms,
+ e.session_id, e.country, e.user_agent, e.referer,
+ e.is_bot ? 1 : 0, e.bot_type, e.device_type, e.browser, e.os, e.authenticated ? 1 : 0, e.created_at,
+ );
+ }
+ for (const v of visitors) this._upsertVisitor.run(this.service, v.day, v.hour, v.vhash, v.authenticated ? 1 : 0);
+ });
+
+ if (opts.timers !== false) {
+ const every = (fn, ms) => { const t = setInterval(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
+ const once = (fn, ms) => { const t = setTimeout(fn, ms); if (t.unref) t.unref(); this._timers.push(t); };
+ every(() => { this.flush(); this._sweep(); }, 5000);
+ every(() => this.aggregate(), 60 * 60 * 1000);
+ once(() => this.aggregate(), 10_000);
+ if (opts.retention !== false) {
+ const r = opts.retention && typeof opts.retention === 'object' ? opts.retention : {};
+ this.pruneJob = retention.schedulePrune(db, { label: `Analytics:${service}`, now: this._now, ...r });
+ }
+ }
+ }
+
+ // ── Per-request reducers ──────────────────────────────────
+
+ _daySalt(day) {
+ if (this._saltDay === day) return this._salt;
+ let row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
+ if (!row) {
+ this.db.prepare('INSERT OR IGNORE INTO analytics_day_salts (service, day, salt) VALUES (?, ?, ?)')
+ .run(this.service, day, crypto.randomBytes(32).toString('hex'));
+ row = this.db.prepare('SELECT salt FROM analytics_day_salts WHERE service = ? AND day = ?').get(this.service, day);
+ }
+ // Earlier days' salts are never needed again: their hashes are already written.
+ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, day);
+ this._saltDay = day;
+ this._salt = row.salt;
+ this._sessions.clear();
+ return this._salt;
+ }
+
+ _visitorHash(ip, ua, day) {
+ return crypto.createHmac('sha256', this._daySalt(day)).update(`${ip}\n${ua}`).digest('hex').slice(0, 16);
+ }
+
+ _sessionId(vhash, nowMs) {
+ let s = this._sessions.get(vhash);
+ if (!s || nowMs - s.last > SESSION_IDLE_MS) {
+ s = { sid: crypto.randomBytes(8).toString('hex'), last: nowMs };
+ this._sessions.set(vhash, s);
+ }
+ s.last = nowMs;
+ return s.sid;
+ }
+
+ /** Hits from this client in the current and previous minute (memory only). */
+ _rateHit(ip, nowMs) {
+ const minute = Math.floor(nowMs / 60000);
+ let r = this._rates.get(ip);
+ if (!r) { r = { minute, count: 0, prev: 0 }; this._rates.set(ip, r); }
+ if (r.minute !== minute) {
+ r.prev = r.minute === minute - 1 ? r.count : 0;
+ r.count = 0;
+ r.minute = minute;
+ }
+ r.count++;
+ return r.count + r.prev;
+ }
+
+ _sweep() {
+ const nowMs = this._now();
+ const minute = Math.floor(nowMs / 60000);
+ for (const [ip, r] of this._rates) if (r.minute < minute - 1) this._rates.delete(ip);
+ for (const [h, s] of this._sessions) if (nowMs - s.last > SESSION_IDLE_MS) this._sessions.delete(h);
+ if (this._rates.size > MAX_TRACKED_CLIENTS) this._rates.clear();
+ if (this._sessions.size > MAX_TRACKED_CLIENTS) this._sessions.clear();
+ }
+
+ _push(event, visitor) {
+ if (this._buffer.length >= MAX_BUFFER) this._buffer.shift();
+ this._buffer.push(event);
+ if (visitor) {
+ const key = `${visitor.hour}|${visitor.vhash}`;
+ const prev = this._visitors.get(key);
+ if (!prev) this._visitors.set(key, visitor);
+ else if (visitor.authenticated) prev.authenticated = true;
+ }
+ if (this._buffer.length >= 100) this.flush();
+ }
+
+ /**
+ * Record one finished request. `entryPath` is the path as it arrived (req.path inside a mounted
+ * router is relative by the time the response finishes). Returns false, recording nothing, for an
+ * opted-out request (Sec-GPC: 1 / DNT: 1); true otherwise.
+ */
+ record(req, res, entryPath, responseTimeMs) {
+ const headers = req.headers || {};
+ if (privacy.optedOut(headers)) return false;
+ entryPath = String(entryPath || '');
+ const nowMs = this._now();
+ const ua = String(headers['user-agent'] || '');
+ const ip = String(req.ip || (req.socket && req.socket.remoteAddress) || '');
+ const bot = privacy.classifyRequest({ headers, path: entryPath });
+ if (this._rateHit(ip, nowMs) > privacy.HIGH_REQUEST_RATE && !bot.isBot) {
+ bot.isBot = true;
+ bot.botType = 'rate_limit';
+ bot.confidence = 0.80;
+ }
+ const parsed = privacy.parseUserAgent(ua);
+ const day = dayOf(nowMs);
+ const vhash = this._visitorHash(ip, ua, day);
+ const authenticated = !!(req.user && (req.user.id != null || req.user.sub != null));
+ const isApi = entryPath.startsWith('/api/') || entryPath.startsWith('/internal/') || entryPath.startsWith('/oauth/');
+ this._push({
+ service: this.service,
+ event_type: isApi ? 'api_call' : 'pageview',
+ path: privacy.routeTemplate(req, entryPath, this._pathOpts),
+ method: METHOD_RE.test(String(req.method || '')) ? req.method : null,
+ status_code: res.statusCode,
+ response_time_ms: responseTimeMs,
+ session_id: this._sessionId(vhash, nowMs),
+ country: privacy.countryCode(headers['cf-ipcountry']),
+ user_agent: privacy.uaClass(ua),
+ referer: privacy.refererOrigin(headers.referer || headers.referrer),
+ is_bot: bot.isBot,
+ bot_type: bot.botType,
+ device_type: parsed.device,
+ browser: parsed.browser,
+ os: parsed.os,
+ authenticated,
+ created_at: sqlTime(nowMs),
+ }, { day, hour: hourOf(nowMs), vhash, authenticated });
+ return true;
+ }
+
+ /**
+ * Express middleware: one row per finished request (health checks, static assets and opted-out
+ * requests skipped).
+ */
+ middleware() {
+ return (req, res, next) => {
+ const entryPath = req.path || '';
+ if (isStaticOrHealth(entryPath) || privacy.optedOut(req.headers)) return next();
+ const started = Date.now();
+ let done = false;
+ const finish = () => {
+ if (done) return;
+ done = true;
+ try { this.record(req, res, entryPath, Date.now() - started); }
+ catch (err) { console.error(`[Analytics:${this.service}] record error:`, err.message); }
+ };
+ res.once('finish', finish);
+ res.once('close', finish);
+ next();
+ };
+ }
+
+ /**
+ * A custom (non-HTTP) event. Personal fields in `data` (ip, user_id, city) are ignored; path,
+ * referer and user_agent are reduced like a request's; session_id is kept only if it is one of
+ * this tracker's rotating ids; any other value outside analytics/event.v1 is stored as NULL, and
+ * an event name outside it throws a TypeError. Pass the triggering request's headers as
+ * `data.headers` when there is one: an opted-out request (Sec-GPC / DNT) records nothing.
+ * Returns whether it recorded.
+ */
+ trackEvent(eventType, data = {}) {
+ const name = String(eventType == null ? 'event' : eventType);
+ if (!EVENT_NAME_RE.test(name)) throw new TypeError(`analytics event name must match ${EVENT_NAME_RE} (analytics/event.v1)`);
+ if (privacy.optedOut(data.headers)) return false;
+ const nowMs = this._now();
+ const ua = data.user_agent ? String(data.user_agent) : '';
+ const parsed = ua ? privacy.parseUserAgent(ua) : {};
+ const method = data.method ? String(data.method).toUpperCase() : null;
+ const status = Number(data.status_code);
+ const ms = Number(data.response_time_ms);
+ const botType = data.bot_type ? String(data.bot_type) : null;
+ const pick = (field, given, fallback) => (ALLOWED[field].includes(given) ? given : (fallback || null));
+ this._push({
+ service: this.service,
+ event_type: name,
+ path: data.path ? privacy.normalisePath(data.path, this._pathOpts) : null,
+ method: method && METHOD_RE.test(method) ? method : null,
+ status_code: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,
+ response_time_ms: Number.isFinite(ms) && ms >= 0 ? Math.round(ms) : null,
+ session_id: SESSION_ID_RE.test(String(data.session_id || '')) ? data.session_id : null,
+ country: privacy.countryCode(data.country),
+ user_agent: ua ? privacy.uaClass(ua) : null,
+ referer: privacy.refererOrigin(data.referer),
+ is_bot: !!data.is_bot,
+ bot_type: botType && /^[a-z0-9_]{1,32}$/.test(botType) ? botType : null,
+ device_type: pick('device', data.device_type, parsed.device),
+ browser: pick('browser', data.browser, parsed.browser),
+ os: pick('os', data.os, parsed.os),
+ authenticated: !!data.authenticated,
+ created_at: sqlTime(nowMs),
+ }, null);
+ return true;
+ }
+
+ /** Write buffered events and visitor hashes. On a busy database they are kept for the next flush. */
+ flush() {
+ if (!this._buffer.length && !this._visitors.size) return;
+ const events = this._buffer.splice(0);
+ const visitors = [...this._visitors.values()];
+ this._visitors.clear();
+ try {
+ this._write(events, visitors);
+ } catch (err) {
+ console.error(`[Analytics:${this.service}] Flush error:`, err.message);
+ this._buffer = events.slice(-MAX_BUFFER);
+ for (const v of visitors) this._visitors.set(`${v.hour}|${v.vhash}`, v);
+ }
+ }
+
+ // ── Rollups ───────────────────────────────────────────────
+
+ /**
+ * Hourly + daily rollups for the last 2 hours / 2 days, plus every earlier day that still has
+ * visitor hashes (finalised here), after which those hashes are deleted.
+ */
+ aggregate() {
+ try {
+ this.flush();
+ const nowMs = this._now();
+ const today = dayOf(nowMs);
+ const hours = new Set([hourOf(nowMs), hourOf(nowMs - 3600000)]);
+ const days = new Set([today, dayOf(nowMs - 86400000)]);
+ const pending = this.db.prepare('SELECT DISTINCT day, hour FROM analytics_visitor_days WHERE service = ? AND day < ?').all(this.service, today);
+ for (const p of pending) { hours.add(p.hour); days.add(p.day); }
+ for (const h of hours) this._aggregateHour(h);
+ for (const d of days) this._aggregateDay(d);
+ this.db.prepare('DELETE FROM analytics_visitor_days WHERE service = ? AND day < ?').run(this.service, today);
+ this.db.prepare('DELETE FROM analytics_day_salts WHERE service = ? AND day < ?').run(this.service, today);
+ } catch (err) {
+ console.error(`[Analytics:${this.service}] Aggregation error:`, err.message);
+ }
+ }
+
+ _aggregateHour(hourStart) {
+ const nextHour = sqlTime(parseSqlTime(hourStart) + 3600000);
+ const svc = this.service;
+ const stats = this.db.prepare(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ?
+ `).get(svc, hourStart, nextHour);
+ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
+
+ // Uniques only grow within a period, so never lower a stored count (restarts, or rows
+ // counted before this tracker when the old one still stored IPs).
+ const v = this.db.prepare('SELECT COUNT(*) AS visitors, COALESCE(SUM(authenticated), 0) AS users FROM analytics_visitor_days WHERE service = ? AND hour = ?').get(svc, hourStart);
+ const prev = this.db.prepare('SELECT unique_visitors, unique_users FROM analytics_hourly WHERE service = ? AND hour = ?').get(svc, hourStart) || {};
+
+ const topPaths = this.db.prepare(`
+ SELECT path, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ? AND is_bot = 0
+ GROUP BY path ORDER BY cnt DESC LIMIT 10
+ `).all(svc, hourStart, nextHour);
+ const topReferers = this.db.prepare(`
+ SELECT referer, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at < ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0
+ GROUP BY referer ORDER BY cnt DESC LIMIT 10
+ `).all(svc, hourStart, nextHour);
+
+ this.db.prepare(`
+ INSERT INTO analytics_hourly
+ (service, hour, pageviews, api_calls, unique_visitors, unique_users, bot_hits, avg_response_ms, error_count, top_paths, top_referers)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(service, hour) DO UPDATE SET
+ pageviews = excluded.pageviews, api_calls = excluded.api_calls,
+ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
+ bot_hits = excluded.bot_hits, avg_response_ms = excluded.avg_response_ms,
+ error_count = excluded.error_count, top_paths = excluded.top_paths, top_referers = excluded.top_referers
+ `).run(
+ svc, hourStart, stats.pageviews, stats.api_calls,
+ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
+ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
+ JSON.stringify(topPaths), JSON.stringify(topReferers),
+ );
+ }
+
+ _aggregateDay(date) {
+ const dayStart = date + ' 00:00:00';
+ const dayEnd = date + ' 23:59:59';
+ const svc = this.service;
+ const q = (sql) => this.db.prepare(sql);
+ const stats = q(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_calls,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS error_count
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND created_at <= ?
+ `).get(svc, dayStart, dayEnd);
+ if (!stats || (stats.pageviews === 0 && stats.api_calls === 0)) return;
+
+ const v = q(`
+ SELECT COUNT(DISTINCT vhash) AS visitors,
+ COUNT(DISTINCT CASE WHEN authenticated = 1 THEN vhash END) AS users
+ FROM analytics_visitor_days WHERE service = ? AND day = ?
+ `).get(svc, date);
+ const prev = q('SELECT unique_visitors, unique_users, new_users FROM analytics_daily WHERE service = ? AND date = ?').get(svc, date) || {};
+
+ const range = [svc, dayStart, dayEnd];
+ const topPaths = q(`SELECT path, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY path ORDER BY cnt DESC LIMIT 15`).all(...range);
+ const topReferers = q(`SELECT referer, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND referer != '' AND referer IS NOT NULL AND is_bot = 0 GROUP BY referer ORDER BY cnt DESC LIMIT 10`).all(...range);
+ const topCountries = q(`SELECT country, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND country IS NOT NULL AND is_bot = 0 GROUP BY country ORDER BY cnt DESC LIMIT 10`).all(...range);
+ const devices = q(`SELECT device_type, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY device_type ORDER BY cnt DESC`).all(...range);
+ const browsers = q(`SELECT browser, COUNT(*) as cnt FROM analytics_events WHERE service = ? AND created_at >= ? AND created_at <= ? AND is_bot = 0 GROUP BY browser ORDER BY cnt DESC`).all(...range);
+
+ q(`
+ INSERT INTO analytics_daily
+ (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, bot_hits,
+ avg_response_ms, error_count, top_paths, top_referers, top_countries, device_breakdown, browser_breakdown)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(service, date) DO UPDATE SET
+ pageviews = excluded.pageviews, api_calls = excluded.api_calls,
+ unique_visitors = excluded.unique_visitors, unique_users = excluded.unique_users,
+ new_users = excluded.new_users, bot_hits = excluded.bot_hits,
+ avg_response_ms = excluded.avg_response_ms, error_count = excluded.error_count,
+ top_paths = excluded.top_paths, top_referers = excluded.top_referers,
+ top_countries = excluded.top_countries, device_breakdown = excluded.device_breakdown,
+ browser_breakdown = excluded.browser_breakdown
+ `).run(
+ svc, date, stats.pageviews, stats.api_calls,
+ Math.max(prev.unique_visitors || 0, v.visitors || 0), Math.max(prev.unique_users || 0, v.users || 0),
+ // New users needs a user id history, which analytics no longer has: keep what was
+ // recorded before, NULL (not measured) from now on.
+ prev.new_users != null ? prev.new_users : null,
+ stats.bot_hits, stats.avg_response_ms || 0, stats.error_count,
+ JSON.stringify(topPaths), JSON.stringify(topReferers), JSON.stringify(topCountries),
+ JSON.stringify(devices), JSON.stringify(browsers),
+ );
+ }
+
+ // ── Reads (admin dashboards) ─────────────────────────────
+ // Same shapes as before. Where the old queries counted distinct IPs or user ids over raw events,
+ // these count distinct rotating session ids (all sessions / signed-in sessions); rollup-based
+ // numbers (daily/hourly unique_visitors) are true unique visitors per period.
+
+ getStats(options = {}) {
+ const { days = 30, hours, service } = options;
+ const svc = service || this.service;
+ const nowMs = this._now();
+ const rangeMs = hours ? hours * 3600000 : days * 86400000;
+ const rangeCutoff = sqlTime(nowMs - rangeMs);
+ const isSubDay = hours && hours < 24;
+ const effectiveDays = hours ? hours / 24 : days;
+ const bucketMin = hours <= 1 ? 5 : hours <= 12 ? 15 : 30;
+ const bucketExpr = `strftime('%Y-%m-%d %H:', created_at) || CAST((CAST(strftime('%M', created_at) AS INTEGER) / ${bucketMin}) * ${bucketMin} AS TEXT)`;
+ const q = (sql) => this.db.prepare(sql);
+ const safe = (fn, fallback) => { try { return fn() || fallback; } catch { return fallback; } };
+
+ let daily = [];
+ if (!isSubDay) {
+ daily = q('SELECT * FROM analytics_daily WHERE service = ? AND date >= ? ORDER BY date ASC').all(svc, dayOf(nowMs - rangeMs));
+ }
+
+ const hourlyLimit = hours ? Math.min(hours, 168) : 24;
+ const hourlyCutoff = sqlTime(nowMs - hourlyLimit * 3600000);
+ const hourly = q('SELECT * FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC').all(svc, hourlyCutoff);
+
+ let timeBuckets = [];
+ if (isSubDay) {
+ timeBuckets = q(`
+ SELECT ${bucketExpr} as bucket,
+ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS api_calls,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS unique_visitors,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS bot_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ?
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(svc, rangeCutoff);
+ }
+
+ let summary;
+ if (hours && hours < 48) {
+ summary = q(`
+ SELECT
+ COUNT(*) FILTER (WHERE event_type = 'pageview' AND is_bot = 0) AS total_pageviews,
+ COUNT(*) FILTER (WHERE event_type = 'api_call' AND is_bot = 0) AS total_api_calls,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0) AS total_unique_visitors,
+ COUNT(DISTINCT session_id) FILTER (WHERE is_bot = 0 AND authenticated = 1) AS total_unique_users,
+ COUNT(*) FILTER (WHERE is_bot = 1) AS total_bot_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS total_errors,
+ CAST(AVG(response_time_ms) AS INTEGER) AS avg_response_ms
+ FROM analytics_events
+ WHERE service = ? AND created_at >= ?
+ `).get(svc, rangeCutoff);
+ } else {
+ summary = q(`
+ SELECT
+ SUM(pageviews) as total_pageviews,
+ SUM(api_calls) as total_api_calls,
+ SUM(unique_visitors) as total_unique_visitors,
+ SUM(unique_users) as total_unique_users,
+ SUM(bot_hits) as total_bot_hits,
+ SUM(error_count) as total_errors,
+ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
+ FROM analytics_daily
+ WHERE service = ? AND date >= ?
+ `).get(svc, dayOf(nowMs - rangeMs));
+ }
+
+ const realtime = q(`
+ SELECT COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
+ FROM analytics_events WHERE service = ? AND created_at >= ?
+ `).get(svc, sqlTime(nowMs - 300000));
+
+ const human = [svc, rangeCutoff];
+ const topPages = q(`
+ SELECT path, COUNT(*) as hits, COUNT(DISTINCT session_id) as visitors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY path ORDER BY hits DESC LIMIT 20
+ `).all(...human);
+ const topReferers = q(`
+ SELECT referer, COUNT(*) as hits FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND referer IS NOT NULL AND referer != '' AND is_bot = 0
+ GROUP BY referer ORDER BY hits DESC LIMIT 15
+ `).all(...human);
+ const botBreakdown = q(`
+ SELECT bot_type, COUNT(*) as hits FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND is_bot = 1
+ GROUP BY bot_type ORDER BY hits DESC LIMIT 15
+ `).all(...human);
+ const statusCodes = q(`
+ SELECT CASE
+ WHEN status_code >= 200 AND status_code < 300 THEN '2xx'
+ WHEN status_code >= 300 AND status_code < 400 THEN '3xx'
+ WHEN status_code >= 400 AND status_code < 500 THEN '4xx'
+ WHEN status_code >= 500 THEN '5xx'
+ ELSE 'other' END as group_code,
+ COUNT(*) as cnt
+ FROM analytics_events WHERE service = ? AND created_at >= ?
+ GROUP BY group_code ORDER BY cnt DESC
+ `).all(...human);
+ const breakdown = (col) => q(`
+ SELECT ${col}, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY ${col} ORDER BY cnt DESC
+ `).all(...human);
+ const deviceBreakdown = breakdown('device_type');
+ const browserBreakdown = breakdown('browser');
+ const osBreakdown = breakdown('os');
+ const countryBreakdown = q(`
+ SELECT country, COUNT(*) as cnt FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND country IS NOT NULL AND is_bot = 0
+ GROUP BY country ORDER BY cnt DESC LIMIT 20
+ `).all(...human);
+
+ const responsePercentiles = safe(() => {
+ const vals = q(`
+ SELECT response_time_ms FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
+ ORDER BY response_time_ms ASC
+ `).pluck().all(...human);
+ if (!vals.length) return {};
+ const pct = (p) => vals[Math.min(Math.floor(vals.length * p), vals.length - 1)];
+ return { p50: pct(0.5), p90: pct(0.9), p95: pct(0.95), p99: pct(0.99), max: vals[vals.length - 1] };
+ }, {});
+
+ const bandwidth = safe(() => {
+ const bw = q(`
+ SELECT COUNT(*) as total_requests,
+ COUNT(*) FILTER (WHERE event_type = 'pageview') AS page_requests,
+ COUNT(*) FILTER (WHERE event_type = 'api_call') AS api_requests,
+ SUM(CASE WHEN event_type = 'pageview' THEN 50000 ELSE 2000 END) AS estimated_bytes
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ `).get(...human);
+ return bw && {
+ total_requests: bw.total_requests || 0,
+ page_requests: bw.page_requests || 0,
+ api_requests: bw.api_requests || 0,
+ estimated_bytes: bw.estimated_bytes || 0,
+ };
+ }, {});
+
+ const authBreakdown = safe(() => q(`
+ SELECT
+ COUNT(*) FILTER (WHERE authenticated = 1) AS authenticated,
+ COUNT(*) FILTER (WHERE authenticated = 0) AS anonymous,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS unique_authenticated_users,
+ COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL) AS total_sessions
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ `).get(...human), {});
+
+ const errorTrend = safe(() => (isSubDay
+ ? q(`
+ SELECT ${bucketExpr} as bucket, COUNT(*) AS total,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors,
+ COUNT(*) FILTER (WHERE status_code >= 500) AS server_errors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(...human)
+ : q(`
+ SELECT hour as bucket, (pageviews + api_calls) as total, error_count as errors
+ FROM analytics_hourly WHERE service = ? AND hour >= ? ORDER BY hour ASC
+ `).all(svc, hourlyCutoff)), []);
+
+ const peakHours = safe(() => q(`
+ SELECT CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day, COUNT(*) AS hits,
+ COUNT(DISTINCT session_id) AS visitors, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY hour_of_day ORDER BY hour_of_day ASC
+ `).all(...human), []);
+
+ const topApiEndpoints = safe(() => q(`
+ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms,
+ COUNT(*) FILTER (WHERE status_code >= 400) AS errors
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND event_type = 'api_call' AND is_bot = 0
+ GROUP BY path, method ORDER BY hits DESC LIMIT 20
+ `).all(...human), []);
+
+ const sessionCount = safe(() => q(`
+ SELECT COUNT(DISTINCT session_id) FROM analytics_events
+ WHERE service = ? AND created_at >= ? AND session_id IS NOT NULL AND is_bot = 0
+ `).pluck().get(...human), 0);
+
+ // New vs returning needs a visitor identity that outlives a day; analytics keeps none.
+ const visitorTypes = { new_visitors: null, returning_visitors: null, not_collected: 'ADR-021' };
+
+ const slowestEndpoints = safe(() => q(`
+ SELECT path, method, COUNT(*) as hits, CAST(AVG(response_time_ms) AS INTEGER) AS avg_ms, MAX(response_time_ms) AS max_ms
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND response_time_ms IS NOT NULL AND is_bot = 0
+ GROUP BY path, method HAVING hits >= 3 ORDER BY avg_ms DESC LIMIT 10
+ `).all(...human), []);
+
+ const authTrend = safe(() => q(`
+ SELECT ${isSubDay ? bucketExpr : "strftime('%Y-%m-%d', created_at)"} AS bucket,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 1) AS auth_visitors,
+ COUNT(DISTINCT session_id) FILTER (WHERE authenticated = 0) AS anon_visitors,
+ COUNT(*) FILTER (WHERE authenticated = 1) AS auth_hits,
+ COUNT(*) FILTER (WHERE authenticated = 0) AS anon_hits
+ FROM analytics_events WHERE service = ? AND created_at >= ? AND is_bot = 0
+ GROUP BY bucket ORDER BY bucket ASC
+ `).all(...human), []);
+
+ return {
+ service: svc,
+ period_days: effectiveDays,
+ period_hours: hours || null,
+ summary: summary || {},
+ realtime: realtime || {},
+ daily,
+ hourly,
+ timeBuckets,
+ topPages,
+ topReferers,
+ botBreakdown,
+ statusCodes,
+ deviceBreakdown,
+ browserBreakdown,
+ osBreakdown,
+ countryBreakdown,
+ responsePercentiles,
+ bandwidth,
+ authBreakdown,
+ errorTrend,
+ peakHours,
+ topApiEndpoints,
+ sessionCount,
+ visitorTypes,
+ slowestEndpoints,
+ authTrend,
+ };
+ }
+
+ getOverview(days = 30) {
+ const cutoff = dayOf(this._now() - days * 86400000);
+ const services = this.db.prepare(`
+ SELECT service, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
+ SUM(unique_visitors) as unique_visitors, SUM(unique_users) as unique_users,
+ SUM(bot_hits) as bot_hits, SUM(error_count) as errors,
+ CAST(AVG(avg_response_ms) AS INTEGER) as avg_response_ms
+ FROM analytics_daily WHERE date >= ? GROUP BY service ORDER BY pageviews DESC
+ `).all(cutoff);
+ const totals = this.db.prepare(`
+ SELECT SUM(pageviews) as pageviews, SUM(api_calls) as api_calls, SUM(unique_visitors) as unique_visitors,
+ SUM(bot_hits) as bot_hits, SUM(error_count) as errors
+ FROM analytics_daily WHERE date >= ?
+ `).get(cutoff);
+ const dailyTrend = this.db.prepare(`
+ SELECT date, SUM(pageviews) as pageviews, SUM(api_calls) as api_calls,
+ SUM(unique_visitors) as unique_visitors, SUM(bot_hits) as bot_hits
+ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
+ `).all(cutoff);
+ const realtime = this.db.prepare(`
+ SELECT service, COUNT(*) as requests, COUNT(DISTINCT session_id) as visitors, COUNT(*) FILTER (WHERE is_bot = 1) as bots
+ FROM analytics_events WHERE created_at >= ? GROUP BY service
+ `).all(sqlTime(this._now() - 300000));
+ return { period_days: days, totals: totals || {}, services, dailyTrend, realtime };
+ }
+
+ /**
+ * Bot analysis. There are no IPs any more: the "top bot" rows are user-agent classes
+ * (`bot:<family>`), the "suspicious" rows are high-volume rotating sessions. The `ip` field is
+ * kept (null) so existing dashboards render; `ua_class` / `session_id` say what the row is.
+ */
+ getBotAnalysis(days = 30) {
+ const since = sqlTime(this._now() - Math.min(Math.max(1, days), 365) * 86400000);
+ const topBotIPs = this.db.prepare(`
+ SELECT NULL AS ip, user_agent AS ua_class, bot_type, COUNT(*) as hits,
+ COUNT(DISTINCT session_id) AS sessions,
+ MIN(created_at) as first_seen, MAX(created_at) as last_seen
+ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
+ GROUP BY user_agent, bot_type ORDER BY hits DESC LIMIT 25
+ `).all(since);
+ const botTrend = this.db.prepare(`
+ SELECT date, SUM(pageviews) + SUM(api_calls) as human_hits, SUM(bot_hits) as bot_hits
+ FROM analytics_daily WHERE date >= ? GROUP BY date ORDER BY date ASC
+ `).all(since.slice(0, 10));
+ const botTypes = this.db.prepare(`
+ SELECT bot_type, COUNT(*) as hits, NULL AS unique_ips, COUNT(DISTINCT session_id) AS unique_sessions
+ FROM analytics_events WHERE is_bot = 1 AND created_at >= ?
+ GROUP BY bot_type ORDER BY hits DESC
+ `).all(since);
+ const suspiciousIPs = this.db.prepare(`
+ SELECT NULL AS ip, session_id, COUNT(*) as total_hits,
+ COUNT(*) FILTER (WHERE status_code >= 400) as error_hits,
+ COUNT(DISTINCT path) as unique_paths,
+ MIN(created_at) as first_seen, MAX(created_at) as last_seen
+ FROM analytics_events
+ WHERE created_at >= ? AND is_bot = 0 AND session_id IS NOT NULL
+ GROUP BY session_id HAVING total_hits > 500
+ ORDER BY total_hits DESC LIMIT 20
+ `).all(sqlTime(this._now() - 7 * 86400000));
+ return { topBotIPs, botTrend, botTypes, suspiciousIPs };
+ }
+
+ destroy() {
+ this.flush();
+ for (const t of this._timers) { clearInterval(t); clearTimeout(t); }
+ this._timers = [];
+ if (this.pruneJob) { this.pruneJob.stop(); this.pruneJob = null; }
+ this._rates.clear();
+ this._sessions.clear();
+ }
+}
+
+module.exports = { AnalyticsTracker, sqlTime, dayOf, hourOf, SESSION_IDLE_MS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,92 @@
@@ -0,0 +1,92 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://openvibe.network/contracts/analytics/event.v1.json",
+ "title": "AnalyticsEvent",
+ "description": "analytics/event.v1: one raw analytics event (ADR-021). Kept by the owning service for at most 30 days, then only its rollups (daily and hourly counts) remain. It never carries an IP address, a user or subject id, a precise location (region or city), a raw user agent, a full referer URL, or a URL with ids or a query string. Requests sent with `Sec-GPC: 1` or `DNT: 1` produce no event. openvibe-shared/analytics stores it as an analytics_events row. For compatibility that row keeps the retired columns ip, user_id and city, always NULL; event.toEvent(row) maps a row to this shape.",
+ "type": "object",
+ "required": ["event", "service", "route", "session_id", "country", "ua_class", "is_bot", "authenticated", "timestamp"],
+ "additionalProperties": false,
+ "properties": {
+ "event": {
+ "description": "Event name: 'pageview' and 'api_call' for HTTP requests, or a custom name given to trackEvent. Row column: event_type.",
+ "type": "string",
+ "pattern": "^[A-Za-z][A-Za-z0-9_.:-]{0,63}$"
+ },
+ "service": {
+ "description": "The recording service, for example 'live', 'openvibe-network' or 'openvibe-yt'.",
+ "type": "string",
+ "pattern": "^[a-z][a-z0-9_-]{0,63}$"
+ },
+ "route": {
+ "description": "The route template, such as '/api/vods/:id/comments' or '/@:user'. It is the matched Express route, or else the request path with ids, usernames and slugs replaced by parameters. It has no query or fragment, and it is null for a custom event without a path. Row column: path.",
+ "type": ["string", "null"],
+ "maxLength": 200,
+ "pattern": "^/(([A-Za-z0-9._~-]+|@?:[A-Za-z_][A-Za-z0-9_]*\\??|\\*)(/([A-Za-z0-9._~-]+|@?:[A-Za-z_][A-Za-z0-9_]*\\??|\\*))*)?$"
+ },
+ "method": {
+ "description": "HTTP method, or null for a custom event.",
+ "type": ["string", "null"],
+ "pattern": "^[A-Z][A-Z-]{0,15}$"
+ },
+ "status": {
+ "description": "HTTP status code. Row column: status_code.",
+ "type": ["integer", "null"],
+ "minimum": 100,
+ "maximum": 599
+ },
+ "duration_ms": {
+ "description": "Time from the request's arrival to the finished response, in milliseconds. Row column: response_time_ms.",
+ "type": ["integer", "null"],
+ "minimum": 0
+ },
+ "session_id": {
+ "description": "A rotating session id: 16 random hex characters. It is renewed after 30 minutes idle and at every UTC midnight, and it is never derived from, or equal to, a user id. It is null when no session applies.",
+ "type": ["string", "null"],
+ "pattern": "^[0-9a-f]{16}$"
+ },
+ "country": {
+ "description": "ISO 3166-1 alpha-2 country code taken from the CDN header, or null. This is the only geography kept.",
+ "type": ["string", "null"],
+ "pattern": "^[A-Z]{2}$"
+ },
+ "ua_class": {
+ "description": "User-agent class: '<browser>/<os>/<device>', 'bot:<family>' for a known crawler, or 'none' when there is no user agent. The raw user agent is never stored. Row column: user_agent.",
+ "type": ["string", "null"],
+ "pattern": "^(none|bot:[a-z0-9]+|[a-z]+/[a-z]+/[a-z]+)$"
+ },
+ "device": {
+ "description": "Row column: device_type.",
+ "enum": ["desktop", "mobile", "tablet", "unknown", null]
+ },
+ "browser": {
+ "enum": ["chrome", "firefox", "safari", "edge", "opera", "ie", "other", "unknown", null]
+ },
+ "os": {
+ "enum": ["windows", "macos", "linux", "android", "ios", "other", "unknown", null]
+ },
+ "referer_origin": {
+ "description": "The scheme://host[:port] of an http(s) referer, or null. Never its path or query. Row column: referer.",
+ "type": ["string", "null"],
+ "maxLength": 300,
+ "pattern": "^https?://[^/?#\\s]+$"
+ },
+ "is_bot": {
+ "description": "True for a known crawler, a missing user agent, a honeypot path, or a client over the request-rate threshold.",
+ "type": "boolean"
+ },
+ "bot_type": {
+ "description": "Why the event is a bot hit: known_crawler, no_useragent, honeypot_hit or rate_limit. A custom event may give its own value.",
+ "type": ["string", "null"],
+ "pattern": "^[a-z0-9_]{1,32}$"
+ },
+ "authenticated": {
+ "description": "Whether the request was signed in. This is a flag and never says who.",
+ "type": "boolean"
+ },
+ "timestamp": {
+ "description": "UTC time in RFC 3339 form with second precision. Row column: created_at, stored as 'YYYY-MM-DD HH:MM:SS' UTC.",
+ "type": "string",
+ "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?Z$"
+ }
+ }
+}
@@ -12,6 +12,7 @@
@@ -12,6 +12,7 @@
"jsonwebtoken": "^9.0.0"
"jsonwebtoken": "^9.0.0"
},
},
"devDependencies": {
"devDependencies": {
+ "better-sqlite3": "^11.10.0",
"express": "^4.22.3"
"express": "^4.22.3"
}
}
},
},
@@ -36,6 +37,61 @@
@@ -36,6 +37,61 @@
"dev": true,
"dev": true,
"license": "MIT"
"license": "MIT"
},
},
+ "node_modules/base64-js": {
+ "version": "1.5.1",
+ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "license": "MIT"
+ },
+ "node_modules/better-sqlite3": {
+ "version": "11.10.0",
+ "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz",
+ "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==",
+ "dev": true,
+ "hasInstallScript": true,
+ "license": "MIT",
+ "dependencies": {
+ "bindings": "^1.5.0",
+ "prebuild-install": "^7.1.1"
+ }
+ },
+ "node_modules/bindings": {
+ "version": "1.5.0",
+ "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+ "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "file-uri-to-path": "1.0.0"
+ }
+ },
+ "node_modules/bl": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+ "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "buffer": "^5.5.0",
+ "inherits": "^2.0.4",
+ "readable-stream": "^3.4.0"
+ }
+ },
"node_modules/body-parser": {
"node_modules/body-parser": {
"version": "1.20.8",
"version": "1.20.8",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.8.tgz",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.8.tgz",
@@ -61,6 +117,31 @@
@@ -61,6 +117,31 @@
"npm": "1.2.8000 || >= 1.4.16"
"npm": "1.2.8000 || >= 1.4.16"
}
}
},
},
+ "node_modules/buffer": {
+ "version": "5.7.1",
+ "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+ "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "license": "MIT",
+ "dependencies": {
+ "base64-js": "^1.3.1",
+ "ieee754": "^1.1.13"
+ }
+ },
"node_modules/buffer-equal-constant-time": {
"node_modules/buffer-equal-constant-time": {
"version": "1.0.1",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
@@ -108,6 +189,13 @@
@@ -108,6 +189,13 @@
"url": "https://github.com/sponsors/ljharb"
"url": "https://github.com/sponsors/ljharb"
}
}
},
},
+ "node_modules/chownr": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+ "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+ "dev": true,
+ "license": "ISC"
+ },
"node_modules/content-disposition": {
"node_modules/content-disposition": {
"version": "0.5.4",
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -165,6 +253,32 @@
@@ -165,6 +253,32 @@
"dev": true,
"dev": true,
"license": "MIT"
"license": "MIT"
},
},
+ "node_modules/decompress-response": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+ "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "mimic-response": "^3.1.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/deep-extend": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+ "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
"node_modules/depd": {
"node_modules/depd": {
"version": "2.0.0",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -186,6 +300,16 @@
@@ -186,6 +300,16 @@
"npm": "1.2.8000 || >= 1.4.16"
"npm": "1.2.8000 || >= 1.4.16"
}
}
},
},
+ "node_modules/detect-libc": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
+ "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/dunder-proto": {
"node_modules/dunder-proto": {
"version": "1.0.1",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -227,6 +351,16 @@
@@ -227,6 +351,16 @@
"node": ">= 0.8"
"node": ">= 0.8"
}
}
},
},
+ "node_modules/end-of-stream": {
+ "version": "1.4.5",
+ "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
+ "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "once": "^1.4.0"
+ }
+ },
"node_modules/es-define-property": {
"node_modules/es-define-property": {
"version": "1.0.1",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -277,6 +411,16 @@
@@ -277,6 +411,16 @@
"node": ">= 0.6"
"node": ">= 0.6"
}
}
},
},
+ "node_modules/expand-template": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+ "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+ "dev": true,
+ "license": "(MIT OR WTFPL)",
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/express": {
"node_modules/express": {
"version": "4.22.3",
"version": "4.22.3",
"resolved": "https://registry.npmjs.org/express/-/express-4.22.3.tgz",
"resolved": "https://registry.npmjs.org/express/-/express-4.22.3.tgz",
@@ -324,6 +468,13 @@
@@ -324,6 +468,13 @@
"url": "https://opencollective.com/express"
"url": "https://opencollective.com/express"
}
}
},
},
+ "node_modules/file-uri-to-path": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/finalhandler": {
"node_modules/finalhandler": {
"version": "1.3.2",
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz",
@@ -363,6 +514,13 @@
@@ -363,6 +514,13 @@
"node": ">= 0.6"
"node": ">= 0.6"
}
}
},
},
+ "node_modules/fs-constants": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/function-bind": {
"node_modules/function-bind": {
"version": "1.1.2",
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@@ -412,6 +570,13 @@
@@ -412,6 +570,13 @@
"node": ">= 0.4"
"node": ">= 0.4"
}
}
},
},
+ "node_modules/github-from-package": {
+ "version": "0.0.0",
+ "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/gopd": {
"node_modules/gopd": {
"version": "1.2.0",
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@@ -485,6 +650,27 @@
@@ -485,6 +650,27 @@
"node": ">=0.10.0"
"node": ">=0.10.0"
}
}
},
},
+ "node_modules/ieee754": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+ "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "license": "BSD-3-Clause"
+ },
"node_modules/inherits": {
"node_modules/inherits": {
"version": "2.0.4",
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -492,6 +678,13 @@
@@ -492,6 +678,13 @@
"dev": true,
"dev": true,
"license": "ISC"
"license": "ISC"
},
},
+ "node_modules/ini": {
+ "version": "1.3.8",
+ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+ "dev": true,
+ "license": "ISC"
+ },
"node_modules/ipaddr.js": {
"node_modules/ipaddr.js": {
"version": "1.9.1",
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -663,12 +856,49 @@
@@ -663,12 +856,49 @@
"node": ">= 0.6"
"node": ">= 0.6"
}
}
},
},
+ "node_modules/mimic-response": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/minimist": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+ "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+ "dev": true,
+ "license": "MIT",
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/mkdirp-classic": {
+ "version": "0.5.3",
+ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/ms": {
"node_modules/ms": {
"version": "2.1.3",
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
"license": "MIT"
},
},
+ "node_modules/napi-build-utils": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
+ "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/negotiator": {
"node_modules/negotiator": {
"version": "0.6.3",
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -679,6 +909,19 @@
@@ -679,6 +909,19 @@
"node": ">= 0.6"
"node": ">= 0.6"
}
}
},
},
+ "node_modules/node-abi": {
+ "version": "3.96.0",
+ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz",
+ "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "semver": "^7.3.5"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
"node_modules/object-inspect": {
"node_modules/object-inspect": {
"version": "1.13.4",
"version": "1.13.4",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
@@ -705,6 +948,16 @@
@@ -705,6 +948,16 @@
"node": ">= 0.8"
"node": ">= 0.8"
}
}
},
},
+ "node_modules/once": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+ "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+ "dev": true,
+ "license": "ISC",
+ "dependencies": {
+ "wrappy": "1"
+ }
+ },
"node_modules/parseurl": {
"node_modules/parseurl": {
"version": "1.3.3",
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -722,6 +975,34 @@
@@ -722,6 +975,34 @@
"dev": true,
"dev": true,
"license": "MIT"
"license": "MIT"
},
},
+ "node_modules/prebuild-install": {
+ "version": "7.1.3",
+ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
+ "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==",
+ "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "detect-libc": "^2.0.0",
+ "expand-template": "^2.0.3",
+ "github-from-package": "0.0.0",
+ "minimist": "^1.2.3",
+ "mkdirp-classic": "^0.5.3",
+ "napi-build-utils": "^2.0.0",
+ "node-abi": "^3.3.0",
+ "pump": "^3.0.0",
+ "rc": "^1.2.7",
+ "simple-get": "^4.0.0",
+ "tar-fs": "^2.0.0",
+ "tunnel-agent": "^0.6.0"
+ },
+ "bin": {
+ "prebuild-install": "bin.js"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
"node_modules/proxy-addr": {
"node_modules/proxy-addr": {
"version": "2.0.8",
"version": "2.0.8",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz",
@@ -740,6 +1021,17 @@
@@ -740,6 +1021,17 @@
"url": "https://opencollective.com/express"
"url": "https://opencollective.com/express"
}
}
},
},
+ "node_modules/pump": {
+ "version": "3.0.4",
+ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
+ "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "end-of-stream": "^1.1.0",
+ "once": "^1.3.1"
+ }
+ },
"node_modules/qs": {
"node_modules/qs": {
"version": "6.16.0",
"version": "6.16.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz",
@@ -783,6 +1075,37 @@
@@ -783,6 +1075,37 @@
"node": ">= 0.8"
"node": ">= 0.8"
}
}
},
},
+ "node_modules/rc": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+ "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+ "dev": true,
+ "license": "(BSD-2-Clause OR MIT OR Apache-2.0)",
+ "dependencies": {
+ "deep-extend": "^0.6.0",
+ "ini": "~1.3.0",
+ "minimist": "^1.2.0",
+ "strip-json-comments": "~2.0.1"
+ },
+ "bin": {
+ "rc": "cli.js"
+ }
+ },
+ "node_modules/readable-stream": {
+ "version": "3.6.2",
+ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+ "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "inherits": "^2.0.3",
+ "string_decoder": "^1.1.1",
+ "util-deprecate": "^1.0.1"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/safe-buffer": {
"node_modules/safe-buffer": {
"version": "5.2.1",
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -946,6 +1269,53 @@
@@ -946,6 +1269,53 @@
"url": "https://github.com/sponsors/ljharb"
"url": "https://github.com/sponsors/ljharb"
}
}
},
},
+ "node_modules/simple-concat": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
+ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "license": "MIT"
+ },
+ "node_modules/simple-get": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
+ "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "license": "MIT",
+ "dependencies": {
+ "decompress-response": "^6.0.0",
+ "once": "^1.3.1",
+ "simple-concat": "^1.0.0"
+ }
+ },
"node_modules/statuses": {
"node_modules/statuses": {
"version": "2.0.2",
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
@@ -956,6 +1326,56 @@
@@ -956,6 +1326,56 @@
"node": ">= 0.8"
"node": ">= 0.8"
}
}
},
},
+ "node_modules/string_decoder": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+ "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "safe-buffer": "~5.2.0"
+ }
+ },
+ "node_modules/strip-json-comments": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+ "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/tar-fs": {
+ "version": "2.1.5",
+ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz",
+ "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "chownr": "^1.1.1",
+ "mkdirp-classic": "^0.5.2",
+ "pump": "^3.0.0",
+ "tar-stream": "^2.1.4"
+ }
+ },
+ "node_modules/tar-stream": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+ "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "bl": "^4.0.3",
+ "end-of-stream": "^1.4.1",
+ "fs-constants": "^1.0.0",
+ "inherits": "^2.0.3",
+ "readable-stream": "^3.1.1"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/toidentifier": {
"node_modules/toidentifier": {
"version": "1.0.1",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -966,6 +1386,19 @@
@@ -966,6 +1386,19 @@
"node": ">=0.6"
"node": ">=0.6"
}
}
},
},
+ "node_modules/tunnel-agent": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+ "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "dependencies": {
+ "safe-buffer": "^5.0.1"
+ },
+ "engines": {
+ "node": "*"
+ }
+ },
"node_modules/type-is": {
"node_modules/type-is": {
"version": "1.6.18",
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@@ -990,6 +1423,13 @@
@@ -990,6 +1423,13 @@
"node": ">= 0.8"
"node": ">= 0.8"
}
}
},
},
+ "node_modules/util-deprecate": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/utils-merge": {
"node_modules/utils-merge": {
"version": "1.0.1",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@@ -1009,6 +1449,13 @@
@@ -1009,6 +1449,13 @@
"engines": {
"engines": {
"node": ">= 0.8"
"node": ">= 0.8"
}
}
+ },
+ "node_modules/wrappy": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+ "dev": true,
+ "license": "ISC"
}
}
}
}
}
}
@@ -7,7 +7,14 @@
@@ -7,7 +7,14 @@
".": "./index.js",
".": "./index.js",
"./package.json": "./package.json",
"./package.json": "./package.json",
"./account-switcher": "./account-switcher.js",
"./account-switcher": "./account-switcher.js",
- "./analytics": "./analytics.js",
+ "./analytics": "./analytics/index.js",
+ "./analytics/event": "./analytics/event.js",
+ "./analytics/privacy": "./analytics/privacy.js",
+ "./analytics/prune-cli": "./analytics/prune-cli.js",
+ "./analytics/retention": "./analytics/retention.js",
+ "./analytics/schema": "./analytics/schema.js",
+ "./analytics/tracker": "./analytics/tracker.js",
+ "./analytics/event.v1.json": "./docs/schemas/analytics-event.v1.json",
"./app-icon": "./app-icon.js",
"./app-icon": "./app-icon.js",
"./auth-client": "./auth-client.js",
"./auth-client": "./auth-client.js",
"./brand": "./brand.js",
"./brand": "./brand.js",
@@ -43,6 +50,8 @@
@@ -43,6 +50,8 @@
},
},
"files": [
"files": [
"*.js",
"*.js",
+ "analytics/",
+ "docs/schemas/",
"scripts/",
"scripts/",
"CHANGELOG.md"
"CHANGELOG.md"
],
],
@@ -62,6 +71,7 @@
@@ -62,6 +71,7 @@
"jsonwebtoken": "^9.0.0"
"jsonwebtoken": "^9.0.0"
},
},
"devDependencies": {
"devDependencies": {
+ "better-sqlite3": "^11.10.0",
"express": "^4.22.3"
"express": "^4.22.3"
}
}
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,166 @@
@@ -0,0 +1,166 @@
+'use strict';
+/**
+ * The analytics-prune command (analytics/prune-cli.js), as Live, Tools and Network wire it:
+ * - a dry run (with and without --scrub) changes nothing;
+ * - --apply needs --backup <new file|dir> or --no-backup, refuses an existing target and days > 30;
+ * - the backup is verified and owner-only (it can hold pre-ADR IPs), then prune + scrub run and the
+ * rollup totals are unchanged;
+ * - several databases (Tools' apps) back up into a directory; a service's defaults, extra options and
+ * path options (Network) are honoured.
+ *
+ * node test/analytics-cli.test.js
+ */
+const assert = require('assert');
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const Database = require('better-sqlite3');
+const { AnalyticsTracker, retention } = require('openvibe-shared/analytics');
+const { sqlTime } = require('openvibe-shared/analytics/tracker');
+const cli = require('openvibe-shared/analytics/prune-cli');
+
+let failures = 0;
+async function check(name, fn) {
+ try { await fn(); console.log(' ✓', name); }
+ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }
+}
+
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-shared-cli-'));
+const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+const NETWORK = { paramPrefixes: ['avatar', 'anon', 'projects'], pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']] };
+const out = [];
+const log = (l) => out.push(l);
+const main = (argv, extra = {}) => cli.main(argv, { Database, log, ...extra });
+const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
+
+function dumpAll(file) {
+ const db = new Database(file, { readonly: true });
+ try {
+ return db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all()
+ .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');
+ } finally { db.close(); }
+}
+
+/** A legacy database: 10 rows past the cutoff, 3 inside it (edge rows 2 min clear of the CLI's own clock). */
+function legacyDb(file) {
+ fs.mkdirSync(path.dirname(file), { recursive: true });
+ const db = new Database(file);
+ db.pragma('journal_mode = WAL');
+ db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT); INSERT INTO users VALUES (7, 'keepme');");
+ new AnalyticsTracker(db, 'live', { timers: false }).destroy();
+ const nowMs = Date.now() + 120000;
+ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, user_id, session_id, ip, city, user_agent, referer, created_at)
+ VALUES ('live', 'pageview', ?, 'GET', 200, ?, ?, ?, ?, ?, ?, ?)`);
+ for (let i = 0; i < 10; i++) ins.run(`/@old${i}?x=${i}`, 7, 'tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/x?y=1', sqlTime(nowMs - (40 + i) * 86400000));
+ ins.run('/internal/users/by-username/keepme', null, null, '127.0.0.1', null, 'node', '', sqlTime(nowMs - 3 * 86400000));
+ ins.run('/avatar/keepme?s=96', 9, 'abcdef0123456789', '198.51.100.52', 'Oslo', CHROME, 'https://www.reddit.com/r/a', sqlTime(nowMs - 2 * 86400000));
+ ins.run('/vod/12345?t=9', null, null, '198.51.100.53', null, CHROME, '', sqlTime(nowMs - 1 * 86400000));
+ db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
+ .run('live', '2026-07-01', 100, 40, 30, 10, 2, JSON.stringify([{ path: '/@keepme', cnt: 3 }, { path: '/internal/users/by-username/keepme', cnt: 2 }]), JSON.stringify([{ referer: 'https://www.google.com/search?q=keepme', cnt: 4 }]));
+ db.pragma('wal_checkpoint(TRUNCATE)');
+ db.close();
+ return file;
+}
+
+(async () => {
+ await check('argument parsing: defaults, repeatable --db, extra options, refusals', () => {
+ const a = cli.parseArgs(['--db', 'a.db', '--db', 'b.db', '--app', 'yt', '--scrub'], { '--app': 'apps' });
+ assert.deepStrictEqual([a.dbs, a.db, a.apps, a.scrub, a.apply, a.days, a.batch], [['a.db', 'b.db'], 'a.db', ['yt'], true, false, 30, 5000]);
+ assert.throws(() => cli.parseArgs(['--days', '31']), /1 to 30/);
+ assert.throws(() => cli.parseArgs(['--batch', '0']), /batch/);
+ assert.throws(() => cli.parseArgs(['--backup', 'x', '--no-backup']), /exclusive/);
+ assert.throws(() => cli.parseArgs(['--app', 'yt']), /unknown option --app/);
+ assert.throws(() => cli.parseArgs(['--db']), /needs a value/);
+ assert.deepStrictEqual(cli.namedTargets(['/srv/tools/apps/yt/data/analytics.db', '/x/analytics.db', '/y/analytics.db']).map((t) => t.name), ['yt', 'analytics', 'analytics-2']);
+ });
+
+ await check('dry run (with and without --scrub) changes nothing; the default database is the service\'s', async () => {
+ const file = legacyDb(path.join(tmp, 'dry', 'analytics.db'));
+ const before = { hash: sha(file), dump: dumpAll(file) };
+ assert.strictEqual(await main([], { defaultDb: file }), 0);
+ assert.strictEqual(await main(['--scrub', '--days', '7'], { defaultDb: (args, env) => env.TEST_ANALYTICS_DB, env: { TEST_ANALYTICS_DB: file } }), 0);
+ assert.strictEqual(sha(file), before.hash);
+ assert.strictEqual(dumpAll(file), before.dump);
+ assert.ok(out.some((l) => /dry run: nothing changed/.test(l)));
+ assert.ok(out.some((l) => /prune\s+10 rows/.test(l)), out.join('\n'));
+ assert.strictEqual(await main([]), 2, 'no database at all');
+ assert.strictEqual(await main(['--db', path.join(tmp, 'missing.db')]), 2);
+ assert.strictEqual(await main(['--help'], { usage: 'custom usage' }), 0);
+ assert.strictEqual(out[out.length - 1], 'custom usage');
+ await assert.rejects(cli.main([], {}), /Database/);
+ });
+
+ await check('--apply refuses without a backup choice, an existing target, or days > 30', async () => {
+ const file = legacyDb(path.join(tmp, 'refuse.db'));
+ const before = dumpAll(file);
+ assert.strictEqual(await main(['--db', file, '--apply']), 2);
+ const existing = path.join(tmp, 'exists.bak');
+ fs.writeFileSync(existing, 'x');
+ assert.strictEqual(await main(['--db', file, '--apply', '--backup', existing]), 2);
+ assert.strictEqual(await main(['--db', file, '--apply', '--no-backup', '--days', '31']), 2);
+ assert.strictEqual(await main(['--db', file, '--apply', '--no-backup', '--backup', existing]), 2);
+ assert.strictEqual(dumpAll(file), before);
+ assert.strictEqual(fs.readFileSync(existing, 'utf8'), 'x');
+ });
+
+ await check('--apply --scrub --backup: owner-only verified backup, prune + scrub, rollups equal, other tables kept', async () => {
+ const file = legacyDb(path.join(tmp, 'apply.db'));
+ const src = new Database(file, { readonly: true });
+ const totals = retention.rollupTotals(src);
+ src.close();
+ const bak = path.join(tmp, 'apply.backup.db');
+ assert.strictEqual(await main(['--db', file, '--apply', '--scrub', '--backup', bak, '--batch', '2'], NETWORK), 0, out.join('\n'));
+ assert.strictEqual(fs.statSync(bak).mode & 0o777, 0o600);
+ const b = new Database(bak, { readonly: true });
+ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13, 'backup has every row');
+ b.close();
+ const db = new Database(file, { readonly: true });
+ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all(),
+ ['/internal/users/:param/:username', '/avatar/:param', '/vod/:param']);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events WHERE ip IS NOT NULL OR user_id IS NOT NULL OR city IS NOT NULL').pluck().get(), 0);
+ assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()),
+ [{ path: '/@:user', cnt: 3 }, { path: '/internal/users/:param/:username', cnt: 2 }]);
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ assert.strictEqual(db.prepare('SELECT username FROM users WHERE id = 7').pluck().get(), 'keepme');
+ db.close();
+ assert.ok(out.some((l) => /mode 0600, quick_check ok/.test(l)));
+ // Prune-only with an explicit --no-backup works too (nothing left to prune).
+ assert.strictEqual(await main(['--db', file, '--apply', '--no-backup', '--no-vacuum']), 0);
+ });
+
+ await check('several databases (Tools apps): discovery hook, extra option, backups into a directory', async () => {
+ const apps = path.join(tmp, 'apps');
+ const yt = legacyDb(path.join(apps, 'yt', 'data', 'analytics.db'));
+ const food = legacyDb(path.join(apps, 'food', 'data', 'analytics.db'));
+ const tools = {
+ options: { '--app': 'apps' },
+ targets: (args) => (args.apps.length ? args.apps : ['food', 'yt']).map((name) => ({ name, file: path.join(apps, name, 'data', 'analytics.db') })),
+ };
+ const hashes = [sha(yt), sha(food)];
+ assert.strictEqual(await main(['--scrub'], tools), 0);
+ assert.strictEqual(await main(['--app', 'yt'], tools), 0);
+ assert.deepStrictEqual([sha(yt), sha(food)], hashes, 'dry run changed a file');
+ const bdir = path.join(tmp, 'backups');
+ assert.strictEqual(await main(['--apply', '--scrub', '--backup', bdir], tools), 0, out.join('\n'));
+ for (const [name, file] of [['yt', yt], ['food', food]]) {
+ const bfile = path.join(bdir, `${name}.analytics.db`);
+ assert.strictEqual(fs.statSync(bfile).mode & 0o777, 0o600);
+ const b = new Database(bfile, { readonly: true });
+ assert.strictEqual(b.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 13);
+ b.close();
+ const db = new Database(file, { readonly: true });
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 3);
+ assert.strictEqual(db.prepare('SELECT SUM(pageviews) FROM analytics_daily').pluck().get(), 100);
+ db.close();
+ }
+ assert.strictEqual(await main(['--apply', '--backup', bdir], tools), 2, 'existing backup targets are refused');
+ const afile = path.join(tmp, 'a-file');
+ fs.writeFileSync(afile, 'x');
+ assert.strictEqual(await main(['--apply', '--backup', afile], tools), 2, 'several databases need a directory');
+ });
+
+ fs.rmSync(tmp, { recursive: true, force: true });
+ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }
+ console.log('\nanalytics prune CLI: all passed');
+})();
new file mode 100644
new file mode 100644
@@ -0,0 +1,402 @@
@@ -0,0 +1,402 @@
+'use strict';
+/**
+ * ADR-021 analytics bounds (analytics/): what a tracked request may leave behind.
+ * - no IP, user id, city, raw user agent, raw referer or query string anywhere in the database;
+ * the session id is a rotating id, never the user id;
+ * - paths become route templates (the Express route when matched, else the normaliser), with a
+ * service's extra parameter words and path rules (Network's, as options);
+ * - unique visitors come from the day's salted hashes, which are deleted once the day is rolled up;
+ * - `Sec-GPC: 1` / `DNT: 1` requests are not recorded at all: no row, hash, session or counter;
+ * - every row the tracker writes is a valid analytics/event.v1 event; a pre-ADR row is not.
+ *
+ * node test/analytics-privacy.test.js
+ */
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const express = require('express');
+const Database = require('better-sqlite3');
+const analytics = require('openvibe-shared/analytics');
+const { sqlTime } = require('openvibe-shared/analytics/tracker');
+const EVENT_V1 = require('openvibe-shared/analytics/event.v1.json');
+
+const { AnalyticsTracker, privacy, event } = analytics;
+
+let failures = 0;
+async function check(name, fn) {
+ try { await fn(); console.log(' ✓', name); }
+ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }
+}
+
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-shared-analytics-'));
+const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };
+const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
+const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';
+// Network's route shapes, given as options (they were hard-coded in its server/analytics/network.js).
+const NETWORK = { paramPrefixes: ['avatar', 'anon', 'projects'], pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']] };
+
+/** Every value in every table, for "is this anywhere in the file" checks. */
+function dumpAll(db) {
+ const tables = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all();
+ return tables.map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');
+}
+
+async function serve(app, fn) {
+ const server = app.listen(0, '127.0.0.1');
+ await new Promise((r) => server.once('listening', r));
+ try { return await fn(`http://127.0.0.1:${server.address().port}`); }
+ finally { await new Promise((r) => server.close(r)); }
+}
+const settle = () => new Promise((r) => setTimeout(r, 50));
+
+(async () => {
+ // ── Reducers ─────────────────────────────────────────────
+ await check('normalisePath strips queries and replaces ids, slugs and usernames', () => {
+ const n = privacy.normalisePath;
+ const cases = {
+ '/': '/',
+ '': '/',
+ '/vods': '/vods',
+ '/vod/8a7c2f10-1b2c-4d5e-8f90-123456789abc?t=30': '/vod/:param',
+ '/api/vods/123/comments?page=2&token=abc': '/api/vods/:id/comments',
+ '/@JapaneseOldGuy': '/@:user',
+ '/@alex/main-stage?stream=77': '/@:user/:param',
+ '/p/k3yF00bar': '/p/:param',
+ '/recap/some-words': '/recap/:param',
+ '/u/alex/settings': '/u/:param/settings',
+ '/api/users/alex': '/api/users/:param',
+ '/watch/dQw4w9WgXcQ': '/watch/:param',
+ '/files/3f786850e387550fdab836ed7e6dc881de23001b': '/files/:param',
+ '/api/things/usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W': '/api/things/:id',
+ '/x/01J8ZQ4K7M2N3P4Q5R6S7T8V9W': '/x/:id',
+ '/verify/alex%40example.com': '/verify/:param',
+ '/share/[email protected]/x': '/share/:param/x',
+ '/maps/@40.7128,-74.0060,12z': '/maps/@:user',
+ '/dashboard/settings': '/dashboard/settings',
+ '/wp-login.php': '/wp-login.php',
+ '/stream-2024-09-23-highlights': '/:id',
+ '/hello world': '/:id',
+ 'https://openvibe.live/@alex?x=1#y': '/@:user',
+ '/one/two/three/four/five/six/seven/eight/nine': '/one/two/three/four/five/six/seven/eight/*',
+ // Tools' shapes.
+ '/api/info?url=https://youtube.com/watch?v=abc': '/api/info',
+ '/api/jobs/01J8ZQ4K7M2N3P4Q5R6S7T8V9W/events': '/api/jobs/:param/events',
+ '/recipe/chicken-tikka': '/recipe/:param',
+ '/place/@52.37,4.89,14z': '/place/@:user',
+ };
+ for (const [raw, want] of Object.entries(cases)) assert.strictEqual(n(raw), want, raw);
+ for (const t of ['/api/vods/:id/comments', '/@:user/:param', '/api/analytics/channel/:username', '/p/:id']) {
+ assert.strictEqual(n(t), t);
+ assert.strictEqual(n(n(t)), n(t));
+ }
+ assert.strictEqual(n('/dishes/tacos', { paramPrefixes: ['dishes'] }), '/dishes/:param');
+ });
+
+ await check('an over-long template drops whole segments, stays ≤ 200 chars and idempotent', () => {
+ const long = '/' + Array.from({ length: 8 }, (_, i) => 'abcdefghijklmnopqrstuvwxyzabcd' + 'xy'[i % 2]).join('/');
+ const t = privacy.normalisePath(long);
+ assert.ok(t.length <= privacy.MAX_TEMPLATE_LENGTH, t.length);
+ assert.ok(t.endsWith('/*'), t);
+ assert.ok(t.split('/').slice(1, -1).every((s) => /^abcdefghijklmnopqrstuvwxyzabcd[xy]$/.test(s)), t);
+ assert.strictEqual(privacy.normalisePath(t), t);
+ assert.ok(new RegExp(EVENT_V1.properties.route.pattern, 'u').test(t));
+ });
+
+ await check('service options: Network paramPrefixes and pathRules become parameters', () => {
+ const opts = privacy.pathOptions(NETWORK);
+ const t = (p) => privacy.normalisePath(p, opts);
+ const cases = {
+ '/avatar/alex?s=96': '/avatar/:param',
+ '/api/auth/anon/k3yF00barBaz?x=1': '/api/auth/anon/:param',
+ '/internal/users/by-username/alex': '/internal/users/:param/:username',
+ '/api/v1/projects/my-app/apps': '/api/v1/projects/:param/apps',
+ '/oauth/authorize?client_id=live&state=abc': '/oauth/authorize',
+ '/reset-password?token=abc': '/reset-password',
+ '/api/admin/users/42/role': '/api/admin/users/:param/role',
+ };
+ for (const [raw, want] of Object.entries(cases)) assert.strictEqual(t(raw), want, raw);
+ for (const want of Object.values(cases)) assert.strictEqual(t(want), want, `idempotent: ${want}`);
+ // Without the rule the shared normaliser keeps the name; the defaults stay in the merged set.
+ assert.strictEqual(privacy.normalisePath('/internal/users/by-username/alex'), '/internal/users/:param/alex');
+ assert.ok(opts.paramPrefixes.has('vod') && opts.paramPrefixes.has('avatar'));
+ assert.strictEqual(privacy.pathOptions({}), undefined);
+ assert.throws(() => privacy.pathOptions({ pathRules: [['by-username', 'x']] }), /RegExp/);
+ });
+
+ await check('referer → origin, user agent → class, country → ISO code', () => {
+ assert.strictEqual(privacy.refererOrigin('https://www.google.com/search?q=who+is+alex'), 'https://www.google.com');
+ assert.strictEqual(privacy.refererOrigin('http://localhost:3000/@alex'), 'http://localhost:3000');
+ assert.strictEqual(privacy.refererOrigin('android-app://com.foo/'), null);
+ assert.strictEqual(privacy.refererOrigin('not a url'), null);
+ assert.strictEqual(privacy.uaClass(CHROME), 'chrome/windows/desktop');
+ assert.strictEqual(privacy.uaClass(FIREFOX), 'firefox/linux/desktop');
+ assert.strictEqual(privacy.uaClass('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'), 'bot:googlebot');
+ assert.strictEqual(privacy.uaClass(''), 'none');
+ assert.strictEqual(privacy.uaClass('chrome/windows/desktop'), 'chrome/windows/desktop');
+ assert.strictEqual(privacy.uaClass('bot:googlebot'), 'bot:googlebot');
+ assert.strictEqual(privacy.countryCode('de'), 'DE');
+ assert.strictEqual(privacy.countryCode('Berlin'), null);
+ });
+
+ await check('optedOut: Sec-GPC: 1 or DNT: 1, nothing else', () => {
+ assert.strictEqual(privacy.optedOut({ 'sec-gpc': '1' }), true);
+ assert.strictEqual(privacy.optedOut({ dnt: '1' }), true);
+ assert.strictEqual(privacy.optedOut({ dnt: ' 1 ' }), true);
+ assert.strictEqual(privacy.optedOut({ 'sec-gpc': ['0', '1'] }), true);
+ for (const h of [{}, { dnt: '0' }, { 'sec-gpc': '0' }, { dnt: 'yes' }, { 'sec-gpc': '' }, null, undefined]) {
+ assert.strictEqual(privacy.optedOut(h), false, JSON.stringify(h));
+ }
+ assert.strictEqual(analytics.optedOut, privacy.optedOut);
+ });
+
+ // ── A tracked request ────────────────────────────────────
+ await check('a tracked request stores no IP, user id, city, raw UA, raw referer or query', async () => {
+ const db = newDb('track.db');
+ let clock = Date.parse('2026-09-23T10:15:00Z');
+ const tracker = new AnalyticsTracker(db, 'live', { timers: false, now: () => clock });
+ const app = express();
+ app.set('trust proxy', true);
+ app.use(tracker.middleware());
+ app.use((req, res, next) => { if (req.headers.authorization) req.user = { id: 42, sub: SUBJECT, username: 'alex' }; next(); });
+ const router = express.Router();
+ router.get('/things/:thingId', (req, res) => res.json({ ok: true }));
+ app.use('/api', router);
+ app.get('*', (req, res) => res.send('<html></html>'));
+ const hdr = (extra) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.77', referer: 'https://www.google.com/search?q=secret-query', 'cf-ipcountry': 'NL', 'cf-ipcity': 'Amsterdam', ...extra });
+ await serve(app, async (base) => {
+ await (await fetch(`${base}/api/things/98765?token=supersecret`, { headers: hdr({ authorization: 'Bearer x' }) })).text();
+ await (await fetch(`${base}/@alex/main?stream=12`, { headers: hdr() })).text();
+ await (await fetch(`${base}/vod/424242`, { headers: hdr({ 'user-agent': FIREFOX, 'x-forwarded-for': '198.51.100.9' }) })).text();
+ await settle();
+ });
+ tracker.flush();
+
+ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY id').all();
+ assert.strictEqual(rows.length, 3);
+ for (const r of rows) {
+ assert.strictEqual(r.ip, null);
+ assert.strictEqual(r.user_id, null);
+ assert.strictEqual(r.city, null);
+ assert.ok(!/[?#]/.test(r.path), r.path);
+ assert.strictEqual(r.referer, 'https://www.google.com');
+ assert.ok(/^[0-9a-f]{16}$/.test(r.session_id), r.session_id);
+ assert.strictEqual(r.country, 'NL');
+ assert.deepStrictEqual(event.checkRow(r), [], JSON.stringify(r));
+ }
+ assert.deepStrictEqual(rows.map((r) => r.path), ['/api/things/:thingId', '/@:user/:param', '/vod/:param']);
+ assert.deepStrictEqual(rows.map((r) => r.user_agent), ['chrome/windows/desktop', 'chrome/windows/desktop', 'firefox/linux/desktop']);
+ assert.deepStrictEqual(rows.map((r) => r.authenticated), [1, 0, 0]);
+ assert.deepStrictEqual(rows.map((r) => r.event_type), ['api_call', 'pageview', 'pageview']);
+ assert.strictEqual(rows[0].session_id, rows[1].session_id, 'same visitor, same session');
+ assert.notStrictEqual(rows[0].session_id, rows[2].session_id);
+
+ const everything = dumpAll(db);
+ for (const needle of ['203.0.113.77', '198.51.100.9', '127.0.0.1', SUBJECT, 'Amsterdam', 'supersecret', 'secret-query', '98765', '424242', 'Mozilla/5.0', '"alex"']) {
+ assert.ok(!everything.includes(needle), `found ${needle} in analytics.db`);
+ }
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
+ assert.ok(!db.prepare("SELECT name FROM sqlite_master WHERE name IN ('idx_analytics_events_ip', 'idx_analytics_events_user')").get());
+
+ // Rollups: two distinct (ip, ua) visitors, one of them signed in.
+ tracker.aggregate();
+ const hourly = db.prepare("SELECT * FROM analytics_hourly WHERE hour = '2026-09-23 10:00:00'").get();
+ assert.strictEqual(hourly.pageviews, 2);
+ assert.strictEqual(hourly.api_calls, 1);
+ assert.strictEqual(hourly.unique_visitors, 2);
+ assert.strictEqual(hourly.unique_users, 1);
+ const daily = db.prepare("SELECT * FROM analytics_daily WHERE date = '2026-09-23'").get();
+ assert.strictEqual(daily.unique_visitors, 2);
+ assert.strictEqual(daily.unique_users, 1);
+ assert.strictEqual(daily.new_users, null);
+ assert.ok(JSON.parse(daily.top_paths).every((p) => !/\d{3,}/.test(p.path)));
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 2);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 1);
+
+ // Next day: the finished day is rolled up for the last time, then its hashes and salt go.
+ clock = Date.parse('2026-09-24T00:20:00Z');
+ tracker.aggregate();
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 0);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_day_salts').pluck().get(), 0);
+ assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2, 'uniques survive the hashes');
+ clock = Date.parse('2026-09-24T01:20:00Z');
+ tracker.aggregate();
+ assert.strictEqual(db.prepare("SELECT unique_visitors FROM analytics_daily WHERE date = '2026-09-23'").pluck().get(), 2, 'a recompute never lowers uniques');
+
+ // The same visitor gets a new session id (and hash) on a new day.
+ assert.strictEqual(tracker.record({ headers: { 'user-agent': CHROME }, ip: '203.0.113.77', method: 'GET' }, { statusCode: 200 }, '/', 3), true);
+ tracker.flush();
+ const last = db.prepare('SELECT session_id FROM analytics_events ORDER BY id DESC LIMIT 1').pluck().get();
+ assert.notStrictEqual(last, rows[0].session_id);
+
+ // Dashboards keep the pre-ADR shapes.
+ const st = tracker.getStats({ days: 30 });
+ for (const k of ['summary', 'realtime', 'daily', 'hourly', 'topPages', 'authBreakdown', 'visitorTypes', 'authTrend']) assert.ok(k in st, k);
+ assert.ok(tracker.getStats({ hours: 6 }).timeBuckets);
+ const bots = tracker.getBotAnalysis(30);
+ for (const k of ['topBotIPs', 'botTrend', 'botTypes', 'suspiciousIPs']) assert.ok(Array.isArray(bots[k]), k);
+ assert.ok(tracker.getOverview(30).services.length >= 1);
+ tracker.destroy();
+ db.close();
+ });
+
+ await check('Sec-GPC: 1 and DNT: 1 requests leave nothing: no row, hash, session or rate counter', async () => {
+ const db = newDb('optout.db');
+ const clock = Date.parse('2026-09-23T10:15:00Z');
+ const tracker = new AnalyticsTracker(db, 'live', { timers: false, now: () => clock });
+ const app = express();
+ app.set('trust proxy', true);
+ app.use(tracker.middleware());
+ app.get('*', (req, res) => res.send('ok'));
+ const hdr = (extra) => ({ 'user-agent': CHROME, 'x-forwarded-for': '203.0.113.5', 'cf-ipcountry': 'FR', ...extra });
+ await serve(app, async (base) => {
+ await (await fetch(`${base}/@gpc-user`, { headers: hdr({ 'sec-gpc': '1' }) })).text();
+ await (await fetch(`${base}/@dnt-user`, { headers: hdr({ dnt: '1' }) })).text();
+ for (let i = 0; i < 70; i++) await (await fetch(`${base}/flood`, { headers: hdr({ 'sec-gpc': '1', 'x-forwarded-for': '203.0.113.6' }) })).text();
+ await (await fetch(`${base}/counted`, { headers: hdr({ dnt: '0', 'sec-gpc': '0' }) })).text();
+ await settle();
+ });
+ tracker.flush();
+ tracker.aggregate();
+ const rows = db.prepare('SELECT * FROM analytics_events').all();
+ assert.deepStrictEqual(rows.map((r) => r.path), ['/counted'], 'only the request without an opt-out is recorded');
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_visitor_days').pluck().get(), 1);
+ assert.strictEqual(tracker._sessions.size, 1, 'no session for an opted-out visitor');
+ assert.ok(!tracker._rates.has('203.0.113.6'), 'no rate counter for an opted-out client');
+ const hourly = db.prepare('SELECT pageviews, unique_visitors FROM analytics_hourly').get();
+ assert.deepStrictEqual({ ...hourly }, { pageviews: 1, unique_visitors: 1 });
+ // Direct calls honour it too.
+ assert.strictEqual(tracker.record({ headers: { 'user-agent': CHROME, dnt: '1' }, ip: '192.0.2.9', method: 'GET' }, { statusCode: 200 }, '/x', 1), false);
+ assert.strictEqual(tracker.trackEvent('share_clicked', { headers: { 'sec-gpc': '1' }, path: '/@bob' }), false);
+ assert.strictEqual(tracker.trackEvent('share_clicked', { headers: { 'user-agent': CHROME }, path: '/@bob' }), true);
+ tracker.flush();
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_events').pluck().get(), 2);
+ const all = dumpAll(db);
+ for (const needle of ['gpc-user', 'dnt-user', 'flood', '203.0.113.', '192.0.2.9']) assert.ok(!all.includes(needle), needle);
+ tracker.destroy();
+ db.close();
+ });
+
+ await check('Network options on the tracker: unmatched paths use its rules', async () => {
+ const db = newDb('network.db');
+ const tracker = new AnalyticsTracker(db, 'openvibe-network', { timers: false, ...NETWORK });
+ const app = express();
+ app.use(tracker.middleware());
+ app.get('/avatar/:username', (req, res) => res.send('img'));
+ app.use('/internal', (req, res) => res.status(403).end()); // refused before routing
+ app.use('/api/auth/anon', (req, res) => res.status(404).end());
+ await serve(app, async (base) => {
+ for (const p of ['/avatar/alexsecretname?s=96', '/internal/users/by-username/carolsecretname', '/api/auth/anon/0123456789abcdefanon']) {
+ await (await fetch(base + p, { headers: { 'user-agent': CHROME } })).arrayBuffer();
+ }
+ await settle();
+ });
+ tracker.flush();
+ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY id').pluck().all(),
+ ['/avatar/:username', '/internal/users/:param/:username', '/api/auth/anon/:param']);
+ const all = dumpAll(db);
+ for (const needle of ['alexsecretname', 'carolsecretname', '0123456789abcdefanon']) assert.ok(!all.includes(needle), needle);
+ tracker.destroy();
+ db.close();
+ });
+
+ await check('rate check keeps IPs in memory only and still flags floods; custom events are reduced', () => {
+ const db = newDb('rate.db');
+ const clock = Date.parse('2026-09-23T12:00:30Z');
+ const tracker = new AnalyticsTracker(db, 'live', { timers: false, now: () => clock });
+ for (let i = 0; i < 70; i++) tracker.record({ headers: { 'user-agent': CHROME }, ip: '192.0.2.1', method: 'GET' }, { statusCode: 200 }, '/api/x', 1);
+ tracker.flush();
+ assert.ok(db.prepare("SELECT COUNT(*) FROM analytics_events WHERE bot_type = 'rate_limit'").pluck().get() >= 10);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
+ assert.ok(!dumpAll(db).includes('192.0.2.1'));
+ tracker.trackEvent('custom', {
+ ip: '192.0.2.2', user_id: 7, city: 'Paris', path: '/@bob?x=1', referer: 'https://a.example/x?y=1', user_agent: CHROME,
+ session_id: 'user-7', method: 'post', status_code: 999, response_time_ms: -4, browser: 'MyBrowser 1.0', bot_type: 'Weird Type',
+ });
+ tracker.flush();
+ const e = db.prepare("SELECT * FROM analytics_events WHERE event_type = 'custom'").get();
+ assert.strictEqual(e.path, '/@:user');
+ assert.strictEqual(e.referer, 'https://a.example');
+ assert.strictEqual(e.session_id, null);
+ assert.strictEqual(e.user_agent, 'chrome/windows/desktop');
+ assert.strictEqual(e.method, 'POST');
+ assert.strictEqual(e.status_code, null);
+ assert.strictEqual(e.response_time_ms, null);
+ assert.strictEqual(e.browser, 'chrome');
+ assert.strictEqual(e.bot_type, null);
+ assert.deepStrictEqual(event.checkRow(e), []);
+ assert.ok(!dumpAll(db).includes('Paris') && !dumpAll(db).includes('192.0.2.2'));
+ assert.throws(() => tracker.trackEvent('has spaces'), TypeError);
+ assert.throws(() => tracker.trackEvent('x'.repeat(65)), TypeError);
+ for (const r of db.prepare('SELECT * FROM analytics_events').all()) assert.deepStrictEqual(event.checkRow(r), []);
+ tracker.destroy();
+ db.close();
+ });
+
+ // ── analytics/event.v1 ───────────────────────────────────
+ await check('event.v1: the schema file is what the package exports, and a pre-ADR row fails it', () => {
+ assert.strictEqual(event.EVENT_V1, EVENT_V1);
+ assert.strictEqual(EVENT_V1.$id, 'https://openvibe.network/contracts/analytics/event.v1.json');
+ assert.strictEqual(EVENT_V1.additionalProperties, false);
+ for (const banned of ['ip', 'user_id', 'city', 'subject_id', 'user_agent', 'referer', 'url', 'region']) {
+ assert.ok(!(banned in EVENT_V1.properties), `${banned} is not an event.v1 field`);
+ }
+ for (const f of ['event', 'service', 'route', 'session_id', 'country', 'ua_class', 'timestamp']) assert.ok(EVENT_V1.required.includes(f), f);
+ const good = {
+ event: 'pageview', service: 'live', route: '/@:user/:param', method: 'GET', status: 200, duration_ms: 12,
+ session_id: '0123456789abcdef', country: 'NL', ua_class: 'chrome/windows/desktop', device: 'desktop', browser: 'chrome',
+ os: 'windows', referer_origin: 'https://www.google.com', is_bot: false, bot_type: null, authenticated: true, timestamp: '2026-09-23T10:15:00Z',
+ };
+ assert.deepStrictEqual(event.validateEvent(good), []);
+ const bad = (patch) => event.validateEvent({ ...good, ...patch });
+ assert.deepStrictEqual(bad({ ip: '203.0.113.1' }), ['ip: not allowed']);
+ assert.deepStrictEqual(bad({ user_id: 42 }), ['user_id: not allowed']);
+ assert.ok(bad({ route: '/vod/12345?t=1' }).length);
+ assert.ok(bad({ route: '/@alex smith' }).length);
+ assert.ok(bad({ referer_origin: 'https://www.google.com/search?q=x' }).length);
+ assert.ok(bad({ ua_class: CHROME }).length);
+ assert.ok(bad({ session_id: '42' }).length);
+ assert.ok(bad({ country: 'Amsterdam' }).length);
+ assert.ok(bad({ timestamp: '2026-09-23 10:15:00' }).length);
+ assert.ok(bad({ status: 99 }).length);
+ const { session_id, ...noSession } = good; // eslint-disable-line no-unused-vars
+ assert.deepStrictEqual(event.validateEvent(noSession), ['session_id: required']);
+
+ const legacy = {
+ service: 'live', event_type: 'pageview', path: '/@alex?x=1', method: 'GET', status_code: 200, response_time_ms: 5,
+ user_id: 42, session_id: 'user-42', ip: '198.51.100.7', country: null, city: 'Lyon', user_agent: CHROME,
+ referer: 'https://t.co/abc?x=1', is_bot: 0, bot_type: null, device_type: 'desktop', browser: 'chrome', os: 'windows',
+ authenticated: 0, created_at: sqlTime(Date.parse('2026-09-23T10:15:00Z')),
+ };
+ const problems = event.checkRow(legacy).map((p) => p.split(':')[0]);
+ for (const f of ['ip', 'user_id', 'city', 'route', 'session_id', 'ua_class', 'referer_origin']) assert.ok(problems.includes(f), `${f} flagged: ${problems}`);
+ assert.strictEqual(event.toEvent(legacy).timestamp, '2026-09-23T10:15:00Z');
+ });
+
+ await check('the pre-ADR export names are still there (1.x), backed by the ADR-021 code', () => {
+ const root = require('openvibe-shared');
+ assert.strictEqual(root.AnalyticsTracker, AnalyticsTracker);
+ assert.strictEqual(root.classifyRequest, privacy.classifyRequest);
+ assert.strictEqual(root.parseUserAgent, privacy.parseUserAgent);
+ assert.strictEqual(root.BOT_USER_AGENTS, privacy.BOT_USER_AGENTS);
+ assert.strictEqual(root.ANALYTICS_SCHEMA, analytics.schema.ANALYTICS_SCHEMA);
+ assert.strictEqual(root.SUSPICIOUS_PATTERNS.highRequestRate, privacy.HIGH_REQUEST_RATE);
+ assert.ok(Object.isFrozen(root.SUSPICIOUS_PATTERNS));
+ assert.deepStrictEqual(privacy.classifyRequest({ headers: { 'user-agent': 'curl/8.5.0' } }).botType, 'known_crawler');
+ // The old constructor call (db, service) still works; the prune timer is on by default.
+ const db = newDb('compat.db');
+ const t = new AnalyticsTracker(db, 'live');
+ assert.ok(t.pruneJob && typeof t.pruneJob.run === 'function');
+ t.destroy();
+ assert.strictEqual(t.pruneJob, null);
+ const off = new AnalyticsTracker(db, 'live', { retention: false });
+ assert.strictEqual(off.pruneJob, null);
+ off.destroy();
+ db.close();
+ });
+
+ fs.rmSync(tmp, { recursive: true, force: true });
+ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }
+ console.log('\nanalytics privacy: all passed');
+})();
new file mode 100644
new file mode 100644
@@ -0,0 +1,160 @@
@@ -0,0 +1,160 @@
+'use strict';
+/**
+ * ADR-021 retention (analytics/retention.js):
+ * - prune deletes strictly-older raw rows in bounded batches and never touches rollups;
+ * - schedulePrune (the tracker's timer, or a service's own job) prunes and reports;
+ * - scrub rewrites legacy rows and rollup top lists without changing any rollup counter, with a
+ * service's path options (Network's by-username rule), and touches no other table.
+ *
+ * node test/analytics-retention.test.js
+ */
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const Database = require('better-sqlite3');
+const { AnalyticsTracker, retention, event } = require('openvibe-shared/analytics');
+const { sqlTime } = require('openvibe-shared/analytics/tracker');
+
+let failures = 0;
+async function check(name, fn) {
+ try { await fn(); console.log(' ✓', name); }
+ catch (e) { failures++; console.log(' ✗', name, '\n ', e.stack); }
+}
+
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-shared-retention-'));
+const newDb = (name) => { const db = new Database(path.join(tmp, name)); db.pragma('journal_mode = WAL'); return db; };
+const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
+const NETWORK = { paramPrefixes: ['avatar', 'anon', 'projects'], pathRules: [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']] };
+
+function dumpAll(db) {
+ const tables = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").pluck().all();
+ return tables.map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');
+}
+
+/** Rows as the pre-ADR tracker wrote them, around a 30-day cutoff from nowMs. */
+function seedLegacy(db, nowMs) {
+ new AnalyticsTracker(db, 'live', { timers: false }).destroy(); // schema
+ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, method, status_code, response_time_ms, user_id, session_id, ip, city, user_agent, referer, is_bot, created_at)
+ VALUES ('live', 'pageview', ?, 'GET', 200, 5, ?, ?, ?, ?, ?, ?, 0, ?)`);
+ const at = (days, s = 0) => sqlTime(nowMs - days * 86400000 + s * 1000);
+ const cutoffMs = nowMs - 30 * 86400000;
+ const rows = [];
+ for (let i = 0; i < 10; i++) rows.push(['/@old' + i, 5, 'eyJhbGciOi.tokentail' + i, '198.51.100.' + i, 'Lyon', CHROME, 'https://t.co/abc?x=1', at(45 + i)]);
+ rows.push(['/@edge-older', 5, null, '198.51.100.50', null, CHROME, '', sqlTime(cutoffMs - 1000)]);
+ rows.push(['/@edge-exact', 5, null, '198.51.100.51', null, CHROME, '', sqlTime(cutoffMs)]);
+ rows.push(['/@edge-newer', null, null, '198.51.100.52', null, FIREFOX, '', sqlTime(cutoffMs + 1000)]);
+ rows.push(['/vod/12345?t=9', 9, 'abcdef0123456789', '198.51.100.53', 'Oslo', FIREFOX, 'https://www.reddit.com/r/x/comments/1', at(2)]);
+ for (const r of rows) ins.run(...r);
+ const hr = db.prepare('INSERT INTO analytics_hourly (service, hour, pageviews, api_calls, unique_visitors, unique_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?)');
+ const dy = db.prepare('INSERT INTO analytics_daily (service, date, pageviews, api_calls, unique_visitors, unique_users, new_users, top_paths, top_referers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
+ const tops = JSON.stringify([{ path: '/@alex', cnt: 3 }, { path: '/@bob', cnt: 2 }, { path: '/vods', cnt: 1 }]);
+ const refs = JSON.stringify([{ referer: 'https://www.google.com/search?q=alex', cnt: 4 }, { referer: 'https://www.google.com/', cnt: 1 }]);
+ hr.run('live', at(60).slice(0, 13) + ':00:00', 10, 4, 3, 1, tops, refs);
+ dy.run('live', at(60).slice(0, 10), 100, 40, 30, 10, 2, tops, refs);
+ dy.run('live', at(2).slice(0, 10), 50, 20, 15, 5, 1, tops, refs);
+ db.prepare("INSERT INTO analytics_rate_tracking (ip, window_start, hit_count) VALUES ('198.51.100.99', 1, 1)").run();
+}
+
+(async () => {
+ await check('prune: strictly older than the cutoff, bounded batches, rollups untouched', async () => {
+ const db = newDb('prune.db');
+ const nowMs = Date.parse('2026-09-23T12:00:00Z');
+ seedLegacy(db, nowMs);
+ const totals = retention.rollupTotals(db);
+ const bounded = await retention.pruneRawEvents(db, { days: 30, batchSize: 3, maxBatches: 2, now: () => nowMs });
+ assert.strictEqual(bounded.deleted, 6);
+ assert.strictEqual(bounded.complete, false);
+ const out = await retention.pruneRawEvents(db, { days: 30, batchSize: 3, now: () => nowMs });
+ assert.strictEqual(out.deleted, 5);
+ assert.strictEqual(out.cutoff, '2026-08-24 12:00:00');
+ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY created_at').pluck().all(), ['/@edge-exact', '/@edge-newer', '/vod/12345?t=9']);
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_daily').pluck().get(), 2, 'old rollups stay');
+ assert.strictEqual(db.prepare('SELECT COUNT(*) FROM analytics_rate_tracking').pluck().get(), 0);
+ assert.strictEqual((await retention.pruneRawEvents(db, { days: 30, now: () => nowMs })).deleted, 0);
+ assert.throws(() => retention.cutoffFor(31), /1 to 30/);
+ assert.throws(() => retention.cutoffFor(0), /1 to 30/);
+ db.close();
+ });
+
+ await check('schedulePrune: run() prunes and logs, overlapping runs are skipped, days > 30 refused', async () => {
+ const db = newDb('job.db');
+ const nowMs = Date.parse('2026-09-23T12:00:00Z');
+ seedLegacy(db, nowMs);
+ const totals = retention.rollupTotals(db);
+ const logs = [];
+ const job = retention.schedulePrune(db, { initialDelayMs: 1e9, intervalMs: 1e9, label: 'Analytics:test', log: (l) => logs.push(l), now: () => nowMs });
+ try {
+ const [a, b] = await Promise.all([job.run(), job.run()]);
+ assert.strictEqual(a.deleted, 11);
+ assert.strictEqual(b, null, 'overlap skipped');
+ assert.ok(/^\[Analytics:test\] pruned 11 raw events older than 2026-08-24 12:00:00$/.test(logs[0]), logs.join('\n'));
+ } finally { job.stop(); }
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ assert.throws(() => retention.schedulePrune(db, { days: 90 }), /1 to 30/);
+ // The tracker's own timer is the same job.
+ const t = new AnalyticsTracker(db, 'live', { retention: { initialDelayMs: 1e9, intervalMs: 1e9, log: () => {} }, now: () => nowMs });
+ assert.strictEqual((await t.pruneJob.run()).deleted, 0);
+ t.destroy();
+ db.close();
+ });
+
+ await check('scrub: legacy rows and rollup top lists reduced, counters unchanged, rows now event.v1', async () => {
+ const db = newDb('scrub.db');
+ const nowMs = Date.parse('2026-09-23T12:00:00Z');
+ seedLegacy(db, nowMs);
+ await retention.pruneRawEvents(db, { days: 30, now: () => nowMs });
+ const totals = retention.rollupTotals(db);
+ const s = await retention.scrubEvents(db, { batchSize: 2 });
+ assert.strictEqual(s.rows, 3);
+ assert.deepStrictEqual(retention.scrubRollups(db), { hourly: 1, daily: 2 });
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ const rows = db.prepare('SELECT * FROM analytics_events ORDER BY created_at').all();
+ assert.deepStrictEqual(rows.map((x) => x.path), ['/@:user', '/@:user', '/vod/:param']);
+ assert.deepStrictEqual(rows.map((x) => x.authenticated), [1, 0, 1]);
+ assert.deepStrictEqual(rows.map((x) => x.session_id), [null, null, 'abcdef0123456789']);
+ assert.deepStrictEqual(rows.map((x) => x.referer), ['', '', 'https://www.reddit.com']);
+ assert.deepStrictEqual(rows.map((x) => x.user_agent), ['chrome/windows/desktop', 'firefox/linux/desktop', 'firefox/linux/desktop']);
+ for (const x of rows) assert.deepStrictEqual(event.checkRow(x), [], JSON.stringify(x));
+ const daily = db.prepare('SELECT top_paths, top_referers FROM analytics_daily ORDER BY date LIMIT 1').get();
+ assert.deepStrictEqual(JSON.parse(daily.top_paths), [{ path: '/@:user', cnt: 5 }, { path: '/vods', cnt: 1 }]);
+ assert.deepStrictEqual(JSON.parse(daily.top_referers), [{ referer: 'https://www.google.com', cnt: 5 }]);
+ const all = dumpAll(db);
+ for (const needle of ['198.51.100.', 'Lyon', 'Oslo', 'eyJhbGciOi', 'q=alex', '@alex', '/vod/12345', 'Mozilla/5.0']) assert.ok(!all.includes(needle), needle);
+ await retention.scrubEvents(db);
+ assert.deepStrictEqual(retention.scrubRollups(db), { hourly: 0, daily: 0 }, 'idempotent');
+ assert.deepStrictEqual(retention.rollupTotals(db), totals);
+ db.close();
+ });
+
+ await check('scrub with Network path options; other tables in the same database untouched', async () => {
+ const db = newDb('network-scrub.db');
+ db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT); INSERT INTO users VALUES (7, 'keepme');");
+ new AnalyticsTracker(db, 'openvibe-network', { timers: false }).destroy();
+ const ins = db.prepare(`INSERT INTO analytics_events (service, event_type, path, user_id, ip, user_agent, created_at) VALUES ('openvibe-network', 'pageview', ?, 7, '127.0.0.1', 'node', ?)`);
+ const now = sqlTime(Date.now());
+ for (const p of ['/avatar/keepme?s=1', '/api/auth/anon/tok0X9zz99', '/internal/users/by-username/keepme']) ins.run(p, now);
+ db.prepare("INSERT INTO analytics_daily (service, date, pageviews, top_paths) VALUES ('openvibe-network', '2026-07-01', 5, ?)")
+ .run(JSON.stringify([{ path: '/internal/users/by-username/keepme', cnt: 2 }, { path: '/avatar/keepme', cnt: 3 }]));
+ const before = retention.inspect(db, NETWORK);
+ assert.strictEqual(before.toScrub.paths, 3);
+ await retention.scrubEvents(db, NETWORK);
+ retention.scrubRollups(db, NETWORK);
+ assert.deepStrictEqual(db.prepare('SELECT path FROM analytics_events ORDER BY id').pluck().all(),
+ ['/avatar/:param', '/api/auth/anon/:param', '/internal/users/:param/:username']);
+ assert.deepStrictEqual(JSON.parse(db.prepare('SELECT top_paths FROM analytics_daily').pluck().get()),
+ [{ path: '/avatar/:param', cnt: 3 }, { path: '/internal/users/:param/:username', cnt: 2 }]);
+ assert.strictEqual(retention.inspect(db, NETWORK).toScrub.paths, 0);
+ assert.strictEqual(db.prepare('SELECT username FROM users WHERE id = 7').pluck().get(), 'keepme');
+ const analyticsOnly = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'analytics_%'").pluck().all()
+ .map((t) => JSON.stringify(db.prepare(`SELECT * FROM ${t}`).all())).join('\n');
+ for (const needle of ['keepme', 'X9zz99', '127.0.0.1']) assert.ok(!analyticsOnly.includes(needle), needle);
+ db.close();
+ });
+
+ fs.rmSync(tmp, { recursive: true, force: true });
+ if (failures) { console.log(`\n${failures} failed`); process.exit(1); }
+ console.log('\nanalytics retention: all passed');
+})();
@@ -7,7 +7,8 @@ const path = require('path');
@@ -7,7 +7,8 @@ const path = require('path');
const pkg = require('../package.json');
const pkg = require('../package.json');
const ROOT = path.join(__dirname, '..');
const ROOT = path.join(__dirname, '..');
-const modules = fs.readdirSync(ROOT).filter((f) => f.endsWith('.js'));
+const modules = fs.readdirSync(ROOT).filter((f) => f.endsWith('.js'))
+ .concat(fs.readdirSync(path.join(ROOT, 'analytics')).filter((f) => f.endsWith('.js')).map((f) => `analytics/${f}`));
const targets = new Set(Object.values(pkg.exports).map((t) => t.replace(/^\.\//, '')));
const targets = new Set(Object.values(pkg.exports).map((t) => t.replace(/^\.\//, '')));
for (const f of modules) assert.ok(targets.has(f), `${f} is exported`);
for (const f of modules) assert.ok(targets.has(f), `${f} is exported`);
for (const [sub, target] of Object.entries(pkg.exports)) {
for (const [sub, target] of Object.entries(pkg.exports)) {
@@ -20,6 +21,14 @@ for (const sub of ['analytics', 'app-icon', 'auth-client', 'brand', 'chrome-ssr'
@@ -20,6 +21,14 @@ for (const sub of ['analytics', 'app-icon', 'auth-client', 'brand', 'chrome-ssr'
}
}
// Server modules load in Node; the navbar module loads too (UMD, no DOM touched at require time).
// Server modules load in Node; the navbar module loads too (UMD, no DOM touched at require time).
for (const sub of ['analytics', 'app-icon', 'brand', 'chrome-ssr', 'footer', 'icons', 'legal', 'notifications', 'seo', 'url-resolver', 'files', 'nav-icons', 'metrics', 'ready']) require(`openvibe-shared/${sub}`);
for (const sub of ['analytics', 'app-icon', 'brand', 'chrome-ssr', 'footer', 'icons', 'legal', 'notifications', 'seo', 'url-resolver', 'files', 'nav-icons', 'metrics', 'ready']) require(`openvibe-shared/${sub}`);
+// analytics is dependency-free: its parts require only Node built-ins and each other.
+for (const f of fs.readdirSync(path.join(ROOT, 'analytics'))) {
+ const src = fs.readFileSync(path.join(ROOT, 'analytics', f), 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
+ for (const [, dep] of src.matchAll(/require\((['"])([^'"]+)\1\)/g).map((m) => [m[1], m[2]])) {
+ assert.ok(dep.startsWith('.') || ['fs', 'path', 'crypto'].includes(dep), `analytics/${f} requires ${dep}`);
+ }
+}
+assert.strictEqual(require('openvibe-shared/analytics/event.v1.json').$id, 'https://openvibe.network/contracts/analytics/event.v1.json');
assert.strictEqual(typeof require('openvibe-shared/navbar').init, 'function');
assert.strictEqual(typeof require('openvibe-shared/navbar').init, 'function');
assert.ok(/^\d+\.\d+\.\d+$/.test(pkg.version), 'semver version');
assert.ok(/^\d+\.\d+\.\d+$/.test(pkg.version), 'semver version');
assert.ok(fs.readFileSync(path.join(ROOT, 'CHANGELOG.md'), 'utf8').includes(`## ${pkg.version}`), 'CHANGELOG has an entry for this version');
assert.ok(fs.readFileSync(path.join(ROOT, 'CHANGELOG.md'), 'utf8').includes(`## ${pkg.version}`), 'CHANGELOG has an entry for this version');