openvibers/openvibe.network Commits
Network keeps raw bot analytics 7 days (visitors 30): 1.22M of its 2.46M raw rows were bots (#315)
Changes
@@@ -35,13 +35,17 @@ const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];const PATH_OPTS = { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES };++// Raw events are kept 30 days (ADR-021); bot hits only 7, since their daily counts are in the rollups once the day++// closes: on 2026-10-23, 1.22M of the 2.46M raw rows were bots (openvibe-shared 4.10.0's botDays).++const RETENTION = { days: 30, botDays: 7 };++function openAnalytics(db, opts = {}) {-- return new AnalyticsTrackerPg(db, SERVICE, { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES, ...opts });++ return new AnalyticsTrackerPg(db, SERVICE, { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES, retention: RETENTION, ...opts });}/** ADR-021 raw-event retention on demand (the tracker also schedules it at boot, after 5 minutes). */function schedulePrune(tracker, opts = {}) {-- return pruneRawEventsPg(tracker.db, opts);++ return pruneRawEventsPg(tracker.db, { ...RETENTION, ...opts });}--module.exports = { SERVICE, PARAM_PREFIXES, PATH_RULES, PATH_OPTS, analyticsSchema, openAnalytics, schedulePrune };++module.exports = { SERVICE, PARAM_PREFIXES, PATH_RULES, PATH_OPTS, RETENTION, analyticsSchema, openAnalytics, schedulePrune };@@@ -200,6 +200,15 @@ async function dumpAnalytics(db) {assert.strictEqual(JSON.stringify(await db.prepare('SELECT * FROM analytics_daily ORDER BY date').all()), daily);assert.strictEqual(await db.prepare('SELECT COUNT(*) FROM users').pluck().get(), 1);});++ await check('analytics-prune: bot rows go after 7 days (RETENTION.botDays), visitors after 30', async () => {++ const at = sqlTime(Date.now() - 10 * 86400000);++ await db.prepare(`INSERT INTO analytics_events (service, event_type, path, is_bot, created_at) VALUES ('openvibe-network', 'api_call', '/bot-10d', 1, ?)`).run(at);++ await db.prepare(`INSERT INTO analytics_events (service, event_type, path, is_bot, created_at) VALUES ('openvibe-network', 'pageview', '/human-10d', 0, ?)`).run(at);++ assert.deepStrictEqual(networkAnalytics.RETENTION, { days: 30, botDays: 7 });++ await networkAnalytics.schedulePrune(analytics);++ assert.strictEqual(await db.prepare("SELECT COUNT(*) FROM analytics_events WHERE path = '/bot-10d'").pluck().get(), 0, 'a 10-day-old bot row goes');++ assert.strictEqual(await db.prepare("SELECT COUNT(*) FROM analytics_events WHERE path = '/human-10d'").pluck().get(), 1, 'a 10-day-old visitor row stays');++ });analytics.destroy();if (failures) { console.log(`\n${failures} failed`); process.exit(1); }
@@ -35,13 +35,17 @@ const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];
@@ -35,13 +35,17 @@ const PATH_RULES = [[/\/by-username\/[^/?#]+/gi, '/by-username/:username']];
const PATH_OPTS = { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES };
const PATH_OPTS = { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES };
+// Raw events are kept 30 days (ADR-021); bot hits only 7, since their daily counts are in the rollups once the day
+// closes: on 2026-10-23, 1.22M of the 2.46M raw rows were bots (openvibe-shared 4.10.0's botDays).
+const RETENTION = { days: 30, botDays: 7 };
+
function openAnalytics(db, opts = {}) {
function openAnalytics(db, opts = {}) {
- return new AnalyticsTrackerPg(db, SERVICE, { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES, ...opts });
+ return new AnalyticsTrackerPg(db, SERVICE, { paramPrefixes: PARAM_PREFIXES, pathRules: PATH_RULES, retention: RETENTION, ...opts });
}
}
/** ADR-021 raw-event retention on demand (the tracker also schedules it at boot, after 5 minutes). */
/** ADR-021 raw-event retention on demand (the tracker also schedules it at boot, after 5 minutes). */
function schedulePrune(tracker, opts = {}) {
function schedulePrune(tracker, opts = {}) {
- return pruneRawEventsPg(tracker.db, opts);
+ return pruneRawEventsPg(tracker.db, { ...RETENTION, ...opts });
}
}
-module.exports = { SERVICE, PARAM_PREFIXES, PATH_RULES, PATH_OPTS, analyticsSchema, openAnalytics, schedulePrune };
+module.exports = { SERVICE, PARAM_PREFIXES, PATH_RULES, PATH_OPTS, RETENTION, analyticsSchema, openAnalytics, schedulePrune };
@@ -200,6 +200,15 @@ async function dumpAnalytics(db) {
@@ -200,6 +200,15 @@ async function dumpAnalytics(db) {
assert.strictEqual(JSON.stringify(await db.prepare('SELECT * FROM analytics_daily ORDER BY date').all()), daily);
assert.strictEqual(JSON.stringify(await db.prepare('SELECT * FROM analytics_daily ORDER BY date').all()), daily);
assert.strictEqual(await db.prepare('SELECT COUNT(*) FROM users').pluck().get(), 1);
assert.strictEqual(await db.prepare('SELECT COUNT(*) FROM users').pluck().get(), 1);
});
});
+ await check('analytics-prune: bot rows go after 7 days (RETENTION.botDays), visitors after 30', async () => {
+ const at = sqlTime(Date.now() - 10 * 86400000);
+ await db.prepare(`INSERT INTO analytics_events (service, event_type, path, is_bot, created_at) VALUES ('openvibe-network', 'api_call', '/bot-10d', 1, ?)`).run(at);
+ await db.prepare(`INSERT INTO analytics_events (service, event_type, path, is_bot, created_at) VALUES ('openvibe-network', 'pageview', '/human-10d', 0, ?)`).run(at);
+ assert.deepStrictEqual(networkAnalytics.RETENTION, { days: 30, botDays: 7 });
+ await networkAnalytics.schedulePrune(analytics);
+ assert.strictEqual(await db.prepare("SELECT COUNT(*) FROM analytics_events WHERE path = '/bot-10d'").pluck().get(), 0, 'a 10-day-old bot row goes');
+ assert.strictEqual(await db.prepare("SELECT COUNT(*) FROM analytics_events WHERE path = '/human-10d'").pluck().get(), 1, 'a 10-day-old visitor row stays');
+ });
analytics.destroy();
analytics.destroy();
if (failures) { console.log(`\n${failures} failed`); process.exit(1); }
if (failures) { console.log(`\n${failures} failed`); process.exit(1); }