openvibers/openvibe.live Commits

Analytics: count bot API calls at ingest, not as raw rows (J3) (#615)

OP OpenVibers committed 7 days ago · daf935cf6c · 8 files, +125 −18

* Analytics: count bot API calls at ingest, not as raw rows (J3)

* openvibe-shared v4.113.0 (was v4.112.1)

Changes

ddiff --git a/README.md b/README.md
iindex 8fa46b59..f4bcb8dd 100644
---- a/README.md
++++ b/README.md
@@@ -177,5 +177,5 @@ Minimum `.env`: `BASE_URL`, `JWT_SECRET`, `OV_NETWORK_URL`, `OV_NETWORK_INTERNAL
  <!-- versions:start -->
  - openvibe-contracts: v0.289.1
  - openvibe-sdk: v0.82.0
--- openvibe-shared: v4.112.1
++- openvibe-shared: v4.113.0
  <!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 1a8c4bbe..0d536e6b 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -7,7 +7,7 @@
    "domain": "openvibe.live (www, ingest, whip)",
    "contracts": "openvibe-contracts v0.289.1",
    "sdk": "openvibe-sdk v0.82.0",
--  "shared": "openvibe-shared v4.112.1",
++  "shared": "openvibe-shared v4.113.0",
    "capabilities": [
      "live.chat_context.read",
      "live.chat_effects.write",
ddiff --git a/migrations/0026_analytics_bot_api_rollups.sql b/migrations/0026_analytics_bot_api_rollups.sql
nnew file mode 100644
iindex 00000000..095cfd60
---- /dev/null
++++ b/migrations/0026_analytics_bot_api_rollups.sql
@@@ -0,0 +1,21 @@
++-- phase: expand
++-- J3 (2026-11-06): bot API calls are counted at ingest instead of written as one analytics_events row each
++-- (openvibe-shared's countBotApiAtIngest, server/analytics/store.js). The counters are the hour's and day's totals
++-- by endpoint group, status class and bot family; the shared tracker adds them to analytics_hourly and
++-- analytics_daily when it computes those, so a day's api_calls, bot_hits and error_count stay what the raw rows
++-- gave before.
++-- Expand-only: new tables, the existing analytics tables are unchanged. The runtime role creates none.
++CREATE TABLE IF NOT EXISTS analytics_bot_api_hourly (
++    id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
++    service text NOT NULL, hour text NOT NULL,
++    endpoint_group text NOT NULL, status_class text NOT NULL, bot_family text NOT NULL,
++    hits bigint NOT NULL DEFAULT 0,
++    UNIQUE (service, hour, endpoint_group, status_class, bot_family)
++);
++CREATE TABLE IF NOT EXISTS analytics_bot_api_daily (
++    id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
++    service text NOT NULL, date text NOT NULL,
++    endpoint_group text NOT NULL, status_class text NOT NULL, bot_family text NOT NULL,
++    hits bigint NOT NULL DEFAULT 0,
++    UNIQUE (service, date, endpoint_group, status_class, bot_family)
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex 684561ed..01e0b6f4 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -25,7 +25,7 @@
          "onnxruntime-node": "^1.20.0",
          "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.289.1",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.82.0",
--        "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.112.1",
++        "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.113.0",
          "pg": "^8.23.1",
          "sanitize-html": "^2.17.7",
          "sdp-transform": "^3.0.0",
@@@ -2884,9 +2884,9 @@
        }
      },
      "node_modules/openvibe-shared": {
--      "version": "4.112.1",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.112.1",
--      "integrity": "sha512-WuiE0SP+GFh6QDLzbnicjX0Dkgf96d+jXxZDSNfN1tmoEUxYr/2xQDmbGXpNavJjR7fqh5DJ8jSAXgE0KPQfIw==",
++      "version": "4.113.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.113.0",
++      "integrity": "sha512-sYZ5bwT2YdBCDhckJEEhRUM+dgu35v5cvxAFAugMpxrPMbp31uvGgl+bNQYZsZZya65y+1pyRG44FHQyzgcTcA==",
        "license": "MIT",
        "dependencies": {
          "jsonwebtoken": "^9.0.0",
ddiff --git a/package.json b/package.json
iindex 80153574..d6f6cc26 100644
---- a/package.json
++++ b/package.json
@@@ -44,7 +44,7 @@
      "onnxruntime-node": "^1.20.0",
      "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.289.1",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.82.0",
--    "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.112.1",
++    "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.113.0",
      "pg": "^8.23.1",
      "sanitize-html": "^2.17.7",
      "sdp-transform": "^3.0.0",
ddiff --git a/server/analytics/store.js b/server/analytics/store.js
iindex 3fff7c87..7d54add3 100644
---- a/server/analytics/store.js
++++ b/server/analytics/store.js
@@@ -1,8 +1,9 @@
  'use strict';
  /**
   * Page-view analytics (openvibe-shared/analytics/pg, ADR-021) on Live's own PostgreSQL database (plan T4): the tables
-- * are migrations/0001_analytics.sql. record() stays synchronous — the tracker buffers and flushes in the background —
-- * so no request waits on an analytics write. A restore drill runs no timers (LIVE_DRILL).
++ * are migrations/0001_analytics.sql and, for the ingest counters, 0026_analytics_bot_api_rollups.sql. record() stays
++ * synchronous — the tracker buffers and flushes in the background — so no request waits on an analytics write. A
++ * restore drill runs no timers (LIVE_DRILL).
   */
  const analyticsModule = require('openvibe-shared/analytics');
  const { AnalyticsTrackerPg, pruneRawEventsPg } = require('openvibe-shared/analytics/pg');
@@@ -19,10 +20,10 @@ function processDb() {
      });
  }
  
--// Crawler and bot rows are two thirds of the raw table (2026-10-22: 1.65M of 2.47M rows in 30 days, 1.49M of them bot
--// API calls; the table was 1.1 GB, 70% of ov_live). They are counted in the hourly and daily rollups when their day
--// closes (the tracker recomputes only today and yesterday), so after a week only the admin's bot breakdown would read
--// them: they are kept 7 days, the window its suspicious-sessions check already uses, and people's events 30.
++// Bot API calls are counted into the hourly and daily rollups at ingest (countBotApiAtIngest), so no raw row is
++// written for them any more (plan J3: they were 1.49M of 2.47M raw rows on 2026-10-22 and about 80k a day on
++// 2026-11-06). Page views by bots, and a bot API call the shared tracker could not count, still get a raw row: those
++// are kept 7 days, the window the admin's bot breakdown and its suspicious-sessions check use, and people's events 30.
  const BOT_RAW_DAYS = 7;
  const DAY_MS = 24 * 60 * 60 * 1000;
  const sqlTime = (ms) => new Date(ms).toISOString().replace('T', ' ').slice(0, 19);
@@@ -39,13 +40,17 @@ async function pruneBotEvents(db, { days = BOT_RAW_DAYS, now = Date.now, batch =
      return { removed, cutoff };
  }
  
--function createAnalyticsStore({ db = processDb(), drill = false, timers = !drill } = {}) {
--    const tracker = new AnalyticsTrackerPg(db, 'live', { retention: false, timers });
++function createAnalyticsStore({ db = processDb(), drill = false, timers = !drill, countBotApiAtIngest = true } = {}) {
++    const tracker = new AnalyticsTrackerPg(db, 'live', { retention: false, timers, countBotApiAtIngest });
      let closing;
      return {
          tracker,
--        // The owner's migration made the table; the runtime role has no DDL rights.
--        async ready() { await db.value('SELECT 1 FROM analytics_events LIMIT 1'); },
++        // The owner's migrations made the tables; the runtime role has no DDL rights.
++        async ready() {
++            await db.value('SELECT 1 FROM analytics_events LIMIT 1');
++            await db.value('SELECT 1 FROM analytics_bot_api_hourly LIMIT 1');
++            await db.value('SELECT 1 FROM analytics_bot_api_daily LIMIT 1');
++        },
          async prune() {
              const all = await pruneRawEventsPg(db, { days: analyticsModule.retention.MAX_DAYS });
              const bots = await pruneBotEvents(db);
ddiff --git a/test/analytics-bot-ingest.test.js b/test/analytics-bot-ingest.test.js
nnew file mode 100644
iindex 00000000..d37d5475
---- /dev/null
++++ b/test/analytics-bot-ingest.test.js
@@@ -0,0 +1,81 @@
++'use strict';
++/**
++ * J3: Live counts bot API calls at ingest (openvibe-shared's countBotApiAtIngest, server/analytics/store.js), so a
++ * bot's API call makes no raw analytics_events row; page views by bots still do, and the option-off path is
++ * unchanged. The counters survive the 7-day raw-row prune, and the hour's and day's rollups keep the raw rollup's
++ * api_calls, bot_hits and error_count.
++ *
++ *   node test/analytics-bot-ingest.test.js   (with the test preload; `ov test` loads it)
++ */
++const assert = require('assert');
++const db = require('../server/db/database');
++const { createAnalyticsStore } = require('../server/analytics/store');
++
++const BOT_UA = 'curl/8.5.0';
++const HUMAN_UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
++const req = (path, ua, ip) => ({ path, method: 'GET', ip, headers: { 'user-agent': ua }, user: null });
++const res = (status = 200) => ({ statusCode: status });
++
++async function main() {
++    await db.initDb();
++    const d = db.getDb();
++    const store = createAnalyticsStore({ timers: false });
++    try {
++        await store.ready();   // includes the 0026 counter tables
++        const t = store.tracker;
++
++        assert.strictEqual(t.record(req('/api/j3/streams/abc', BOT_UA, '203.0.113.10'), res(200), '/api/j3/streams/abc', 10), true);
++        assert.strictEqual(t.record(req('/api/j3/streams/abc', BOT_UA, '203.0.113.10'), res(404), '/api/j3/streams/abc', 10), true);
++        assert.strictEqual(t.record(req('/j3/about', BOT_UA, '203.0.113.10'), res(200), '/j3/about', 10), true);
++        assert.strictEqual(t.record(req('/api/j3/vods/xyz', HUMAN_UA, '198.51.100.20'), res(200), '/api/j3/vods/xyz', 10), true);
++        await t.flush();
++
++        const raw = await d.many("SELECT event_type, is_bot FROM analytics_events WHERE service = 'live' ORDER BY id");
++        assert.deepStrictEqual(raw.map((r) => [r.event_type, r.is_bot]), [['pageview', 1], ['api_call', 0]], 'no raw row for a bot API call; the bot page view and the person\'s call are rows');
++
++        const counters = await d.many(`SELECT endpoint_group, status_class, bot_family, SUM(hits)::bigint AS hits
++            FROM analytics_bot_api_hourly WHERE service = 'live' GROUP BY 1, 2, 3 ORDER BY 1, 2`);
++        assert.deepStrictEqual(counters.map((r) => [r.endpoint_group, r.status_class, r.bot_family, Number(r.hits)]), [
++            ['/api/j3/streams', '2xx', 'known_crawler', 1],
++            ['/api/j3/streams', '4xx', 'known_crawler', 1],
++        ]);
++        assert.strictEqual(Number(await d.value("SELECT SUM(hits) FROM analytics_bot_api_daily WHERE service = 'live'")), 2);
++
++        await t.aggregate();
++        const hour = await d.maybe("SELECT pageviews, api_calls, bot_hits, error_count FROM analytics_hourly WHERE service = 'live' ORDER BY hour DESC LIMIT 1");
++        assert.deepStrictEqual([hour.pageviews, hour.api_calls, hour.bot_hits, hour.error_count].map(Number), [1, 3, 3, 1], 'the hour keeps the raw rollup\'s counters');
++        const day = await d.maybe("SELECT pageviews, api_calls, bot_hits, error_count FROM analytics_daily WHERE service = 'live' ORDER BY date DESC LIMIT 1");
++        assert.deepStrictEqual([day.pageviews, day.api_calls, day.bot_hits, day.error_count].map(Number), [1, 3, 3, 1], 'the day too');
++
++        const stats = await t.getStats({ hours: 6 });
++        // A sub-day summary's pageviews are people's only; bots are in total_bot_hits, including the counted API calls.
++        assert.deepStrictEqual([stats.summary.total_pageviews, stats.summary.total_bot_hits, stats.summary.total_errors].map(Number), [0, 3, 1]);
++        assert.deepStrictEqual(stats.botBreakdown.map((r) => [r.bot_type, Number(r.hits)]), [['known_crawler', 3]]);
++        const bots = await t.getBotAnalysis(7);
++        assert.deepStrictEqual(bots.botTypes.map((r) => [r.bot_type, Number(r.hits)]), [['known_crawler', 3]]);
++        assert.deepStrictEqual(bots.botApi.map((r) => [r.endpoint_group, r.status_class, Number(r.hits)]), [['/api/j3/streams', '2xx', 1], ['/api/j3/streams', '4xx', 1]]);
++
++        // Option off keeps the old behaviour: a raw row, no counter.
++        const off = createAnalyticsStore({ timers: false, countBotApiAtIngest: false });
++        try {
++            off.tracker.record(req('/api/j3/off', BOT_UA, '203.0.113.30'), res(200), '/api/j3/off', 10);
++            await off.tracker.flush();
++        } finally { await off.close(); }
++        assert.strictEqual(Number(await d.value("SELECT COUNT(*) FROM analytics_events WHERE service = 'live' AND path = '/api/j3/off' AND is_bot = 1")), 1, 'option off writes the raw row');
++        assert.strictEqual(Number(await d.value("SELECT SUM(hits) FROM analytics_bot_api_daily WHERE service = 'live'")), 2, 'option off adds no counter');
++
++        // The counters outlive the raw rows: the 7-day bot prune leaves them, so the admin's families stay readable.
++        const tenDaysAgo = new Date(Date.now() - 10 * 86400000).toISOString().replace('T', ' ').slice(0, 19);
++        await d.exec("INSERT INTO analytics_events (service, event_type, path, is_bot, created_at) VALUES ('live', 'api_call', '/j3/old-bot', 1, $1)", [tenDaysAgo]);
++        await d.exec("INSERT INTO analytics_bot_api_daily (service, date, endpoint_group, status_class, bot_family, hits) VALUES ('live', '2026-10-01', '/j3', '2xx', 'known_crawler', 5)");
++        const pruned = await store.prune();
++        assert.strictEqual(pruned.bots.removed, 1, JSON.stringify(pruned));
++        assert.strictEqual(Number(await d.value("SELECT COUNT(*) FROM analytics_events WHERE path = '/j3/old-bot'")), 0);
++        assert.strictEqual(Number(await d.value("SELECT hits FROM analytics_bot_api_daily WHERE date = '2026-10-01' AND endpoint_group = '/j3'")), 5, 'the counter survives the raw prune');
++    } finally {
++        await store.close();
++    }
++    console.log('analytics bot ingest: all checks passed');
++}
++
++main().catch((err) => { console.error(err); process.exitCode = 1; });
ddiff --git a/test/analytics-privacy.test.js b/test/analytics-privacy.test.js
iindex a859ea56..26dc3481 100644
---- a/test/analytics-privacy.test.js
++++ b/test/analytics-privacy.test.js
@@@ -23,7 +23,7 @@ async function check(name, fn) {
  const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
  const FIREFOX = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
  const SUBJECT = 'usr_01J8ZQ4K7M2N3P4Q5R6S7T8V9W';
--const ANALYTICS_TABLES = ['analytics_events', 'analytics_hourly', 'analytics_daily', 'analytics_visitor_days', 'analytics_day_salts'];
++const ANALYTICS_TABLES = ['analytics_events', 'analytics_hourly', 'analytics_daily', 'analytics_bot_api_hourly', 'analytics_bot_api_daily', 'analytics_visitor_days', 'analytics_day_salts'];
  
  (async () => {
      // ── Path templating ──────────────────────────────────────