openvibers/openvibe.blog Commits

Blog: telemetry flushes each minute and keeps 14 days (#54)

OP OpenVibers committed 29 days ago · db976d15bc · 3 files, +7 −7

The request telemetry added earlier today flushed every 15 s and kept 30 days. The idle gauges alone write three rows per flush. Measured in production, that is about 650 rows an hour at about 270 B each including the indexes. At 30 days that's about 470k rows and 125 MB per service, roughly 4.4 GB across the estate on a disk with 16 GB free.

This changes it to one flush a minute and 14 days of retention: about 55k rows and 15 MB per service.

- The collector's interval (the tick, or `intervalMs` for `createHttpTelemetry` and `telemetry.init`) is 60 s.
- `startTelemetryRetention` passes `olderThanDays: 14`.
- The comments and `test/telemetry.test.js` follow: the retention check inserts rows 20 days and 1 day old and expects one deleted with `olderThanDays: 14`.

Applied migrations are unchanged (their comment still says 30 days; a migration that has run is never edited).

Changes

ddiff --git a/server/index.js b/server/index.js
iindex 4b3414b..ef6a3b4 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -34,8 +34,8 @@ async function start() {
      ctx.outbox.start();
      ctx.worker.start();
      // The request collector aggregates in memory and has no timer of its own (openvibe-sdk 0.46 createHttpTelemetry):
--    // this tick writes each 15-second interval's samples; the stop clears it and flushes once more.
--    ctx.telemetryTick = setInterval(() => { ctx.telemetry.flush().catch(() => {}); }, 15_000);
++    // this tick writes each minute's samples; the stop clears it and flushes once more.
++    ctx.telemetryTick = setInterval(() => { ctx.telemetry.flush().catch(() => {}); }, 60_000);
      ctx.telemetryTick.unref();
      // The network changelog and patch notes run where the worker runs (one process).
      if (config.worker.enabled) ctx.changelog.start();
ddiff --git a/server/worker.js b/server/worker.js
iindex c89cb1a..403ff1f 100644
---- a/server/worker.js
++++ b/server/worker.js
@@@ -9,7 +9,7 @@
   *                           objects attached to published posts still exist; gone → the explicit
   *                           broken-asset state; an outage changes nothing (check_failed)
   *   telemetry retention     openvibe-sdk/telemetry startTelemetryRetention(): a minute after start, then once a day,
-- *                           telemetry_samples rows older than 30 days are deleted (a redeploy never pushes it a day out)
++ *                           telemetry_samples rows older than 14 days are deleted (a redeploy never pushes it a day out)
   */
  const { pruneTelemetry, startTelemetryRetention } = require('openvibe-sdk/telemetry');
  
@@@ -46,7 +46,7 @@ function createWorker({ config, store, posts, media, outbox, log = console }) {
      }
  
      async function pruneSamples() {
--        try { return await pruneTelemetry({ db: store.db }); } catch (err) { log.warn('[Blog] telemetry retention failed:', err.message); return null; }
++        try { return await pruneTelemetry({ db: store.db, olderThanDays: 14 }); } catch (err) { log.warn('[Blog] telemetry retention failed:', err.message); return null; }
      }
  
      return {
@@@ -61,7 +61,7 @@ function createWorker({ config, store, posts, media, outbox, log = console }) {
                  mediaTimer = setInterval(() => verifyMedia().catch((err) => log.warn('[Blog] media verification failed:', err.message)), config.media.verifyIntervalMs);
                  mediaTimer.unref();
              }
--            retention = startTelemetryRetention({ db: store.db, log });
++            retention = startTelemetryRetention({ db: store.db, log, olderThanDays: 14 });
              setTimeout(scheduleTick, 1000).unref();
          },
          async stop() { clearInterval(scheduleTimer); clearInterval(mediaTimer); if (retention) await retention.stop(); },
ddiff --git a/test/telemetry.test.js b/test/telemetry.test.js
iindex 16081b3..32f9447 100644
---- a/test/telemetry.test.js
++++ b/test/telemetry.test.js
@@@ -46,10 +46,10 @@ const read = (p) => fs.readFileSync(path.join(__dirname, '..', p), 'utf8');
          assert.deepStrictEqual(requests(), []);
      });
  
--    await check('the worker retention drops telemetry_samples rows older than 30 days and keeps newer ones', async () => {
++    await check('the worker retention drops telemetry_samples rows older than 14 days and keeps newer ones', async () => {
          const db = t.ctx.store.db;
          const insert = (days) => db.query(`INSERT INTO telemetry_samples (at, service, operation) VALUES (now() - make_interval(days => $1), 'blog', 'http.request')`, [days]);
--        await insert(45);
++        await insert(20);
          await insert(1);
          assert.strictEqual(await t.ctx.worker.pruneSamples(), 1);
          const left = await db.query('SELECT count(*)::int AS n FROM telemetry_samples');