openvibers/openvibe.live Commits

Real-user measurements survive restarts (app_state rum_state, saved every 5 minutes and at shutdown) and the host reads the full snapshot at GET /metrics/rum (direct loopback only); at this traffic a deploy reset them before any page had five samples (#231)

OP OpenVibers committed 27 days ago · 2b41d20486 · 3 files, +56 −1

Changes

ddiff --git a/server/index.js b/server/index.js
iindex 08aaf661..c722c566 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -187,6 +187,7 @@ const stopper = gracefulStop({
// No new background runs from here on (the jobs helper's loops); and end SSE streams so
// the drain can actually complete instead of always hitting the deadline.
() => { require('./utils/jobs').stopAll(); },
++ async () => { if (!drill.enabled) await db.setState('rum_state', JSON.stringify(rum.state())).catch(() => {}); },
() => { require('./streaming/live-events').closeAll(); },
],
beforeDrain: async () => {
@@@ -882,6 +883,16 @@ observability.registerDomainGauges(metricsRegistry, {
metricsRegistry.gauge({ name: 'live_rum_p75', help: 'Real-user p75 per page kind and device class (ms; cls unitless), from the newest beacons', labelNames: ['metric', 'page', 'device'], collect: () => rum.p75Gauge() });
metricsRegistry.gauge({ name: 'live_rum_errors', help: 'Script errors visitors reported since this process started: the ten most frequent messages', labelNames: ['message'], collect: () => rum.errorsGauge() });
app.get('/api/admin/rum', requireAuth, permissions.requireAdmin, (req, res) => { res.set('Cache-Control', 'no-store'); res.json(rum.snapshot()); });
++// The same snapshot for the host, direct loopback only like /metrics (nginx refuses /metrics* from outside).
++app.get('/metrics/rum', (req, res) => {
++ if (!require('openvibe-shared/metrics').isLoopbackDirect(req)) return res.status(404).json({ error: 'Not found' });
++ res.set('Cache-Control', 'no-store'); res.json(rum.snapshot());
++});
++// The reservoirs survive restarts: app_state rum_state (numbers only), saved every 5 minutes and at shutdown.
++if (!drill.enabled) {
++ db.getState('rum_state').then((raw) => { try { rum.restore(JSON.parse(raw || 'null')); } catch { /* start empty */ } }).catch(() => {});
++ require('./utils/jobs').every('rum-save', 5 * 60_000, () => db.setState('rum_state', JSON.stringify(rum.state())));
++}
// ── Updates / Changelog ──────────────────────────────────────
const { execFile } = require('child_process');
ddiff --git a/server/web/rum.js b/server/web/rum.js
iindex f42cd416..5b8ec041 100644
---- a/server/web/rum.js
++++ b/server/web/rum.js
@@@ -14,6 +14,8 @@
* rum.p75Gauge() the p75s for /metrics (live_rum_p75{metric,page,device})
* rum.errorsGauge() the ten most frequent error messages for /metrics (live_rum_errors{message}), so they can
* be read on the host without a staff session
++ * rum.state() / restore() the reservoirs as numbers, kept across restarts (at this traffic a deploy otherwise reset
++ * them before any page had five samples)
*/
const express = require('express');
@@@ -121,7 +123,31 @@ function createRum({ now = () => Date.now() } = {}) {
return [...errors.entries()].sort((a, b) => b[1] - a[1]).slice(0, 10).map(([message, count]) => ({ labels: { message: message.slice(0, 100) }, value: count }));
}
-- return { router, record, snapshot, p75Gauge, errorsGauge };
++ /** The reservoirs and error counts as plain numbers, to survive a restart (server/index.js keeps them in app_state). */
++ function state() {
++ return { v: 1, at: now(), pages: Object.fromEntries(buckets), errors: [...errors] };
++ }
++ /** Put a saved state back under what arrived since boot (a beacon can land before the read). Older than 14 days,
++ * malformed or out-of-range parts are left out. */
++ function restore(s) {
++ if (!s || s.v !== 1 || typeof s.pages !== 'object' || !(now() - Number(s.at) < 14 * 86400000)) return false;
++ for (const [key, saved] of Object.entries(s.pages)) {
++ if (!/^[a-z]+\|(phone|tablet|desktop)$/.test(key) || !saved || typeof saved.samples !== 'object') continue;
++ const b = buckets.get(key) || { n: 0, samples: Object.fromEntries(Object.keys(METRICS).map((m) => [m, []])) };
++ b.n += Math.max(0, Number(saved.n) || 0);
++ for (const [m, max] of Object.entries(METRICS)) {
++ const old = (Array.isArray(saved.samples[m]) ? saved.samples[m] : []).map(Number).filter((v) => Number.isFinite(v) && v >= 0 && v <= max);
++ b.samples[m] = old.concat(b.samples[m]).slice(-KEEP);
++ }
++ buckets.set(key, b);
++ }
++ for (const [msg, c] of Array.isArray(s.errors) ? s.errors.slice(0, MAX_ERRORS) : []) {
++ if (typeof msg === 'string' && msg && errors.size < MAX_ERRORS) errors.set(msg.slice(0, 140), (errors.get(msg.slice(0, 140)) || 0) + (Number(c) || 0));
++ }
++ return true;
++ }
++
++ return { router, record, snapshot, p75Gauge, errorsGauge, state, restore };
}
module.exports = { createRum, pageKind, deviceClass };
ddiff --git a/test/rum.test.js b/test/rum.test.js
iindex 36156b18..d313c100 100644
---- a/test/rum.test.js
++++ b/test/rum.test.js
@@@ -76,6 +76,24 @@ const { createRum, pageKind, deviceClass } = require('../server/web/rum');
assert.ok(rum.snapshot().errors.length <= 25 && rum.snapshot().errors[0].message === 'TypeError: x is undefined');
server.close();
++ // Across a restart: the state is numbers only, and a new process puts it back under what arrived since its boot.
++ const saved = JSON.parse(JSON.stringify(rum.state()));
++ assert.ok(!JSON.stringify(saved).includes('10.0.0.'), 'no address in the saved state');
++ const next = createRum({ now: () => t });
++ next.record({ path: '/', w: 390, lcp: 1234 }); // a beacon before the read
++ assert.strictEqual(next.restore(saved), true);
++ const homeNext = next.snapshot().pages['home|phone'];
++ assert.strictEqual(homeNext.beacons, rum.snapshot().pages['home|phone'].beacons + 1);
++ assert.strictEqual(homeNext.lcp.n, 11, 'the saved samples, then the new one');
++ assert.deepStrictEqual(next.p75Gauge().find((x) => x.labels.metric === 'lcp' && x.labels.page === 'home').value, 8000);
++ assert.ok(next.snapshot().errors.some((e) => e.message === 'TypeError: x is undefined'));
++ assert.strictEqual(createRum({ now: () => t + 15 * 86400000 }).restore(saved), false, 'older than 14 days: start empty');
++ const junk = createRum(); junk.restore({ v: 1, at: Date.now(), pages: { 'evil|key': { n: 9, samples: {} }, 'home|phone': { n: 2, samples: { lcp: [100, 'a', -5, 1e9] } } } });
++ assert.deepStrictEqual([Object.keys(junk.snapshot().pages), junk.snapshot().pages['home|phone'].lcp.n], [['home|phone'], 1], 'malformed keys and values are left out');
++ const index = fs.readFileSync(path.join(__dirname, '../server/index.js'), 'utf8');
++ assert.ok(index.includes("app.get('/metrics/rum'") && index.includes('isLoopbackDirect(req)'), 'the host reads the snapshot on loopback only');
++ assert.ok(/if \(!drill\.enabled\) \{\s*db\.getState\('rum_state'\)/.test(index), 'restored at boot, never in a restore drill');
++
// app.js: every page load, once idle, through the runtime's versioned loader (a quarter of them gave too few
// samples at this traffic; automated browsers are skipped in the script itself).
const app_js = fs.readFileSync(path.join(__dirname, '../public/js/app.js'), 'utf8');