openvibers/openvibe.media Commits

Under VOD_DISK_STOP_GB free (3) the recorder stops the largest running recording, one per minute, so a long stream cannot fill the disk PostgreSQL shares (#162)

OP OpenVibers committed 21 days ago · 895a390048 · 3 files, +109 −2

Changes

ddiff --git a/.env.example b/.env.example
iindex 4680c58..c10021c 100644
---- a/.env.example
++++ b/.env.example
@@@ -107,6 +107,9 @@ MEDIA_CAPABILITY_GATE=report
# VOD_STOP_GRACE_MS=60000
# VOD_DISK_WARN_GB=15
# VOD_DISK_CRIT_GB=5
++# Under this much free space the largest running recording is stopped (one per minute), so a long stream cannot fill
++# the disk PostgreSQL and every service share. Under VOD_DISK_CRIT_GB only new recordings are refused.
++# VOD_DISK_STOP_GB=3
# MAX_VOD_SIZE_MB=2048
# VOD_NO_NORMALIZE=0
ddiff --git a/server/vod/recorder.js b/server/vod/recorder.js
iindex 68a1161..921e805 100644
---- a/server/vod/recorder.js
++++ b/server/vod/recorder.js
@@@ -67,6 +67,9 @@ function checkRtmpUrl(url, allow = config.rtmpPull.allow) {
// Disk guardian thresholds (free bytes on the VOD volume).
const DISK_WARN_BYTES = (parseFloat(process.env.VOD_DISK_WARN_GB) || 15) * 1024 * 1024 * 1024;
const DISK_CRIT_BYTES = (parseFloat(process.env.VOD_DISK_CRIT_GB) || 5) * 1024 * 1024 * 1024;
++// The last resort: under this, the largest running recording is stopped (checkDisk), because a full disk stops
++// PostgreSQL and every service on the host with it.
++const DISK_STOP_BYTES = (parseFloat(process.env.VOD_DISK_STOP_GB) || 3) * 1024 * 1024 * 1024;
function _isControlledFfmpegError(line, expectedShutdown) {
if (!line || !expectedShutdown) return false;
@@@ -586,11 +589,12 @@ class StreamRecorder {
/**
* Disk guardian: sample free space on the VOD volume. Under 'critical' no
-- * new recordings start (existing ones keep running). Called on an interval.
++ * new recordings start (existing ones keep running); under VOD_DISK_STOP_GB the
++ * largest running one is stopped too (_stopLargestForDisk). Called on an interval.
*/
checkDisk() {
let free = 0;
-- try { free = require('./vod-storage').diskUsage(config.vod.path).available || 0; } catch { return; }
++ try { free = this._freeBytes(); } catch { return; }
if (!free) return;
const prev = this._diskState;
const state = free < DISK_CRIT_BYTES ? 'critical' : free < DISK_WARN_BYTES ? 'warning' : 'ok';
@@@ -601,11 +605,43 @@ class StreamRecorder {
else console.warn(`[VOD] Disk ${state.toUpperCase()} — only ${gb}GB free on the VOD volume`);
}
if (state === 'ok') return;
++ if (free < DISK_STOP_BYTES) this._stopLargestForDisk(free);
// Under pressure: nudge the storage sweep to offload cold VODs.
try { const vs = require('./vod-storage'); if (typeof vs.runSweep === 'function') vs.runSweep().catch(() => {}); } catch { /* */ }
}
++ /** Free bytes on the VOD volume. */
++ _freeBytes() {
++ return require('./vod-storage').diskUsage(config.vod.path).available || 0;
++ }
++
++ /**
++ * The last resort under VOD_DISK_STOP_GB free (default 3): stop the largest running recording, one per check.
++ * Under 'critical' only new recordings are refused, and running ones keep writing: on 2026-10-23 two long
++ * streams and their DVR copies grew the disk about 5 GB an hour. Unchecked, that fills the disk, and PostgreSQL
++ * and every service on the host stop with it. The stopped recording finalizes as usual (its remux is skipped
++ * when there is no room for the copy) and the storage sweep can then move it. → the vod id stopped, or null.
++ */
++ _stopLargestForDisk(free) {
++ let pick = null;
++ for (const [vodId, rec] of this.activeRecordings) {
++ if (rec._expectedShutdown) continue;
++ let size = 0;
++ try { size = fs.statSync(rec.filePath).size; } catch { /* nothing written yet */ }
++ if (!pick || size > pick.size) pick = { vodId, size };
++ }
++ if (!pick) return null;
++ const gb = (n) => Number((n / 1024 / 1024 / 1024).toFixed(1));
++ console.error(`[VOD] Disk at ${gb(free)} GB free (under VOD_DISK_STOP_GB): stopping the largest recording, vod ${pick.vodId} (${gb(pick.size)} GB), so the disk does not fill`);
++ try {
++ const vs = require('./vod-storage');
++ if (typeof vs.emitStorageEvent === 'function') vs.emitStorageEvent('storage.alert', 'recording_stopped_disk', { vod_id: pick.vodId, free_gb: gb(free), size_gb: gb(pick.size) }).catch(() => {});
++ } catch { /* the stop matters more than the alert */ }
++ this.stopRecording(pick.vodId);
++ return pick.vodId;
++ }
++
/** Stop all active recordings (for graceful shutdown). */
stopAll() {
for (const [vodId] of this.activeRecordings) {
ddiff --git a/test/recorder-disk-floor.test.js b/test/recorder-disk-floor.test.js
nnew file mode 100644
iindex 0000000..2a81058
---- /dev/null
++++ b/test/recorder-disk-floor.test.js
@@@ -0,0 +1,68 @@
++'use strict';
++// The recorder's last resort (recorder._stopLargestForDisk via checkDisk): under VOD_DISK_STOP_GB free (default 3),
++// the largest running recording is stopped, one per check, because running recordings keep writing under 'critical'
++// and a full disk stops PostgreSQL and every service. Above the floor (critical or not) nothing running is stopped.
++// ffmpeg is a stand-in process, as in recorder-lifecycle.test.js.
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const { EventEmitter } = require('events');
++
++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-media-floor-'));
++const dir = (n) => { const d = path.join(tmp, n); fs.mkdirSync(d, { recursive: true }); return d; };
++Object.assign(process.env, {
++ VOD_PATH: dir('vods'), CLIPS_PATH: dir('clips'), FILES_PATH: dir('files'),
++ THUMBNAILS_PATH: dir('thumbnails'), PASTES_PATH: dir('pastes'), OBJECTS_PATH: dir('objects'),
++});
++delete process.env.MEDIA_RTMP_PULL_ALLOW;
++delete process.env.VOD_DISK_STOP_GB;
++
++const cp = require('child_process');
++cp.spawn = () => {
++ const p = new EventEmitter();
++ p.stderr = new EventEmitter(); p.stdout = new EventEmitter(); p.killed = false;
++ p.kill = () => { p.killed = true; setImmediate(() => p.emit('exit', 0, null)); return true; };
++ return p;
++};
++
++const GiB = 1024 * 1024 * 1024;
++
++(async () => {
++ console.error = () => {}; console.warn = () => {};
++ const recorder = require('../server/vod/recorder');
++ const db = require('../server/db/database');
++ await db.upsertApp({ app_id: 'live', api_key: 'live-key-floor-test' });
++ const newVod = async () => db.getVodById(Number((await db.createVod({ app_id: 'live', title: 'x' })).lastInsertRowid));
++ const settled = async (n) => { for (let i = 0; i < 100 && recorder.activeCount() > n; i++) await new Promise((res) => setTimeout(res, 10)); };
++
++ const small = await newVod();
++ const big = await newVod();
++ const rs = await recorder.startRtmp(small, 'rtmp://127.0.0.1:1935/live/a');
++ const rb = await recorder.startRtmp(big, 'rtmp://127.0.0.1:1935/live/b');
++ fs.writeFileSync(rs.filePath, Buffer.alloc(1000));
++ fs.writeFileSync(rb.filePath, Buffer.alloc(5000));
++ assert.strictEqual(recorder.activeCount(), 2);
++
++ let free = 4 * GiB; // critical (under 5), above the 3 GB floor
++ recorder._freeBytes = () => free;
++ recorder.checkDisk();
++ await settled(2);
++ assert.strictEqual(recorder.activeCount(), 2, 'critical alone stops nothing that is running');
++
++ free = 2 * GiB;
++ recorder.checkDisk();
++ await settled(1);
++ assert.strictEqual(recorder.activeCount(), 1, 'under the floor: one recording stopped');
++ assert.ok(!recorder.activeRecordings.has(big.id) && recorder.activeRecordings.has(small.id), 'the largest went, the smaller keeps recording');
++
++ recorder.checkDisk(); // still under the floor on the next check: the next largest
++ await settled(0);
++ assert.strictEqual(recorder.activeCount(), 0);
++ assert.strictEqual(recorder._stopLargestForDisk(free), null, 'nothing left to stop');
++
++ await db.close();
++ fs.rmSync(tmp, { recursive: true, force: true });
++ console.log('recorder-disk-floor: ok');
++ process.exit(0);
++})().catch((err) => { process.stderr.write(`${err.stack}\n`); process.exit(1); });