openvibers/openvibe.media Commits
Under VOD_DISK_STOP_GB free (3) the recorder stops the largest running recording, one per minute, so a long stream cannot fill the disk PostgreSQL shares (#162)
Changes
@@@ -107,6 +107,9 @@ MEDIA_CAPABILITY_GATE=report# VOD_STOP_GRACE_MS=60000# VOD_DISK_WARN_GB=15# VOD_DISK_CRIT_GB=5++# Under this much free space the largest running recording is stopped (one per minute), so a long stream cannot fill++# the disk PostgreSQL and every service share. Under VOD_DISK_CRIT_GB only new recordings are refused.++# VOD_DISK_STOP_GB=3# MAX_VOD_SIZE_MB=2048# VOD_NO_NORMALIZE=0@@@ -67,6 +67,9 @@ function checkRtmpUrl(url, allow = config.rtmpPull.allow) {// Disk guardian thresholds (free bytes on the VOD volume).const DISK_WARN_BYTES = (parseFloat(process.env.VOD_DISK_WARN_GB) || 15) * 1024 * 1024 * 1024;const DISK_CRIT_BYTES = (parseFloat(process.env.VOD_DISK_CRIT_GB) || 5) * 1024 * 1024 * 1024;++// The last resort: under this, the largest running recording is stopped (checkDisk), because a full disk stops++// PostgreSQL and every service on the host with it.++const DISK_STOP_BYTES = (parseFloat(process.env.VOD_DISK_STOP_GB) || 3) * 1024 * 1024 * 1024;function _isControlledFfmpegError(line, expectedShutdown) {if (!line || !expectedShutdown) return false;@@@ -586,11 +589,12 @@ class StreamRecorder {/*** Disk guardian: sample free space on the VOD volume. Under 'critical' no-- * new recordings start (existing ones keep running). Called on an interval.++ * new recordings start (existing ones keep running); under VOD_DISK_STOP_GB the++ * largest running one is stopped too (_stopLargestForDisk). Called on an interval.*/checkDisk() {let free = 0;-- try { free = require('./vod-storage').diskUsage(config.vod.path).available || 0; } catch { return; }++ try { free = this._freeBytes(); } catch { return; }if (!free) return;const prev = this._diskState;const state = free < DISK_CRIT_BYTES ? 'critical' : free < DISK_WARN_BYTES ? 'warning' : 'ok';@@@ -601,11 +605,43 @@ class StreamRecorder {else console.warn(`[VOD] Disk ${state.toUpperCase()} — only ${gb}GB free on the VOD volume`);}if (state === 'ok') return;++ if (free < DISK_STOP_BYTES) this._stopLargestForDisk(free);// Under pressure: nudge the storage sweep to offload cold VODs.try { const vs = require('./vod-storage'); if (typeof vs.runSweep === 'function') vs.runSweep().catch(() => {}); } catch { /* */ }}++ /** Free bytes on the VOD volume. */++ _freeBytes() {++ return require('./vod-storage').diskUsage(config.vod.path).available || 0;++ }++++ /**++ * The last resort under VOD_DISK_STOP_GB free (default 3): stop the largest running recording, one per check.++ * Under 'critical' only new recordings are refused, and running ones keep writing: on 2026-10-23 two long++ * streams and their DVR copies grew the disk about 5 GB an hour. Unchecked, that fills the disk, and PostgreSQL++ * and every service on the host stop with it. The stopped recording finalizes as usual (its remux is skipped++ * when there is no room for the copy) and the storage sweep can then move it. → the vod id stopped, or null.++ */++ _stopLargestForDisk(free) {++ let pick = null;++ for (const [vodId, rec] of this.activeRecordings) {++ if (rec._expectedShutdown) continue;++ let size = 0;++ try { size = fs.statSync(rec.filePath).size; } catch { /* nothing written yet */ }++ if (!pick || size > pick.size) pick = { vodId, size };++ }++ if (!pick) return null;++ const gb = (n) => Number((n / 1024 / 1024 / 1024).toFixed(1));++ console.error(`[VOD] Disk at ${gb(free)} GB free (under VOD_DISK_STOP_GB): stopping the largest recording, vod ${pick.vodId} (${gb(pick.size)} GB), so the disk does not fill`);++ try {++ const vs = require('./vod-storage');++ if (typeof vs.emitStorageEvent === 'function') vs.emitStorageEvent('storage.alert', 'recording_stopped_disk', { vod_id: pick.vodId, free_gb: gb(free), size_gb: gb(pick.size) }).catch(() => {});++ } catch { /* the stop matters more than the alert */ }++ this.stopRecording(pick.vodId);++ return pick.vodId;++ }++/** Stop all active recordings (for graceful shutdown). */stopAll() {for (const [vodId] of this.activeRecordings) {nnew file mode 100644@@@ -0,0 +1,68 @@++'use strict';++// The recorder's last resort (recorder._stopLargestForDisk via checkDisk): under VOD_DISK_STOP_GB free (default 3),++// the largest running recording is stopped, one per check, because running recordings keep writing under 'critical'++// and a full disk stops PostgreSQL and every service. Above the floor (critical or not) nothing running is stopped.++// ffmpeg is a stand-in process, as in recorder-lifecycle.test.js.++const assert = require('assert');++const fs = require('fs');++const os = require('os');++const path = require('path');++const { EventEmitter } = require('events');++++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-media-floor-'));++const dir = (n) => { const d = path.join(tmp, n); fs.mkdirSync(d, { recursive: true }); return d; };++Object.assign(process.env, {++ VOD_PATH: dir('vods'), CLIPS_PATH: dir('clips'), FILES_PATH: dir('files'),++ THUMBNAILS_PATH: dir('thumbnails'), PASTES_PATH: dir('pastes'), OBJECTS_PATH: dir('objects'),++});++delete process.env.MEDIA_RTMP_PULL_ALLOW;++delete process.env.VOD_DISK_STOP_GB;++++const cp = require('child_process');++cp.spawn = () => {++ const p = new EventEmitter();++ p.stderr = new EventEmitter(); p.stdout = new EventEmitter(); p.killed = false;++ p.kill = () => { p.killed = true; setImmediate(() => p.emit('exit', 0, null)); return true; };++ return p;++};++++const GiB = 1024 * 1024 * 1024;++++(async () => {++ console.error = () => {}; console.warn = () => {};++ const recorder = require('../server/vod/recorder');++ const db = require('../server/db/database');++ await db.upsertApp({ app_id: 'live', api_key: 'live-key-floor-test' });++ const newVod = async () => db.getVodById(Number((await db.createVod({ app_id: 'live', title: 'x' })).lastInsertRowid));++ const settled = async (n) => { for (let i = 0; i < 100 && recorder.activeCount() > n; i++) await new Promise((res) => setTimeout(res, 10)); };++++ const small = await newVod();++ const big = await newVod();++ const rs = await recorder.startRtmp(small, 'rtmp://127.0.0.1:1935/live/a');++ const rb = await recorder.startRtmp(big, 'rtmp://127.0.0.1:1935/live/b');++ fs.writeFileSync(rs.filePath, Buffer.alloc(1000));++ fs.writeFileSync(rb.filePath, Buffer.alloc(5000));++ assert.strictEqual(recorder.activeCount(), 2);++++ let free = 4 * GiB; // critical (under 5), above the 3 GB floor++ recorder._freeBytes = () => free;++ recorder.checkDisk();++ await settled(2);++ assert.strictEqual(recorder.activeCount(), 2, 'critical alone stops nothing that is running');++++ free = 2 * GiB;++ recorder.checkDisk();++ await settled(1);++ assert.strictEqual(recorder.activeCount(), 1, 'under the floor: one recording stopped');++ assert.ok(!recorder.activeRecordings.has(big.id) && recorder.activeRecordings.has(small.id), 'the largest went, the smaller keeps recording');++++ recorder.checkDisk(); // still under the floor on the next check: the next largest++ await settled(0);++ assert.strictEqual(recorder.activeCount(), 0);++ assert.strictEqual(recorder._stopLargestForDisk(free), null, 'nothing left to stop');++++ await db.close();++ fs.rmSync(tmp, { recursive: true, force: true });++ console.log('recorder-disk-floor: ok');++ process.exit(0);++})().catch((err) => { process.stderr.write(`${err.stack}\n`); process.exit(1); });
@@ -107,6 +107,9 @@ MEDIA_CAPABILITY_GATE=report
@@ -107,6 +107,9 @@ MEDIA_CAPABILITY_GATE=report
# VOD_STOP_GRACE_MS=60000
# VOD_STOP_GRACE_MS=60000
# VOD_DISK_WARN_GB=15
# VOD_DISK_WARN_GB=15
# VOD_DISK_CRIT_GB=5
# VOD_DISK_CRIT_GB=5
+# Under this much free space the largest running recording is stopped (one per minute), so a long stream cannot fill
+# the disk PostgreSQL and every service share. Under VOD_DISK_CRIT_GB only new recordings are refused.
+# VOD_DISK_STOP_GB=3
# MAX_VOD_SIZE_MB=2048
# MAX_VOD_SIZE_MB=2048
# VOD_NO_NORMALIZE=0
# VOD_NO_NORMALIZE=0
@@ -67,6 +67,9 @@ function checkRtmpUrl(url, allow = config.rtmpPull.allow) {
@@ -67,6 +67,9 @@ function checkRtmpUrl(url, allow = config.rtmpPull.allow) {
// Disk guardian thresholds (free bytes on the VOD volume).
// Disk guardian thresholds (free bytes on the VOD volume).
const DISK_WARN_BYTES = (parseFloat(process.env.VOD_DISK_WARN_GB) || 15) * 1024 * 1024 * 1024;
const DISK_WARN_BYTES = (parseFloat(process.env.VOD_DISK_WARN_GB) || 15) * 1024 * 1024 * 1024;
const DISK_CRIT_BYTES = (parseFloat(process.env.VOD_DISK_CRIT_GB) || 5) * 1024 * 1024 * 1024;
const DISK_CRIT_BYTES = (parseFloat(process.env.VOD_DISK_CRIT_GB) || 5) * 1024 * 1024 * 1024;
+// The last resort: under this, the largest running recording is stopped (checkDisk), because a full disk stops
+// PostgreSQL and every service on the host with it.
+const DISK_STOP_BYTES = (parseFloat(process.env.VOD_DISK_STOP_GB) || 3) * 1024 * 1024 * 1024;
function _isControlledFfmpegError(line, expectedShutdown) {
function _isControlledFfmpegError(line, expectedShutdown) {
if (!line || !expectedShutdown) return false;
if (!line || !expectedShutdown) return false;
@@ -586,11 +589,12 @@ class StreamRecorder {
@@ -586,11 +589,12 @@ class StreamRecorder {
/**
/**
* Disk guardian: sample free space on the VOD volume. Under 'critical' no
* Disk guardian: sample free space on the VOD volume. Under 'critical' no
- * new recordings start (existing ones keep running). Called on an interval.
+ * new recordings start (existing ones keep running); under VOD_DISK_STOP_GB the
+ * largest running one is stopped too (_stopLargestForDisk). Called on an interval.
*/
*/
checkDisk() {
checkDisk() {
let free = 0;
let free = 0;
- try { free = require('./vod-storage').diskUsage(config.vod.path).available || 0; } catch { return; }
+ try { free = this._freeBytes(); } catch { return; }
if (!free) return;
if (!free) return;
const prev = this._diskState;
const prev = this._diskState;
const state = free < DISK_CRIT_BYTES ? 'critical' : free < DISK_WARN_BYTES ? 'warning' : 'ok';
const state = free < DISK_CRIT_BYTES ? 'critical' : free < DISK_WARN_BYTES ? 'warning' : 'ok';
@@ -601,11 +605,43 @@ class StreamRecorder {
@@ -601,11 +605,43 @@ class StreamRecorder {
else console.warn(`[VOD] Disk ${state.toUpperCase()} — only ${gb}GB free on the VOD volume`);
else console.warn(`[VOD] Disk ${state.toUpperCase()} — only ${gb}GB free on the VOD volume`);
}
}
if (state === 'ok') return;
if (state === 'ok') return;
+ if (free < DISK_STOP_BYTES) this._stopLargestForDisk(free);
// Under pressure: nudge the storage sweep to offload cold VODs.
// Under pressure: nudge the storage sweep to offload cold VODs.
try { const vs = require('./vod-storage'); if (typeof vs.runSweep === 'function') vs.runSweep().catch(() => {}); } catch { /* */ }
try { const vs = require('./vod-storage'); if (typeof vs.runSweep === 'function') vs.runSweep().catch(() => {}); } catch { /* */ }
}
}
+ /** Free bytes on the VOD volume. */
+ _freeBytes() {
+ return require('./vod-storage').diskUsage(config.vod.path).available || 0;
+ }
+
+ /**
+ * The last resort under VOD_DISK_STOP_GB free (default 3): stop the largest running recording, one per check.
+ * Under 'critical' only new recordings are refused, and running ones keep writing: on 2026-10-23 two long
+ * streams and their DVR copies grew the disk about 5 GB an hour. Unchecked, that fills the disk, and PostgreSQL
+ * and every service on the host stop with it. The stopped recording finalizes as usual (its remux is skipped
+ * when there is no room for the copy) and the storage sweep can then move it. → the vod id stopped, or null.
+ */
+ _stopLargestForDisk(free) {
+ let pick = null;
+ for (const [vodId, rec] of this.activeRecordings) {
+ if (rec._expectedShutdown) continue;
+ let size = 0;
+ try { size = fs.statSync(rec.filePath).size; } catch { /* nothing written yet */ }
+ if (!pick || size > pick.size) pick = { vodId, size };
+ }
+ if (!pick) return null;
+ const gb = (n) => Number((n / 1024 / 1024 / 1024).toFixed(1));
+ console.error(`[VOD] Disk at ${gb(free)} GB free (under VOD_DISK_STOP_GB): stopping the largest recording, vod ${pick.vodId} (${gb(pick.size)} GB), so the disk does not fill`);
+ try {
+ const vs = require('./vod-storage');
+ if (typeof vs.emitStorageEvent === 'function') vs.emitStorageEvent('storage.alert', 'recording_stopped_disk', { vod_id: pick.vodId, free_gb: gb(free), size_gb: gb(pick.size) }).catch(() => {});
+ } catch { /* the stop matters more than the alert */ }
+ this.stopRecording(pick.vodId);
+ return pick.vodId;
+ }
+
/** Stop all active recordings (for graceful shutdown). */
/** Stop all active recordings (for graceful shutdown). */
stopAll() {
stopAll() {
for (const [vodId] of this.activeRecordings) {
for (const [vodId] of this.activeRecordings) {
new file mode 100644
new file mode 100644
@@ -0,0 +1,68 @@
@@ -0,0 +1,68 @@
+'use strict';
+// The recorder's last resort (recorder._stopLargestForDisk via checkDisk): under VOD_DISK_STOP_GB free (default 3),
+// the largest running recording is stopped, one per check, because running recordings keep writing under 'critical'
+// and a full disk stops PostgreSQL and every service. Above the floor (critical or not) nothing running is stopped.
+// ffmpeg is a stand-in process, as in recorder-lifecycle.test.js.
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { EventEmitter } = require('events');
+
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-media-floor-'));
+const dir = (n) => { const d = path.join(tmp, n); fs.mkdirSync(d, { recursive: true }); return d; };
+Object.assign(process.env, {
+ VOD_PATH: dir('vods'), CLIPS_PATH: dir('clips'), FILES_PATH: dir('files'),
+ THUMBNAILS_PATH: dir('thumbnails'), PASTES_PATH: dir('pastes'), OBJECTS_PATH: dir('objects'),
+});
+delete process.env.MEDIA_RTMP_PULL_ALLOW;
+delete process.env.VOD_DISK_STOP_GB;
+
+const cp = require('child_process');
+cp.spawn = () => {
+ const p = new EventEmitter();
+ p.stderr = new EventEmitter(); p.stdout = new EventEmitter(); p.killed = false;
+ p.kill = () => { p.killed = true; setImmediate(() => p.emit('exit', 0, null)); return true; };
+ return p;
+};
+
+const GiB = 1024 * 1024 * 1024;
+
+(async () => {
+ console.error = () => {}; console.warn = () => {};
+ const recorder = require('../server/vod/recorder');
+ const db = require('../server/db/database');
+ await db.upsertApp({ app_id: 'live', api_key: 'live-key-floor-test' });
+ const newVod = async () => db.getVodById(Number((await db.createVod({ app_id: 'live', title: 'x' })).lastInsertRowid));
+ const settled = async (n) => { for (let i = 0; i < 100 && recorder.activeCount() > n; i++) await new Promise((res) => setTimeout(res, 10)); };
+
+ const small = await newVod();
+ const big = await newVod();
+ const rs = await recorder.startRtmp(small, 'rtmp://127.0.0.1:1935/live/a');
+ const rb = await recorder.startRtmp(big, 'rtmp://127.0.0.1:1935/live/b');
+ fs.writeFileSync(rs.filePath, Buffer.alloc(1000));
+ fs.writeFileSync(rb.filePath, Buffer.alloc(5000));
+ assert.strictEqual(recorder.activeCount(), 2);
+
+ let free = 4 * GiB; // critical (under 5), above the 3 GB floor
+ recorder._freeBytes = () => free;
+ recorder.checkDisk();
+ await settled(2);
+ assert.strictEqual(recorder.activeCount(), 2, 'critical alone stops nothing that is running');
+
+ free = 2 * GiB;
+ recorder.checkDisk();
+ await settled(1);
+ assert.strictEqual(recorder.activeCount(), 1, 'under the floor: one recording stopped');
+ assert.ok(!recorder.activeRecordings.has(big.id) && recorder.activeRecordings.has(small.id), 'the largest went, the smaller keeps recording');
+
+ recorder.checkDisk(); // still under the floor on the next check: the next largest
+ await settled(0);
+ assert.strictEqual(recorder.activeCount(), 0);
+ assert.strictEqual(recorder._stopLargestForDisk(free), null, 'nothing left to stop');
+
+ await db.close();
+ fs.rmSync(tmp, { recursive: true, force: true });
+ console.log('recorder-disk-floor: ok');
+ process.exit(0);
+})().catch((err) => { process.stderr.write(`${err.stack}\n`); process.exit(1); });