openvibers/openvibe.media Commits
The disk guard raises drain_stalled only when eligible VODs could not be moved or free space is under the recorder's VOD_DISK_WARN_GB, and logs that nothing is eligible once per change instead of every pass (#155)
Changes
@@@ -691,7 +691,7 @@ shutdown.`server/vod/vod-storage.js` keeps the local VOD volume healthy without anyone watching it:--- **Drain policy** — a drain starts when the disk is above `hotDiskPressurePct` (70%) **or** has less than `minFreeGb` (25 GB) free, and stops only when it is under `localLowWaterPct` (60%) **and** has `targetFreeGb` (40 GB) free. Above `criticalDiskPct` (90%) recordings finished more than `criticalMinAgeHours` (2 h) ago are eligible; otherwise 1 day. Least-recently-watched VODs go first. All values live in `media_settings` as `storage_tier.*` (`PUT /api/v1/:app/admin/storage/tiers/settings`).++- **Drain policy** — a drain starts when the disk is above `hotDiskPressurePct` (70%) **or** has less than `minFreeGb` (25 GB) free, and stops only when it is under `localLowWaterPct` (60%) **and** has `targetFreeGb` (40 GB) free. Above `criticalDiskPct` (90%) recordings finished more than `criticalMinAgeHours` (2 h) ago are eligible; otherwise 1 day. Least-recently-watched VODs go first. A pass that frees nothing raises `storage.alert drain_stalled` only when eligible VODs could not be moved, or when free space is under the recorder's `VOD_DISK_WARN_GB` (`recorderWarnGb`, 15 GB); a disk shared with other services that stays above the targets with nothing old enough to move is logged once, when that starts. All values live in `media_settings` as `storage_tier.*` (`PUT /api/v1/:app/admin/storage/tiers/settings`).- **Only real files are candidates.** Rows without a file (a recording that was refused or never produced output, legacy paths) are quarantined as `health_status='missing_file'` and excluded — they used to fill the candidate list and starve the drain.- **Cadence** — while a drain is still needed the sweep re-runs every `pressureRetryMs` (2 min), otherwise every `sweepIntervalMs` (15 min). A VOD whose upload failed is skipped for 30 min so one bad file cannot pin the drain.- **Bounded uploads** — each upload is aborted past `uploadTimeoutFloorMs + size / uploadMinThroughputMBps`; a watchdog logs a sweep that has run for hours.@@@ -92,6 +92,13 @@ const DEFAULTS = {// free space is below minFreeGb and keep going until targetFreeGb is free again.minFreeGb: 25,targetFreeGb: 40,++ // The recorder's real guard rails (server/vod/recorder.js): it warns below VOD_DISK_WARN_GB free and++ // refuses new recordings below VOD_DISK_CRIT_GB. The disk is shared with ~50 services, so the++ // percentage/GB targets above can stay unmet for days with nothing Media could move; a drain that++ // freed nothing is alert-worthy only when there was something eligible to move, or when free space++ // is under these.++ recorderWarnGb: parseFloat(process.env.VOD_DISK_WARN_GB) || 15,++ recorderCritGb: parseFloat(process.env.VOD_DISK_CRIT_GB) || 5,// While a drain is still needed after a pass, re-run this soon instead of waiting// for the regular interval — recovery from a full disk should take minutes.pressureRetryMs: 2 * 60 * 1000,@@@ -938,6 +945,20 @@ function drainSatisfied(disk, settings) {return disk.usePct <= settings.localLowWaterPct && freeGb >= settings.targetFreeGb;}++/**++ * What a pressure pass means for alerting. `nothingEligible`: under pressure with no candidate at all++ * (nothing old enough on local disk; not Media's to fix). `stalled`: a pressure pass freed nothing++ * although candidates existed (uploads failing, back-off, holds) or free space is under the++ * recorder's warn guard rail. Only `stalled` counts toward drain_stalled.++ */++function assessDrain({ underPressure, candidates, migrated, disk }, settings) {++ const freeGb = disk && disk.total ? disk.available / GB : Infinity;++ const belowRecorderGuard = freeGb < settings.recorderWarnGb;++ const nothingEligible = !!underPressure && !candidates;++ const stalled = !!underPressure && !migrated && (candidates > 0 || belowRecorderGuard);++ return { nothingEligible, stalled, belowRecorderGuard };++}++function isCritical(disk, settings) {return !!disk && !!disk.total && disk.usePct >= settings.criticalDiskPct;}@@@ -960,8 +981,9 @@ const sweepState = {lastResult: null,nextRunAt: null,startedAt: null,-- stalledPasses: 0, // consecutive pressure passes that freed nothing++ stalledPasses: 0, // consecutive pressure passes that freed nothing although they could have (assessDrain)stalled: false,++ nothingEligible: false, // last pass was under pressure with nothing old enough to move (logged once per change)};const alertLastSentAt = new Map(); // kind → epoch ms@@@ -1012,8 +1034,6 @@ async function runSweep() {let candidates;if (underPressure) {const minAge = critical ? `-${settings.criticalMinAgeHours} hours` : '-1 day';-- console.log(`[VodStorage] Disk at ${disk.usePct}%, ${(disk.available / GB).toFixed(1)} GB free (${critical ? 'CRITICAL' : 'pressure'}) — draining to ≤${settings.localLowWaterPct}% / ≥${settings.targetFreeGb} GB free`-- + (ghosts ? `; quarantined ${ghosts} file-less VOD row(s)` : ''));candidates = await db.all(`SELECT id, file_path, file_size FROM vodsWHERE ${OFFLOADABLE_WHERE}@@@ -1021,6 +1041,11 @@ async function runSweep() {ORDER BY (last_accessed_at IS NOT NULL), last_accessed_at ASC, view_count ASC, file_size DESCLIMIT 200`, [minAge]);++ // With nothing old enough to move the pass can do nothing; say so once per change, not every pass.++ if (candidates.length || ghosts || !sweepState.nothingEligible) {++ console.log(`[VodStorage] Disk at ${disk.usePct}%, ${(disk.available / GB).toFixed(1)} GB free (${critical ? 'CRITICAL' : 'pressure'}) — draining to ≤${settings.localLowWaterPct}% / ≥${settings.targetFreeGb} GB free`++ + (ghosts ? `; quarantined ${ghosts} file-less VOD row(s)` : ''));++ }} else {if (ghosts) console.log(`[VodStorage] Quarantined ${ghosts} file-less VOD row(s)`);candidates = await db.all(`@@@ -1065,24 +1090,31 @@ async function runSweep() {// Say what happened whenever it matters: the old sweep only logged on success, so// weeks of "draining" that freed nothing looked healthy in the journal.-- if (underPressure || errors.length || quarantined || skippedHeld) {++ const diskAfter = diskUsage(config.vod.path);++ const verdict = assessDrain({ underPressure, candidates: candidates.length, migrated, disk: diskAfter }, settings);++ const nothingEligibleNews = verdict.nothingEligible && !sweepState.nothingEligible;++ if ((underPressure && (candidates.length || nothingEligibleNews)) || errors.length || quarantined || skippedHeld) {const freedMb = (bytesFreed / 1048576).toFixed(1);console.log(`[VodStorage] Offload pass: ${candidates.length} candidate(s), ${migrated} uploaded (${freedMb} MB freed), `+ `${errors.length} failed, ${skippedBackoff} in retry back-off, ${quarantined} quarantined`+ (skippedHeld ? `, ${skippedHeld} under a retention hold` : '')-- + (underPressure ? `, disk now ${diskUsage(config.vod.path).usePct}%` : ''));++ + (underPressure ? `, disk now ${diskAfter.usePct}%` : ''));for (const e of errors.slice(0, 5)) console.warn(`[VodStorage] VOD ${e.id}: ${e.error}`);-- if (underPressure && !migrated && !candidates.length) {-- console.warn('[VodStorage] Disk is under pressure but nothing is eligible to offload — check for stuck recordings or non-VOD files in the VOD directory');++ if (nothingEligibleNews) {++ console.warn(`[VodStorage] Disk is under pressure but nothing is eligible to offload (recordings under ${critical ? `${settings.criticalMinAgeHours} hours` : 'a day'} old stay local); `++ + `not logged again until that changes. Alerts only below ${settings.recorderWarnGb} GB free (VOD_DISK_WARN_GB).`);}}++ if (sweepState.nothingEligible && !verdict.nothingEligible && underPressure && candidates.length) {++ console.log(`[VodStorage] ${candidates.length} VOD(s) now eligible to offload`);++ }++ sweepState.nothingEligible = verdict.nothingEligible;-- // Stall detection: a pressure pass that freed nothing is the failure mode that-- // went unnoticed for weeks. Count them, and after a couple in a row raise an-- // alert; announce recovery once the drain works again.-- const diskAfter = diskUsage(config.vod.path);++ // Stall detection: a pressure pass that freed nothing although it could have is the++ // failure mode that went unnoticed for weeks. Count them, and after a couple in a row++ // raise an alert; announce recovery once the drain works again (or has nothing to do).const stillNeedsDrain = underPressure && !drainSatisfied(diskAfter, settings);-- if (underPressure && !migrated) {++ if (verdict.stalled) {sweepState.stalledPasses++;if (sweepState.stalledPasses >= settings.alertAfterStalledPasses) {sweepState.stalled = true;@@@ -1092,8 +1124,9 @@ async function runSweep() {stalled_passes: sweepState.stalledPasses,candidates: candidates.length,errors: errors.slice(0, 3),++ below_recorder_warn_gb: verdict.belowRecorderGuard,hint: !candidates.length-- ? 'nothing eligible to offload (stuck recordings? non-VOD files in the VOD directory?)'++ ? `under ${settings.recorderWarnGb} GB free (VOD_DISK_WARN_GB) and nothing eligible to offload (stuck recordings? other services filling the disk?)`: errors.length ? 'uploads failing — check B2 credentials/bucket and network' : 'all candidates in retry back-off',}, settings);}@@@ -1492,6 +1525,7 @@ module.exports = {needsDrain,drainSatisfied,isCritical,++ assessDrain,planNextDelayMs,uploadTimeoutMs,providerOf,@@@ -16,7 +16,7 @@ const assert = require('assert');// The revisioned tier policies load once the database is open (server/index.js does this at boot).await storage.tierConfig.init(storage.DEFAULTS);await require('../server/objects/tier-policy').init();-- const { DEFAULTS, OFFLOADABLE_WHERE, needsDrain, drainSatisfied, isCritical, planNextDelayMs, uploadTimeoutMs } = storage;++ const { DEFAULTS, OFFLOADABLE_WHERE, needsDrain, drainSatisfied, isCritical, assessDrain, planNextDelayMs, uploadTimeoutMs } = storage;const GB = 1024 * 1024 * 1024;const disk = (usePct, freeGb, totalGb = 96) => ({ total: totalGb * GB, used: (totalGb - freeGb) * GB, available: freeGb * GB, usePct, mount: '/' });@@@ -57,6 +57,21 @@ const assert = require('assert');assert.strictEqual(isCritical(disk(89, 10), DEFAULTS), false);console.log('✅ free-space budget: percent ceiling OR GB floor start a drain; both targets end it');++ // ── Alerting measures what Media can act on: a shared 96 GB disk sits above the targets for days ──++ const shared = disk(70, 28); // under pressure (≥70%), well above the recorder's 15 GB warn line++ assert.deepStrictEqual(assessDrain({ underPressure: true, candidates: 0, migrated: 0, disk: shared }, DEFAULTS),++ { nothingEligible: true, stalled: false, belowRecorderGuard: false }, 'nothing old enough to move is not a stall');++ assert.strictEqual(assessDrain({ underPressure: true, candidates: 3, migrated: 0, disk: shared }, DEFAULTS).stalled, true,++ 'eligible candidates that could not be moved are a stall');++ assert.strictEqual(assessDrain({ underPressure: true, candidates: 3, migrated: 1, disk: shared }, DEFAULTS).stalled, false, 'progress is no stall');++ const tight = disk(88, 12); // under VOD_DISK_WARN_GB (15 GB) free++ assert.deepStrictEqual(assessDrain({ underPressure: true, candidates: 0, migrated: 0, disk: tight }, DEFAULTS),++ { nothingEligible: true, stalled: true, belowRecorderGuard: true }, 'under the recorder guard rail it alerts even with nothing eligible');++ assert.strictEqual(assessDrain({ underPressure: false, candidates: 0, migrated: 0, disk: disk(50, 48) }, DEFAULTS).stalled, false);++ assert.strictEqual(DEFAULTS.recorderWarnGb, parseFloat(process.env.VOD_DISK_WARN_GB) || 15);++ assert.strictEqual(DEFAULTS.recorderCritGb, parseFloat(process.env.VOD_DISK_CRIT_GB) || 5);++ console.log('✅ drain_stalled only when eligible VODs could not move, or free space is under the recorder guard rail');++// ── Cadence: come back quickly while a drain is making (or could make) progress ──assert.strictEqual(planNextDelayMs({ stillNeedsDrain: true, migrated: 3 }, DEFAULTS), DEFAULTS.pressureRetryMs, 'progress under pressure → soon');assert.strictEqual(planNextDelayMs({ stillNeedsDrain: true, migrated: 0, skippedBackoff: 2 }, DEFAULTS), DEFAULTS.pressureRetryMs, 'candidates in back-off → soon');
@@ -691,7 +691,7 @@ shutdown.
@@ -691,7 +691,7 @@ shutdown.
`server/vod/vod-storage.js` keeps the local VOD volume healthy without anyone watching it:
`server/vod/vod-storage.js` keeps the local VOD volume healthy without anyone watching it:
-- **Drain policy** — a drain starts when the disk is above `hotDiskPressurePct` (70%) **or** has less than `minFreeGb` (25 GB) free, and stops only when it is under `localLowWaterPct` (60%) **and** has `targetFreeGb` (40 GB) free. Above `criticalDiskPct` (90%) recordings finished more than `criticalMinAgeHours` (2 h) ago are eligible; otherwise 1 day. Least-recently-watched VODs go first. All values live in `media_settings` as `storage_tier.*` (`PUT /api/v1/:app/admin/storage/tiers/settings`).
+- **Drain policy** — a drain starts when the disk is above `hotDiskPressurePct` (70%) **or** has less than `minFreeGb` (25 GB) free, and stops only when it is under `localLowWaterPct` (60%) **and** has `targetFreeGb` (40 GB) free. Above `criticalDiskPct` (90%) recordings finished more than `criticalMinAgeHours` (2 h) ago are eligible; otherwise 1 day. Least-recently-watched VODs go first. A pass that frees nothing raises `storage.alert drain_stalled` only when eligible VODs could not be moved, or when free space is under the recorder's `VOD_DISK_WARN_GB` (`recorderWarnGb`, 15 GB); a disk shared with other services that stays above the targets with nothing old enough to move is logged once, when that starts. All values live in `media_settings` as `storage_tier.*` (`PUT /api/v1/:app/admin/storage/tiers/settings`).
- **Only real files are candidates.** Rows without a file (a recording that was refused or never produced output, legacy paths) are quarantined as `health_status='missing_file'` and excluded — they used to fill the candidate list and starve the drain.
- **Only real files are candidates.** Rows without a file (a recording that was refused or never produced output, legacy paths) are quarantined as `health_status='missing_file'` and excluded — they used to fill the candidate list and starve the drain.
- **Cadence** — while a drain is still needed the sweep re-runs every `pressureRetryMs` (2 min), otherwise every `sweepIntervalMs` (15 min). A VOD whose upload failed is skipped for 30 min so one bad file cannot pin the drain.
- **Cadence** — while a drain is still needed the sweep re-runs every `pressureRetryMs` (2 min), otherwise every `sweepIntervalMs` (15 min). A VOD whose upload failed is skipped for 30 min so one bad file cannot pin the drain.
- **Bounded uploads** — each upload is aborted past `uploadTimeoutFloorMs + size / uploadMinThroughputMBps`; a watchdog logs a sweep that has run for hours.
- **Bounded uploads** — each upload is aborted past `uploadTimeoutFloorMs + size / uploadMinThroughputMBps`; a watchdog logs a sweep that has run for hours.
@@ -92,6 +92,13 @@ const DEFAULTS = {
@@ -92,6 +92,13 @@ const DEFAULTS = {
// free space is below minFreeGb and keep going until targetFreeGb is free again.
// free space is below minFreeGb and keep going until targetFreeGb is free again.
minFreeGb: 25,
minFreeGb: 25,
targetFreeGb: 40,
targetFreeGb: 40,
+ // The recorder's real guard rails (server/vod/recorder.js): it warns below VOD_DISK_WARN_GB free and
+ // refuses new recordings below VOD_DISK_CRIT_GB. The disk is shared with ~50 services, so the
+ // percentage/GB targets above can stay unmet for days with nothing Media could move; a drain that
+ // freed nothing is alert-worthy only when there was something eligible to move, or when free space
+ // is under these.
+ recorderWarnGb: parseFloat(process.env.VOD_DISK_WARN_GB) || 15,
+ recorderCritGb: parseFloat(process.env.VOD_DISK_CRIT_GB) || 5,
// While a drain is still needed after a pass, re-run this soon instead of waiting
// While a drain is still needed after a pass, re-run this soon instead of waiting
// for the regular interval — recovery from a full disk should take minutes.
// for the regular interval — recovery from a full disk should take minutes.
pressureRetryMs: 2 * 60 * 1000,
pressureRetryMs: 2 * 60 * 1000,
@@ -938,6 +945,20 @@ function drainSatisfied(disk, settings) {
@@ -938,6 +945,20 @@ function drainSatisfied(disk, settings) {
return disk.usePct <= settings.localLowWaterPct && freeGb >= settings.targetFreeGb;
return disk.usePct <= settings.localLowWaterPct && freeGb >= settings.targetFreeGb;
}
}
+/**
+ * What a pressure pass means for alerting. `nothingEligible`: under pressure with no candidate at all
+ * (nothing old enough on local disk; not Media's to fix). `stalled`: a pressure pass freed nothing
+ * although candidates existed (uploads failing, back-off, holds) or free space is under the
+ * recorder's warn guard rail. Only `stalled` counts toward drain_stalled.
+ */
+function assessDrain({ underPressure, candidates, migrated, disk }, settings) {
+ const freeGb = disk && disk.total ? disk.available / GB : Infinity;
+ const belowRecorderGuard = freeGb < settings.recorderWarnGb;
+ const nothingEligible = !!underPressure && !candidates;
+ const stalled = !!underPressure && !migrated && (candidates > 0 || belowRecorderGuard);
+ return { nothingEligible, stalled, belowRecorderGuard };
+}
+
function isCritical(disk, settings) {
function isCritical(disk, settings) {
return !!disk && !!disk.total && disk.usePct >= settings.criticalDiskPct;
return !!disk && !!disk.total && disk.usePct >= settings.criticalDiskPct;
}
}
@@ -960,8 +981,9 @@ const sweepState = {
@@ -960,8 +981,9 @@ const sweepState = {
lastResult: null,
lastResult: null,
nextRunAt: null,
nextRunAt: null,
startedAt: null,
startedAt: null,
- stalledPasses: 0, // consecutive pressure passes that freed nothing
+ stalledPasses: 0, // consecutive pressure passes that freed nothing although they could have (assessDrain)
stalled: false,
stalled: false,
+ nothingEligible: false, // last pass was under pressure with nothing old enough to move (logged once per change)
};
};
const alertLastSentAt = new Map(); // kind → epoch ms
const alertLastSentAt = new Map(); // kind → epoch ms
@@ -1012,8 +1034,6 @@ async function runSweep() {
@@ -1012,8 +1034,6 @@ async function runSweep() {
let candidates;
let candidates;
if (underPressure) {
if (underPressure) {
const minAge = critical ? `-${settings.criticalMinAgeHours} hours` : '-1 day';
const minAge = critical ? `-${settings.criticalMinAgeHours} hours` : '-1 day';
- console.log(`[VodStorage] Disk at ${disk.usePct}%, ${(disk.available / GB).toFixed(1)} GB free (${critical ? 'CRITICAL' : 'pressure'}) — draining to ≤${settings.localLowWaterPct}% / ≥${settings.targetFreeGb} GB free`
- + (ghosts ? `; quarantined ${ghosts} file-less VOD row(s)` : ''));
candidates = await db.all(`
candidates = await db.all(`
SELECT id, file_path, file_size FROM vods
SELECT id, file_path, file_size FROM vods
WHERE ${OFFLOADABLE_WHERE}
WHERE ${OFFLOADABLE_WHERE}
@@ -1021,6 +1041,11 @@ async function runSweep() {
@@ -1021,6 +1041,11 @@ async function runSweep() {
ORDER BY (last_accessed_at IS NOT NULL), last_accessed_at ASC, view_count ASC, file_size DESC
ORDER BY (last_accessed_at IS NOT NULL), last_accessed_at ASC, view_count ASC, file_size DESC
LIMIT 200
LIMIT 200
`, [minAge]);
`, [minAge]);
+ // With nothing old enough to move the pass can do nothing; say so once per change, not every pass.
+ if (candidates.length || ghosts || !sweepState.nothingEligible) {
+ console.log(`[VodStorage] Disk at ${disk.usePct}%, ${(disk.available / GB).toFixed(1)} GB free (${critical ? 'CRITICAL' : 'pressure'}) — draining to ≤${settings.localLowWaterPct}% / ≥${settings.targetFreeGb} GB free`
+ + (ghosts ? `; quarantined ${ghosts} file-less VOD row(s)` : ''));
+ }
} else {
} else {
if (ghosts) console.log(`[VodStorage] Quarantined ${ghosts} file-less VOD row(s)`);
if (ghosts) console.log(`[VodStorage] Quarantined ${ghosts} file-less VOD row(s)`);
candidates = await db.all(`
candidates = await db.all(`
@@ -1065,24 +1090,31 @@ async function runSweep() {
@@ -1065,24 +1090,31 @@ async function runSweep() {
// Say what happened whenever it matters: the old sweep only logged on success, so
// Say what happened whenever it matters: the old sweep only logged on success, so
// weeks of "draining" that freed nothing looked healthy in the journal.
// weeks of "draining" that freed nothing looked healthy in the journal.
- if (underPressure || errors.length || quarantined || skippedHeld) {
+ const diskAfter = diskUsage(config.vod.path);
+ const verdict = assessDrain({ underPressure, candidates: candidates.length, migrated, disk: diskAfter }, settings);
+ const nothingEligibleNews = verdict.nothingEligible && !sweepState.nothingEligible;
+ if ((underPressure && (candidates.length || nothingEligibleNews)) || errors.length || quarantined || skippedHeld) {
const freedMb = (bytesFreed / 1048576).toFixed(1);
const freedMb = (bytesFreed / 1048576).toFixed(1);
console.log(`[VodStorage] Offload pass: ${candidates.length} candidate(s), ${migrated} uploaded (${freedMb} MB freed), `
console.log(`[VodStorage] Offload pass: ${candidates.length} candidate(s), ${migrated} uploaded (${freedMb} MB freed), `
+ `${errors.length} failed, ${skippedBackoff} in retry back-off, ${quarantined} quarantined`
+ `${errors.length} failed, ${skippedBackoff} in retry back-off, ${quarantined} quarantined`
+ (skippedHeld ? `, ${skippedHeld} under a retention hold` : '')
+ (skippedHeld ? `, ${skippedHeld} under a retention hold` : '')
- + (underPressure ? `, disk now ${diskUsage(config.vod.path).usePct}%` : ''));
+ + (underPressure ? `, disk now ${diskAfter.usePct}%` : ''));
for (const e of errors.slice(0, 5)) console.warn(`[VodStorage] VOD ${e.id}: ${e.error}`);
for (const e of errors.slice(0, 5)) console.warn(`[VodStorage] VOD ${e.id}: ${e.error}`);
- if (underPressure && !migrated && !candidates.length) {
- console.warn('[VodStorage] Disk is under pressure but nothing is eligible to offload — check for stuck recordings or non-VOD files in the VOD directory');
+ if (nothingEligibleNews) {
+ console.warn(`[VodStorage] Disk is under pressure but nothing is eligible to offload (recordings under ${critical ? `${settings.criticalMinAgeHours} hours` : 'a day'} old stay local); `
+ + `not logged again until that changes. Alerts only below ${settings.recorderWarnGb} GB free (VOD_DISK_WARN_GB).`);
}
}
}
}
+ if (sweepState.nothingEligible && !verdict.nothingEligible && underPressure && candidates.length) {
+ console.log(`[VodStorage] ${candidates.length} VOD(s) now eligible to offload`);
+ }
+ sweepState.nothingEligible = verdict.nothingEligible;
- // Stall detection: a pressure pass that freed nothing is the failure mode that
- // went unnoticed for weeks. Count them, and after a couple in a row raise an
- // alert; announce recovery once the drain works again.
- const diskAfter = diskUsage(config.vod.path);
+ // Stall detection: a pressure pass that freed nothing although it could have is the
+ // failure mode that went unnoticed for weeks. Count them, and after a couple in a row
+ // raise an alert; announce recovery once the drain works again (or has nothing to do).
const stillNeedsDrain = underPressure && !drainSatisfied(diskAfter, settings);
const stillNeedsDrain = underPressure && !drainSatisfied(diskAfter, settings);
- if (underPressure && !migrated) {
+ if (verdict.stalled) {
sweepState.stalledPasses++;
sweepState.stalledPasses++;
if (sweepState.stalledPasses >= settings.alertAfterStalledPasses) {
if (sweepState.stalledPasses >= settings.alertAfterStalledPasses) {
sweepState.stalled = true;
sweepState.stalled = true;
@@ -1092,8 +1124,9 @@ async function runSweep() {
@@ -1092,8 +1124,9 @@ async function runSweep() {
stalled_passes: sweepState.stalledPasses,
stalled_passes: sweepState.stalledPasses,
candidates: candidates.length,
candidates: candidates.length,
errors: errors.slice(0, 3),
errors: errors.slice(0, 3),
+ below_recorder_warn_gb: verdict.belowRecorderGuard,
hint: !candidates.length
hint: !candidates.length
- ? 'nothing eligible to offload (stuck recordings? non-VOD files in the VOD directory?)'
+ ? `under ${settings.recorderWarnGb} GB free (VOD_DISK_WARN_GB) and nothing eligible to offload (stuck recordings? other services filling the disk?)`
: errors.length ? 'uploads failing — check B2 credentials/bucket and network' : 'all candidates in retry back-off',
: errors.length ? 'uploads failing — check B2 credentials/bucket and network' : 'all candidates in retry back-off',
}, settings);
}, settings);
}
}
@@ -1492,6 +1525,7 @@ module.exports = {
@@ -1492,6 +1525,7 @@ module.exports = {
needsDrain,
needsDrain,
drainSatisfied,
drainSatisfied,
isCritical,
isCritical,
+ assessDrain,
planNextDelayMs,
planNextDelayMs,
uploadTimeoutMs,
uploadTimeoutMs,
providerOf,
providerOf,
@@ -16,7 +16,7 @@ const assert = require('assert');
@@ -16,7 +16,7 @@ const assert = require('assert');
// The revisioned tier policies load once the database is open (server/index.js does this at boot).
// The revisioned tier policies load once the database is open (server/index.js does this at boot).
await storage.tierConfig.init(storage.DEFAULTS);
await storage.tierConfig.init(storage.DEFAULTS);
await require('../server/objects/tier-policy').init();
await require('../server/objects/tier-policy').init();
- const { DEFAULTS, OFFLOADABLE_WHERE, needsDrain, drainSatisfied, isCritical, planNextDelayMs, uploadTimeoutMs } = storage;
+ const { DEFAULTS, OFFLOADABLE_WHERE, needsDrain, drainSatisfied, isCritical, assessDrain, planNextDelayMs, uploadTimeoutMs } = storage;
const GB = 1024 * 1024 * 1024;
const GB = 1024 * 1024 * 1024;
const disk = (usePct, freeGb, totalGb = 96) => ({ total: totalGb * GB, used: (totalGb - freeGb) * GB, available: freeGb * GB, usePct, mount: '/' });
const disk = (usePct, freeGb, totalGb = 96) => ({ total: totalGb * GB, used: (totalGb - freeGb) * GB, available: freeGb * GB, usePct, mount: '/' });
@@ -57,6 +57,21 @@ const assert = require('assert');
@@ -57,6 +57,21 @@ const assert = require('assert');
assert.strictEqual(isCritical(disk(89, 10), DEFAULTS), false);
assert.strictEqual(isCritical(disk(89, 10), DEFAULTS), false);
console.log('✅ free-space budget: percent ceiling OR GB floor start a drain; both targets end it');
console.log('✅ free-space budget: percent ceiling OR GB floor start a drain; both targets end it');
+ // ── Alerting measures what Media can act on: a shared 96 GB disk sits above the targets for days ──
+ const shared = disk(70, 28); // under pressure (≥70%), well above the recorder's 15 GB warn line
+ assert.deepStrictEqual(assessDrain({ underPressure: true, candidates: 0, migrated: 0, disk: shared }, DEFAULTS),
+ { nothingEligible: true, stalled: false, belowRecorderGuard: false }, 'nothing old enough to move is not a stall');
+ assert.strictEqual(assessDrain({ underPressure: true, candidates: 3, migrated: 0, disk: shared }, DEFAULTS).stalled, true,
+ 'eligible candidates that could not be moved are a stall');
+ assert.strictEqual(assessDrain({ underPressure: true, candidates: 3, migrated: 1, disk: shared }, DEFAULTS).stalled, false, 'progress is no stall');
+ const tight = disk(88, 12); // under VOD_DISK_WARN_GB (15 GB) free
+ assert.deepStrictEqual(assessDrain({ underPressure: true, candidates: 0, migrated: 0, disk: tight }, DEFAULTS),
+ { nothingEligible: true, stalled: true, belowRecorderGuard: true }, 'under the recorder guard rail it alerts even with nothing eligible');
+ assert.strictEqual(assessDrain({ underPressure: false, candidates: 0, migrated: 0, disk: disk(50, 48) }, DEFAULTS).stalled, false);
+ assert.strictEqual(DEFAULTS.recorderWarnGb, parseFloat(process.env.VOD_DISK_WARN_GB) || 15);
+ assert.strictEqual(DEFAULTS.recorderCritGb, parseFloat(process.env.VOD_DISK_CRIT_GB) || 5);
+ console.log('✅ drain_stalled only when eligible VODs could not move, or free space is under the recorder guard rail');
+
// ── Cadence: come back quickly while a drain is making (or could make) progress ──
// ── Cadence: come back quickly while a drain is making (or could make) progress ──
assert.strictEqual(planNextDelayMs({ stillNeedsDrain: true, migrated: 3 }, DEFAULTS), DEFAULTS.pressureRetryMs, 'progress under pressure → soon');
assert.strictEqual(planNextDelayMs({ stillNeedsDrain: true, migrated: 3 }, DEFAULTS), DEFAULTS.pressureRetryMs, 'progress under pressure → soon');
assert.strictEqual(planNextDelayMs({ stillNeedsDrain: true, migrated: 0, skippedBackoff: 2 }, DEFAULTS), DEFAULTS.pressureRetryMs, 'candidates in back-off → soon');
assert.strictEqual(planNextDelayMs({ stillNeedsDrain: true, migrated: 0, skippedBackoff: 2 }, DEFAULTS), DEFAULTS.pressureRetryMs, 'candidates in back-off → soon');