openvibers/openvibe.host Commits
Share protected-session wait budget across checks (#371)
Changes
@@@ -281,7 +281,7 @@ async function computePlan(ctx, svc, { mode = 'deploy', to = null, fetch = true,async function guardSessions(ctx, svc, opts, record, when) {const { exec, log } = ctx;const first = await probes.countProtected(exec, svc);-- record.protectedSessions = { count: first.count, label: first.label, unknown: first.unknown || null, checkedWhen: when };++ record.protectedSessions = { ...record.protectedSessions, count: first.count, label: first.label, unknown: first.unknown || null, checkedWhen: when };if (first.notRunning || first.count === 0) return;const describe = (r) => (r.count == null ? `${r.label}: unknown (${r.unknown}) — treated as active` : `${r.count} ${r.label}`);if (opts.force) {@@@ -301,19 +301,25 @@ async function guardSessions(ctx, svc, opts, record, when) {if (!wait) {throw new OpError(`${describe(first)} — refusing to restart ${svc.units.join(', ')} (${when}). Re-run with --wait-idle to hold until idle, or --force to drop them.`, EXIT.PROTECTED, { result: 'refused' });}-- const maxMs = svc.drain.waitMaxSeconds * 1000;++ const priorWaitedSeconds = record.protectedSessions.waitedSeconds || 0;++ const maxMs = Math.max(0, svc.drain.waitMaxSeconds - priorWaitedSeconds) * 1000;const pollMs = svc.drain.pollSeconds * 1000;const start = exec.now();let quiet = 0;let last = first;-- log(`${describe(first)}; --wait-idle: holding the restart (${when}); need ${svc.drain.quietChecks} idle checks ${svc.drain.pollSeconds}s apart, max ${svc.drain.waitMaxSeconds}s`);++ if (maxMs === 0 && svc.drain.timeoutPolicy === 'continue') {++ record.protectedSessions.timedOut = true;++ log(`!!! protected-session wait budget exhausted at ${svc.drain.waitMaxSeconds}s; proceeding with ${describe(first)} still active`);++ return;++ }++ log(`${describe(first)}; --wait-idle: holding the restart (${when}); need ${svc.drain.quietChecks} idle checks ${svc.drain.pollSeconds}s apart, max ${Math.ceil(maxMs / 1000)}s remaining of ${svc.drain.waitMaxSeconds}s total`);const poll = async () => {while (exec.now() - start < maxMs) {await exec.sleep(pollMs);last = await probes.countProtected(exec, svc);if (last.notRunning || last.count === 0) quiet += 1; else quiet = 0;if (quiet >= svc.drain.quietChecks) {-- record.protectedSessions.waitedSeconds = Math.round((exec.now() - start) / 1000);++ record.protectedSessions.waitedSeconds = priorWaitedSeconds + Math.round((exec.now() - start) / 1000);record.protectedSessions.count = 0;log(`idle for ${quiet} checks — proceeding (waited ${record.protectedSessions.waitedSeconds}s)`);return;@@@ -321,7 +327,7 @@ async function guardSessions(ctx, svc, opts, record, when) {log(`${describe(last)}; still holding (${Math.round((exec.now() - start) / 1000)}s)`);}if (svc.drain.timeoutPolicy === 'continue') {-- record.protectedSessions.waitedSeconds = Math.round((exec.now() - start) / 1000);++ record.protectedSessions.waitedSeconds = priorWaitedSeconds + Math.round((exec.now() - start) / 1000);record.protectedSessions.timedOut = true;log(`!!! protected-session wait reached ${svc.drain.waitMaxSeconds}s; proceeding with ${describe(last)} still active`);return;@@@ -247,11 +247,12 @@ runTests([test('a service configured to continue at the wait limit logs the active sessions and deploys', async () => {const host = await liveHost({ drain: { policy: 'wait', waitMaxSeconds: 2, pollSeconds: 1, timeoutPolicy: 'continue' } });-- const to = host.push({ 'server/index.js': 'v2();' }, 'v2');++ const to = host.push({ 'server/index.js': 'v2();', 'server/db/schema.sql': 'CREATE TABLE b(y);' }, 'v2');host.liveStreams = [{ is_live: true }];-- const r = await host.cli('deploy', 'live');++ const r = await host.cli('deploy', 'live', '--wait-idle');assert.strictEqual(r.code, 0, r.out);assert.match(r.out, /protected-session wait reached 2s; proceeding with 1 live streams still active/);++ assert.strictEqual((await lastRecord(host, 'live')).protectedSessions.waitedSeconds, 2, 'the second guard shares the same total wait budget');assert.strictEqual(host.currentSha(), to);assert.deepStrictEqual(host.restarts(), ['openvibe-live.service']);}),
@@ -281,7 +281,7 @@ async function computePlan(ctx, svc, { mode = 'deploy', to = null, fetch = true,
@@ -281,7 +281,7 @@ async function computePlan(ctx, svc, { mode = 'deploy', to = null, fetch = true,
async function guardSessions(ctx, svc, opts, record, when) {
async function guardSessions(ctx, svc, opts, record, when) {
const { exec, log } = ctx;
const { exec, log } = ctx;
const first = await probes.countProtected(exec, svc);
const first = await probes.countProtected(exec, svc);
- record.protectedSessions = { count: first.count, label: first.label, unknown: first.unknown || null, checkedWhen: when };
+ record.protectedSessions = { ...record.protectedSessions, count: first.count, label: first.label, unknown: first.unknown || null, checkedWhen: when };
if (first.notRunning || first.count === 0) return;
if (first.notRunning || first.count === 0) return;
const describe = (r) => (r.count == null ? `${r.label}: unknown (${r.unknown}) — treated as active` : `${r.count} ${r.label}`);
const describe = (r) => (r.count == null ? `${r.label}: unknown (${r.unknown}) — treated as active` : `${r.count} ${r.label}`);
if (opts.force) {
if (opts.force) {
@@ -301,19 +301,25 @@ async function guardSessions(ctx, svc, opts, record, when) {
@@ -301,19 +301,25 @@ async function guardSessions(ctx, svc, opts, record, when) {
if (!wait) {
if (!wait) {
throw new OpError(`${describe(first)} — refusing to restart ${svc.units.join(', ')} (${when}). Re-run with --wait-idle to hold until idle, or --force to drop them.`, EXIT.PROTECTED, { result: 'refused' });
throw new OpError(`${describe(first)} — refusing to restart ${svc.units.join(', ')} (${when}). Re-run with --wait-idle to hold until idle, or --force to drop them.`, EXIT.PROTECTED, { result: 'refused' });
}
}
- const maxMs = svc.drain.waitMaxSeconds * 1000;
+ const priorWaitedSeconds = record.protectedSessions.waitedSeconds || 0;
+ const maxMs = Math.max(0, svc.drain.waitMaxSeconds - priorWaitedSeconds) * 1000;
const pollMs = svc.drain.pollSeconds * 1000;
const pollMs = svc.drain.pollSeconds * 1000;
const start = exec.now();
const start = exec.now();
let quiet = 0;
let quiet = 0;
let last = first;
let last = first;
- log(`${describe(first)}; --wait-idle: holding the restart (${when}); need ${svc.drain.quietChecks} idle checks ${svc.drain.pollSeconds}s apart, max ${svc.drain.waitMaxSeconds}s`);
+ if (maxMs === 0 && svc.drain.timeoutPolicy === 'continue') {
+ record.protectedSessions.timedOut = true;
+ log(`!!! protected-session wait budget exhausted at ${svc.drain.waitMaxSeconds}s; proceeding with ${describe(first)} still active`);
+ return;
+ }
+ log(`${describe(first)}; --wait-idle: holding the restart (${when}); need ${svc.drain.quietChecks} idle checks ${svc.drain.pollSeconds}s apart, max ${Math.ceil(maxMs / 1000)}s remaining of ${svc.drain.waitMaxSeconds}s total`);
const poll = async () => {
const poll = async () => {
while (exec.now() - start < maxMs) {
while (exec.now() - start < maxMs) {
await exec.sleep(pollMs);
await exec.sleep(pollMs);
last = await probes.countProtected(exec, svc);
last = await probes.countProtected(exec, svc);
if (last.notRunning || last.count === 0) quiet += 1; else quiet = 0;
if (last.notRunning || last.count === 0) quiet += 1; else quiet = 0;
if (quiet >= svc.drain.quietChecks) {
if (quiet >= svc.drain.quietChecks) {
- record.protectedSessions.waitedSeconds = Math.round((exec.now() - start) / 1000);
+ record.protectedSessions.waitedSeconds = priorWaitedSeconds + Math.round((exec.now() - start) / 1000);
record.protectedSessions.count = 0;
record.protectedSessions.count = 0;
log(`idle for ${quiet} checks — proceeding (waited ${record.protectedSessions.waitedSeconds}s)`);
log(`idle for ${quiet} checks — proceeding (waited ${record.protectedSessions.waitedSeconds}s)`);
return;
return;
@@ -321,7 +327,7 @@ async function guardSessions(ctx, svc, opts, record, when) {
@@ -321,7 +327,7 @@ async function guardSessions(ctx, svc, opts, record, when) {
log(`${describe(last)}; still holding (${Math.round((exec.now() - start) / 1000)}s)`);
log(`${describe(last)}; still holding (${Math.round((exec.now() - start) / 1000)}s)`);
}
}
if (svc.drain.timeoutPolicy === 'continue') {
if (svc.drain.timeoutPolicy === 'continue') {
- record.protectedSessions.waitedSeconds = Math.round((exec.now() - start) / 1000);
+ record.protectedSessions.waitedSeconds = priorWaitedSeconds + Math.round((exec.now() - start) / 1000);
record.protectedSessions.timedOut = true;
record.protectedSessions.timedOut = true;
log(`!!! protected-session wait reached ${svc.drain.waitMaxSeconds}s; proceeding with ${describe(last)} still active`);
log(`!!! protected-session wait reached ${svc.drain.waitMaxSeconds}s; proceeding with ${describe(last)} still active`);
return;
return;
@@ -247,11 +247,12 @@ runTests([
@@ -247,11 +247,12 @@ runTests([
test('a service configured to continue at the wait limit logs the active sessions and deploys', async () => {
test('a service configured to continue at the wait limit logs the active sessions and deploys', async () => {
const host = await liveHost({ drain: { policy: 'wait', waitMaxSeconds: 2, pollSeconds: 1, timeoutPolicy: 'continue' } });
const host = await liveHost({ drain: { policy: 'wait', waitMaxSeconds: 2, pollSeconds: 1, timeoutPolicy: 'continue' } });
- const to = host.push({ 'server/index.js': 'v2();' }, 'v2');
+ const to = host.push({ 'server/index.js': 'v2();', 'server/db/schema.sql': 'CREATE TABLE b(y);' }, 'v2');
host.liveStreams = [{ is_live: true }];
host.liveStreams = [{ is_live: true }];
- const r = await host.cli('deploy', 'live');
+ const r = await host.cli('deploy', 'live', '--wait-idle');
assert.strictEqual(r.code, 0, r.out);
assert.strictEqual(r.code, 0, r.out);
assert.match(r.out, /protected-session wait reached 2s; proceeding with 1 live streams still active/);
assert.match(r.out, /protected-session wait reached 2s; proceeding with 1 live streams still active/);
+ assert.strictEqual((await lastRecord(host, 'live')).protectedSessions.waitedSeconds, 2, 'the second guard shares the same total wait budget');
assert.strictEqual(host.currentSha(), to);
assert.strictEqual(host.currentSha(), to);
assert.deepStrictEqual(host.restarts(), ['openvibe-live.service']);
assert.deepStrictEqual(host.restarts(), ['openvibe-live.service']);
}),
}),