openvibers/openrestream Commits
Let cutover-preflight --slot all check every Live slot read-only (#10)
Changes
@@@ -3,7 +3,7 @@/*** Read-only pre-flight for the RTMP cutover (docs/cutover.md). Run on the host as root:*-- * sudo node /opt/openre.stream/current/scripts/cutover-preflight.js [--only a,b] [--slot <id>] [--json]++ * sudo node /opt/openre.stream/current/scripts/cutover-preflight.js [--only a,b] [--slot <id>|all] [--json]** Checks (in this order; --only / --skip pick some):* release `current` points at the expected release (--expect-release, else origin/main of@@@ -21,7 +21,8 @@* OV_OAUTH_CLIENT_SECRET, and the running Live process was started with them* events Events has Live's openre.session.* subscription to /internal/openre-events, enabled,* with the secret Live holds (compared by hash, never printed), and no dead deliveries-- * slot only with --slot <id>: the Live slot and its OpenRe definition, for the per-slot step++ * slot only with --slot <id>: the Live slot and its OpenRe definition, for the per-slot step;++ * --slot all (or --all-slots): every Live slot, one slot:<id> line each, same rules** "port": from the host itself a connection to its own public address never crosses the provider* edge, so the result is MANUAL there. Prove it from outside: run `--only port` on any machine@@@ -420,9 +421,8 @@ async function checkEvents(ctx) {return [r.pending ? 'WARN' : 'PASS', `${s.id} enabled, secret matches Live's${r.pending ? `, ${r.pending} deliveries waiting or retrying` : ', nothing waiting'}`];}--async function checkSlot(ctx) {++async function checkSlot(ctx, id = Number(ctx.opts.slot)) {const { opts } = ctx;-- const id = Number(opts.slot);const live = await withDb(ctx.deps, opts.liveDb, async (db) => {const slot = await db.prepare('SELECT id, user_id, slug, title, protocol, streaming_method, ingest_authority, openre_stream_id FROM managed_streams WHERE id = ?').get(id);if (!slot) return { slot: null };@@@ -461,6 +461,21 @@ async function checkSlot(ctx) {return ['PASS', facts.join('; ')];}++/** `--slot all`: every Live managed stream, one result each, by the same per-slot rules. */++async function checkAllSlots(ctx) {++ const rows = await withDb(ctx.deps, ctx.opts.liveDb, (db) => db.prepare('SELECT id FROM managed_streams ORDER BY id').all());++ if (rows.error) return [{ id: 'slot', status: 'FAIL', detail: rows.error }];++ if (!rows.length) return [{ id: 'slot', status: 'WARN', detail: 'Live has no managed streams' }];++ const out = [];++ for (const { id } of rows) {++ let status;++ let detail;++ try { [status, detail] = await checkSlot(ctx, Number(id)); } catch (err) { status = 'FAIL'; detail = `check crashed: ${err.message}`; }++ out.push({ id: `slot:${id}`, status, detail });++ }++ return out;++}++const RUNNERS = { release: checkRelease, service: checkService, env: checkEnv, bind: checkBind, dns: checkDns, port: checkPort, db: checkDb, 'live-env': checkLiveEnv, events: checkEvents, slot: checkSlot };// ── entry ───────────────────────────────────────────────────────────────────@@@ -473,6 +488,7 @@ function parseArgs(argv) {const a = argv[i];if (a === '--json') opts.json = true;else if (a === '--strict') opts.strict = true;++ else if (a === '--all-slots') opts.slot = 'all';else if (a === '--only') { opts.only = take(i).split(',').map((s) => s.trim()).filter(Boolean); i++; }else if (a === '--skip') { opts.skip = take(i).split(',').map((s) => s.trim()).filter(Boolean); i++; }else if (a === '--host-ip') { opts.hostIps.push(...take(i).split(',').map((s) => s.trim()).filter(Boolean)); i++; }@@@ -486,7 +502,7 @@ function parseArgs(argv) {opts.openreUrl = String(opts.openreUrl).replace(/\/+$/, '');opts.api = String(opts.api).replace(/\/+$/, '');for (const c of [...(opts.only || []), ...opts.skip]) if (!CHECKS.includes(c)) throw new Error(`unknown check ${c} (${CHECKS.join(', ')})`);-- if (opts.slot != null && !/^\d+$/.test(String(opts.slot))) throw new Error('--slot takes a Live managed stream id');++ if (opts.slot != null && !/^(\d+|all)$/.test(String(opts.slot))) throw new Error('--slot takes a Live managed stream id, or `all`');if (!Number.isInteger(opts.port) || opts.port <= 0) throw new Error('--port must be a TCP port');return opts;}@@@ -497,6 +513,10 @@ async function preflight(opts, deps = realDeps()) {const results = [];for (const id of wanted) {if (id === 'slot' && opts.slot == null) { results.push({ id, status: 'SKIP', detail: 'pass --slot <id>' }); continue; }++ if (id === 'slot' && opts.slot === 'all') {++ try { results.push(...await checkAllSlots(ctx)); } catch (err) { results.push({ id, status: 'FAIL', detail: `check crashed: ${err.message}` }); }++ continue;++ }let status;let detail;try { [status, detail] = await RUNNERS[id](ctx); } catch (err) { status = 'FAIL'; detail = `check crashed: ${err.message}`; }@@@ -515,7 +535,7 @@ function format(report) {if (require.main === module) {let opts;try { opts = parseArgs(process.argv.slice(2)); } catch (err) { console.error(`cutover-preflight: ${err.message}`); process.exit(2); }-- if (opts.help) { console.log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, 37).join('\n')); process.exit(0); }++ if (opts.help) { console.log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, 38).join('\n')); process.exit(0); }preflight(opts).then((report) => {console.log(opts.json ? JSON.stringify(report, null, 2) : format(report));process.exit(report.ok ? 0 : 1);@@@ -395,9 +395,43 @@ t('slot: offline RTMP slot with a subject and a migrated definition passes; liveassert.strictEqual(r.slot.status, 'SKIP');});++t('slot all: one line per Live slot by the same rules, read-only, FAIL if any slot fails', async () => {++ const dbs = await makeDbs();++ let r = await preflight(opts(dbs, ['--only', 'slot', '--all-slots']), host(dbs));++ assert.deepStrictEqual(r.results.map((x) => x.id), ['slot:12']);++ assert.strictEqual(r.ok, true);++ assert.strictEqual(byId(r)['slot:12'].detail, byId(await preflight(opts(dbs, ['--only', 'slot', '--slot', '12']), host(dbs))).slot.detail);++++ const l = new Database(dbs.livePath);++ await l.prepare("INSERT INTO users (id, username) VALUES (6, 'nosub')").run();++ await l.prepare("INSERT INTO managed_streams (id, user_id, slug, title, protocol, streaming_method) VALUES (13, 6, 'x', 'X', 'rtmp', 'obs'), (14, 5, 'y', 'Y', 'webrtc', 'browser')").run();++ l.close();++ const before = (await dbs.openre.prepare('SELECT count(*) AS n FROM migration_map').get()).n;++ r = await preflight(opts(dbs, ['--only', 'slot', '--slot', 'all']), host(dbs));++ assert.deepStrictEqual(r.results.map((x) => [x.id, x.status]), [['slot:12', 'PASS'], ['slot:13', 'FAIL'], ['slot:14', 'FAIL']]);++ assert.match(r.results[1].detail, /no canonical subject/);++ assert.match(r.results[2].detail, /not an RTMP slot/);++ assert.strictEqual(r.ok, false);++ assert.strictEqual(Number((await dbs.openre.prepare('SELECT count(*) AS n FROM migration_map').get()).n), Number(before), 'writes nothing');++ assert.match(format(r), /^PASS {3}slot:12 /m);++++ // no --only: the per-slot lines replace the single slot line++ r = await preflight(opts(dbs, ['--all-slots']), host(dbs));++ assert.deepStrictEqual(r.results.filter((x) => x.id.startsWith('slot')).map((x) => x.id), ['slot:12', 'slot:13', 'slot:14']);++++ const empty = await makeDbs();++ const e = new Database(empty.livePath);++ await e.prepare('DELETE FROM managed_streams').run();++ e.close();++ r = await preflight(opts(empty, ['--only', 'slot', '--slot', 'all']), host(empty));++ assert.strictEqual(r.results[0].status, 'WARN');++});++t('arguments, ss parsing and the secret comparison', () => {assert.throws(() => parseArgs(['--only', 'nope']), /unknown check nope/);assert.throws(() => parseArgs(['--slot', 'abc']), /--slot/);++ assert.strictEqual(parseArgs(['--slot', 'all']).slot, 'all');++ assert.strictEqual(parseArgs(['--all-slots']).slot, 'all');assert.throws(() => parseArgs(['--frobnicate']), /unknown argument/);assert.deepStrictEqual(parseArgs(['--host-ip', '1.2.3.4,5.6.7.8', '--host-ip', '9.9.9.9']).hostIps, ['1.2.3.4', '5.6.7.8', '9.9.9.9']);assert.deepStrictEqual(parseSsListeners('LISTEN 0 511 127.0.0.1:1936 0.0.0.0:*\nLISTEN 0 511 [::]:1936 [::]:*\nLISTEN 0 511 *:1936 *:*\nLISTEN 0 511 127.0.0.1:19360 0.0.0.0:*\n', 1936), ['127.0.0.1', '::', '*']);
@@ -3,7 +3,7 @@
@@ -3,7 +3,7 @@
/**
/**
* Read-only pre-flight for the RTMP cutover (docs/cutover.md). Run on the host as root:
* Read-only pre-flight for the RTMP cutover (docs/cutover.md). Run on the host as root:
*
*
- * sudo node /opt/openre.stream/current/scripts/cutover-preflight.js [--only a,b] [--slot <id>] [--json]
+ * sudo node /opt/openre.stream/current/scripts/cutover-preflight.js [--only a,b] [--slot <id>|all] [--json]
*
*
* Checks (in this order; --only / --skip pick some):
* Checks (in this order; --only / --skip pick some):
* release `current` points at the expected release (--expect-release, else origin/main of
* release `current` points at the expected release (--expect-release, else origin/main of
@@ -21,7 +21,8 @@
@@ -21,7 +21,8 @@
* OV_OAUTH_CLIENT_SECRET, and the running Live process was started with them
* OV_OAUTH_CLIENT_SECRET, and the running Live process was started with them
* events Events has Live's openre.session.* subscription to /internal/openre-events, enabled,
* events Events has Live's openre.session.* subscription to /internal/openre-events, enabled,
* with the secret Live holds (compared by hash, never printed), and no dead deliveries
* with the secret Live holds (compared by hash, never printed), and no dead deliveries
- * slot only with --slot <id>: the Live slot and its OpenRe definition, for the per-slot step
+ * slot only with --slot <id>: the Live slot and its OpenRe definition, for the per-slot step;
+ * --slot all (or --all-slots): every Live slot, one slot:<id> line each, same rules
*
*
* "port": from the host itself a connection to its own public address never crosses the provider
* "port": from the host itself a connection to its own public address never crosses the provider
* edge, so the result is MANUAL there. Prove it from outside: run `--only port` on any machine
* edge, so the result is MANUAL there. Prove it from outside: run `--only port` on any machine
@@ -420,9 +421,8 @@ async function checkEvents(ctx) {
@@ -420,9 +421,8 @@ async function checkEvents(ctx) {
return [r.pending ? 'WARN' : 'PASS', `${s.id} enabled, secret matches Live's${r.pending ? `, ${r.pending} deliveries waiting or retrying` : ', nothing waiting'}`];
return [r.pending ? 'WARN' : 'PASS', `${s.id} enabled, secret matches Live's${r.pending ? `, ${r.pending} deliveries waiting or retrying` : ', nothing waiting'}`];
}
}
-async function checkSlot(ctx) {
+async function checkSlot(ctx, id = Number(ctx.opts.slot)) {
const { opts } = ctx;
const { opts } = ctx;
- const id = Number(opts.slot);
const live = await withDb(ctx.deps, opts.liveDb, async (db) => {
const live = await withDb(ctx.deps, opts.liveDb, async (db) => {
const slot = await db.prepare('SELECT id, user_id, slug, title, protocol, streaming_method, ingest_authority, openre_stream_id FROM managed_streams WHERE id = ?').get(id);
const slot = await db.prepare('SELECT id, user_id, slug, title, protocol, streaming_method, ingest_authority, openre_stream_id FROM managed_streams WHERE id = ?').get(id);
if (!slot) return { slot: null };
if (!slot) return { slot: null };
@@ -461,6 +461,21 @@ async function checkSlot(ctx) {
@@ -461,6 +461,21 @@ async function checkSlot(ctx) {
return ['PASS', facts.join('; ')];
return ['PASS', facts.join('; ')];
}
}
+/** `--slot all`: every Live managed stream, one result each, by the same per-slot rules. */
+async function checkAllSlots(ctx) {
+ const rows = await withDb(ctx.deps, ctx.opts.liveDb, (db) => db.prepare('SELECT id FROM managed_streams ORDER BY id').all());
+ if (rows.error) return [{ id: 'slot', status: 'FAIL', detail: rows.error }];
+ if (!rows.length) return [{ id: 'slot', status: 'WARN', detail: 'Live has no managed streams' }];
+ const out = [];
+ for (const { id } of rows) {
+ let status;
+ let detail;
+ try { [status, detail] = await checkSlot(ctx, Number(id)); } catch (err) { status = 'FAIL'; detail = `check crashed: ${err.message}`; }
+ out.push({ id: `slot:${id}`, status, detail });
+ }
+ return out;
+}
+
const RUNNERS = { release: checkRelease, service: checkService, env: checkEnv, bind: checkBind, dns: checkDns, port: checkPort, db: checkDb, 'live-env': checkLiveEnv, events: checkEvents, slot: checkSlot };
const RUNNERS = { release: checkRelease, service: checkService, env: checkEnv, bind: checkBind, dns: checkDns, port: checkPort, db: checkDb, 'live-env': checkLiveEnv, events: checkEvents, slot: checkSlot };
// ── entry ───────────────────────────────────────────────────────────────────
// ── entry ───────────────────────────────────────────────────────────────────
@@ -473,6 +488,7 @@ function parseArgs(argv) {
@@ -473,6 +488,7 @@ function parseArgs(argv) {
const a = argv[i];
const a = argv[i];
if (a === '--json') opts.json = true;
if (a === '--json') opts.json = true;
else if (a === '--strict') opts.strict = true;
else if (a === '--strict') opts.strict = true;
+ else if (a === '--all-slots') opts.slot = 'all';
else if (a === '--only') { opts.only = take(i).split(',').map((s) => s.trim()).filter(Boolean); i++; }
else if (a === '--only') { opts.only = take(i).split(',').map((s) => s.trim()).filter(Boolean); i++; }
else if (a === '--skip') { opts.skip = take(i).split(',').map((s) => s.trim()).filter(Boolean); i++; }
else if (a === '--skip') { opts.skip = take(i).split(',').map((s) => s.trim()).filter(Boolean); i++; }
else if (a === '--host-ip') { opts.hostIps.push(...take(i).split(',').map((s) => s.trim()).filter(Boolean)); i++; }
else if (a === '--host-ip') { opts.hostIps.push(...take(i).split(',').map((s) => s.trim()).filter(Boolean)); i++; }
@@ -486,7 +502,7 @@ function parseArgs(argv) {
@@ -486,7 +502,7 @@ function parseArgs(argv) {
opts.openreUrl = String(opts.openreUrl).replace(/\/+$/, '');
opts.openreUrl = String(opts.openreUrl).replace(/\/+$/, '');
opts.api = String(opts.api).replace(/\/+$/, '');
opts.api = String(opts.api).replace(/\/+$/, '');
for (const c of [...(opts.only || []), ...opts.skip]) if (!CHECKS.includes(c)) throw new Error(`unknown check ${c} (${CHECKS.join(', ')})`);
for (const c of [...(opts.only || []), ...opts.skip]) if (!CHECKS.includes(c)) throw new Error(`unknown check ${c} (${CHECKS.join(', ')})`);
- if (opts.slot != null && !/^\d+$/.test(String(opts.slot))) throw new Error('--slot takes a Live managed stream id');
+ if (opts.slot != null && !/^(\d+|all)$/.test(String(opts.slot))) throw new Error('--slot takes a Live managed stream id, or `all`');
if (!Number.isInteger(opts.port) || opts.port <= 0) throw new Error('--port must be a TCP port');
if (!Number.isInteger(opts.port) || opts.port <= 0) throw new Error('--port must be a TCP port');
return opts;
return opts;
}
}
@@ -497,6 +513,10 @@ async function preflight(opts, deps = realDeps()) {
@@ -497,6 +513,10 @@ async function preflight(opts, deps = realDeps()) {
const results = [];
const results = [];
for (const id of wanted) {
for (const id of wanted) {
if (id === 'slot' && opts.slot == null) { results.push({ id, status: 'SKIP', detail: 'pass --slot <id>' }); continue; }
if (id === 'slot' && opts.slot == null) { results.push({ id, status: 'SKIP', detail: 'pass --slot <id>' }); continue; }
+ if (id === 'slot' && opts.slot === 'all') {
+ try { results.push(...await checkAllSlots(ctx)); } catch (err) { results.push({ id, status: 'FAIL', detail: `check crashed: ${err.message}` }); }
+ continue;
+ }
let status;
let status;
let detail;
let detail;
try { [status, detail] = await RUNNERS[id](ctx); } catch (err) { status = 'FAIL'; detail = `check crashed: ${err.message}`; }
try { [status, detail] = await RUNNERS[id](ctx); } catch (err) { status = 'FAIL'; detail = `check crashed: ${err.message}`; }
@@ -515,7 +535,7 @@ function format(report) {
@@ -515,7 +535,7 @@ function format(report) {
if (require.main === module) {
if (require.main === module) {
let opts;
let opts;
try { opts = parseArgs(process.argv.slice(2)); } catch (err) { console.error(`cutover-preflight: ${err.message}`); process.exit(2); }
try { opts = parseArgs(process.argv.slice(2)); } catch (err) { console.error(`cutover-preflight: ${err.message}`); process.exit(2); }
- if (opts.help) { console.log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, 37).join('\n')); process.exit(0); }
+ if (opts.help) { console.log(fs.readFileSync(__filename, 'utf8').split('\n').slice(2, 38).join('\n')); process.exit(0); }
preflight(opts).then((report) => {
preflight(opts).then((report) => {
console.log(opts.json ? JSON.stringify(report, null, 2) : format(report));
console.log(opts.json ? JSON.stringify(report, null, 2) : format(report));
process.exit(report.ok ? 0 : 1);
process.exit(report.ok ? 0 : 1);
@@ -395,9 +395,43 @@ t('slot: offline RTMP slot with a subject and a migrated definition passes; live
@@ -395,9 +395,43 @@ t('slot: offline RTMP slot with a subject and a migrated definition passes; live
assert.strictEqual(r.slot.status, 'SKIP');
assert.strictEqual(r.slot.status, 'SKIP');
});
});
+t('slot all: one line per Live slot by the same rules, read-only, FAIL if any slot fails', async () => {
+ const dbs = await makeDbs();
+ let r = await preflight(opts(dbs, ['--only', 'slot', '--all-slots']), host(dbs));
+ assert.deepStrictEqual(r.results.map((x) => x.id), ['slot:12']);
+ assert.strictEqual(r.ok, true);
+ assert.strictEqual(byId(r)['slot:12'].detail, byId(await preflight(opts(dbs, ['--only', 'slot', '--slot', '12']), host(dbs))).slot.detail);
+
+ const l = new Database(dbs.livePath);
+ await l.prepare("INSERT INTO users (id, username) VALUES (6, 'nosub')").run();
+ await l.prepare("INSERT INTO managed_streams (id, user_id, slug, title, protocol, streaming_method) VALUES (13, 6, 'x', 'X', 'rtmp', 'obs'), (14, 5, 'y', 'Y', 'webrtc', 'browser')").run();
+ l.close();
+ const before = (await dbs.openre.prepare('SELECT count(*) AS n FROM migration_map').get()).n;
+ r = await preflight(opts(dbs, ['--only', 'slot', '--slot', 'all']), host(dbs));
+ assert.deepStrictEqual(r.results.map((x) => [x.id, x.status]), [['slot:12', 'PASS'], ['slot:13', 'FAIL'], ['slot:14', 'FAIL']]);
+ assert.match(r.results[1].detail, /no canonical subject/);
+ assert.match(r.results[2].detail, /not an RTMP slot/);
+ assert.strictEqual(r.ok, false);
+ assert.strictEqual(Number((await dbs.openre.prepare('SELECT count(*) AS n FROM migration_map').get()).n), Number(before), 'writes nothing');
+ assert.match(format(r), /^PASS {3}slot:12 /m);
+
+ // no --only: the per-slot lines replace the single slot line
+ r = await preflight(opts(dbs, ['--all-slots']), host(dbs));
+ assert.deepStrictEqual(r.results.filter((x) => x.id.startsWith('slot')).map((x) => x.id), ['slot:12', 'slot:13', 'slot:14']);
+
+ const empty = await makeDbs();
+ const e = new Database(empty.livePath);
+ await e.prepare('DELETE FROM managed_streams').run();
+ e.close();
+ r = await preflight(opts(empty, ['--only', 'slot', '--slot', 'all']), host(empty));
+ assert.strictEqual(r.results[0].status, 'WARN');
+});
+
t('arguments, ss parsing and the secret comparison', () => {
t('arguments, ss parsing and the secret comparison', () => {
assert.throws(() => parseArgs(['--only', 'nope']), /unknown check nope/);
assert.throws(() => parseArgs(['--only', 'nope']), /unknown check nope/);
assert.throws(() => parseArgs(['--slot', 'abc']), /--slot/);
assert.throws(() => parseArgs(['--slot', 'abc']), /--slot/);
+ assert.strictEqual(parseArgs(['--slot', 'all']).slot, 'all');
+ assert.strictEqual(parseArgs(['--all-slots']).slot, 'all');
assert.throws(() => parseArgs(['--frobnicate']), /unknown argument/);
assert.throws(() => parseArgs(['--frobnicate']), /unknown argument/);
assert.deepStrictEqual(parseArgs(['--host-ip', '1.2.3.4,5.6.7.8', '--host-ip', '9.9.9.9']).hostIps, ['1.2.3.4', '5.6.7.8', '9.9.9.9']);
assert.deepStrictEqual(parseArgs(['--host-ip', '1.2.3.4,5.6.7.8', '--host-ip', '9.9.9.9']).hostIps, ['1.2.3.4', '5.6.7.8', '9.9.9.9']);
assert.deepStrictEqual(parseSsListeners('LISTEN 0 511 127.0.0.1:1936 0.0.0.0:*\nLISTEN 0 511 [::]:1936 [::]:*\nLISTEN 0 511 *:1936 *:*\nLISTEN 0 511 127.0.0.1:19360 0.0.0.0:*\n', 1936), ['127.0.0.1', '::', '*']);
assert.deepStrictEqual(parseSsListeners('LISTEN 0 511 127.0.0.1:1936 0.0.0.0:*\nLISTEN 0 511 [::]:1936 [::]:*\nLISTEN 0 511 *:1936 *:*\nLISTEN 0 511 127.0.0.1:19360 0.0.0.0:*\n', 1936), ['127.0.0.1', '::', '*']);