openvibers/openvibe.live Commits
Security crawl for stream keys and secrets (roadmap WS-R task 5: stream-key exposure, internal-secret leak). test/security-crawl.js boots the real server/index.js as a restore-drill instance (reads only, no outbound connection) on a seeded temp database, lists every GET route Express knows after boot (so a route added later is crawled without anyone listing it) and requests each one, with seeded ids, a query string and nonsense ids, plus the SPA/SSR pages and the probes, as several signed-in people (real RS256 session tokens from a test Network key).
Changes
@@@ -21,7 +21,8 @@ const config = require('openvibe-shared/config');const db = require('../db/database');const NOT_CONFIG = /^(arena_backfill_cursor_|star_streamer$|deploy_last_announced$|relay_users_backfilled$|bucks_bits_migration_done$|stats_vibes_reset_at$|storage_tier\.|money_writes_frozen)/;--const SECRET = /(api[_-]?key|secret|token|password|client_secret|service_account|private[_-]?key|salt|access_key)/i;++// A webhook URL is its own credential (anyone holding it can post), so it is secret-class too.++const SECRET = /(api[_-]?key|secret|token|password|client_secret|service_account|private[_-]?key|salt|access[_-]?key|webhook)/i;const PUBLIC_KEYS = new Set(['site_name', 'site_description', 'motd', 'registration_open', 'require_email', 'nsfw_enabled']);const SYSTEM = { type: 'service', id: 'live' };@@@ -63,8 +63,10 @@ function canModerateContentOwner(actor, contentOwner) {return false;}--// Settings whose values are secrets or control money/AI spend — owner-only to view/edit.--const SENSITIVE_KEY_RE = /(api[_-]?key|secret|token|password|client_id|client_secret|service_account|private[_-]?key)/i;++// Settings whose values are secrets or control money/AI spend — owner-only to view/edit. A webhook URL++// carries its own credential (anyone holding a Discord webhook URL can post as it), and an AWS access++// key id is half of a key pair.++const SENSITIVE_KEY_RE = /(api[_-]?key|secret|token|password|client_id|client_secret|service_account|private[_-]?key|access[_-]?key|webhook)/i;// money_writes_frozen (the Billing-cutover freeze) is owner-only too; every admin can READ it at// GET /api/admin/money.const SENSITIVE_KEY_PREFIXES = ['ai_', 'stripe_', 'ccbill_', 'crypto_', 'tts_google_', 'money_'];nnew file mode 100644@@@ -0,0 +1,239 @@++'use strict';++/**++ * The crawler behind test/security-stream-keys.test.js and test/security-secrets.test.js (roadmap++ * WS-R task 5). Not a test itself (no .test.js): it boots the real server/index.js in the calling++ * process as a restore-drill instance (LIVE_DRILL, server/drill.js: HTTP on one loopback port, reads++ * only, no outbound connection, no job, no socket server) against a temp database the caller seeded,++ * lists every GET route Express knows after boot, and requests each one.++ *++ * Listing the routes from the booted app (not from a hand-kept list) is the point: a route added++ * next month is crawled without anyone remembering to add it here.++ *++ * Sign-in is real: the caller gets an RSA key pair whose public half is the "Network" key Live++ * verifies (OV_NETWORK_PUBLIC_KEY), and signs user session tokens with the private half.++ */++const assert = require('assert');++const crypto = require('crypto');++const fs = require('fs');++const http = require('http');++const net = require('net');++const os = require('os');++const path = require('path');++const { spawnSync } = require('child_process');++++const REPO = path.resolve(__dirname, '..');++const ISSUER = 'https://network.crawl.test';++++/** A temp directory with the drill's DB_PATH and DATA_DIR (both must lie outside the checkout). */++function tempEnv(name) {++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), `live-${name}-`));++ const DB_PATH = path.join(dir, 'db', 'live.db');++ const DATA_DIR = path.join(dir, 'data');++ fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });++ fs.mkdirSync(DATA_DIR, { recursive: true });++ return { dir, DB_PATH, DATA_DIR, cleanup: () => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* */ } } };++}++++/**++ * Run `script` in a child Node process in normal mode (not a drill) against the temp database, the++ * way production would have written the rows a drill later reads. Its stdout's last line, if JSON,++ * is returned parsed.++ */++function seed(tmp, script, env = {}) {++ const r = spawnSync(process.execPath, ['-e', script], {++ cwd: REPO, encoding: 'utf8',++ env: { ...process.env, DB_PATH: tmp.DB_PATH, DATA_DIR: tmp.DATA_DIR, NODE_ENV: 'test', ...env },++ });++ assert.strictEqual(r.status, 0, `seeding failed:\n${r.stdout}\n${r.stderr}`);++ const last = r.stdout.trim().split('\n').pop() || '';++ try { return JSON.parse(last); } catch { return null; }++}++++/** The "Network" signing key: its public half is what Live verifies session tokens with. */++function networkKeys(tmp) {++ const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 });++ const publicKeyPath = path.join(tmp.dir, 'network-public.pem');++ fs.writeFileSync(publicKeyPath, publicKey.export({ type: 'spki', format: 'pem' }));++ const jwt = require('jsonwebtoken');++ const sign = (claims) => jwt.sign({ ...claims }, privateKey, { algorithm: 'RS256', issuer: ISSUER, expiresIn: '1h' });++ return { publicKeyPath, sign, env: { OV_NETWORK_PUBLIC_KEY: publicKeyPath, OV_NETWORK_URL: ISSUER } };++}++++function freePort() {++ return new Promise((resolve) => { const s = net.createServer().listen(0, '127.0.0.1', () => { const p = s.address().port; s.close(() => resolve(p)); }); });++}++++/** The path an Express 4 layer is mounted at ('' for app-level middleware), or null if it is a pattern we cannot turn back into a path. */++function mountPath(layer) {++ if (!layer.regexp || layer.regexp.fast_slash) return '';++ let src = layer.regexp.source.replace(/^\^/, '').replace(/\\\/\?\(\?=\\\/\|\$\)$/i, '').replace(/\/\?\(\?=\/\|\$\)$/i, '');++ let i = 0;++ src = src.replace(/\(\?:\(\[\^\\\/\]\+\?\)\)/g, () => `:${(layer.keys[i++] || {}).name || 'param'}`);++ src = src.replace(/\\\//g, '/').replace(/\\\./g, '.').replace(/\\-/g, '-');++ return /[\\^$()|[\]*+?]/.test(src) ? null : src;++}++++/** Every route of an Express app: [{ path, methods }] with `path` a string template (':param'). */++function listRoutes(app) {++ const out = [];++ const walk = (stack, prefix) => {++ for (const layer of stack) {++ if (layer.route) {++ const methods = Object.keys(layer.route.methods).filter((m) => layer.route.methods[m]);++ for (const p of [].concat(layer.route.path)) {++ if (typeof p === 'string') out.push({ path: prefix + p, methods });++ }++ } else if (layer.handle && Array.isArray(layer.handle.stack)) {++ const mp = mountPath(layer);++ if (mp !== null) walk(layer.handle.stack, prefix + mp);++ } else {++ // A plain middleware mounted on a path (e.g. the "moved" answers): it answers everything under it.++ const mp = mountPath(layer);++ if (mp) out.push({ path: prefix + mp, methods: ['_all'] });++ }++ }++ };++ walk(app._router.stack, '');++ const seen = new Set();++ return out.filter((r) => { const k = `${r.methods.join(',')} ${r.path}`; if (seen.has(k)) return false; seen.add(k); return true; });++}++++/**++ * Concrete paths for a route template. `values(name)` gives the candidate values of a parameter;++ * a template yields one path per candidate position (candidate i for every parameter), so a route++ * with an unknown parameter is tried with each seeded id without multiplying out.++ */++function expand(template, values) {++ const names = [];++ const t = `/${template.replace(/^\/+/, '')}`.replace(/\*/g, 'x').replace(/:([A-Za-z0-9_]+)\??(\([^)]*\))?/g, (m, n) => { names.push(n); return `:${n}`; });++ if (!names.length) return [t];++ const lists = names.map((n) => values(n));++ const width = Math.max(...lists.map((l) => l.length));++ const paths = new Set();++ for (let i = 0; i < width; i++) {++ let j = 0;++ paths.add(t.replace(/:([A-Za-z0-9_]+)/g, () => { const l = lists[j++]; return encodeURIComponent(String(l[Math.min(i, l.length - 1)])); }));++ }++ return [...paths];++}++++/**++ * Boot server/index.js in this process as a drill instance on a free loopback port. `env` is added++ * to process.env first (the tests put their sentinel secrets there). Resolves with the app, its++ * routes, and request(method, path, { headers, timeoutMs }).++ */++async function boot(tmp, env = {}) {++ const port = await freePort();++ Object.assign(process.env, env, { LIVE_DRILL: '1', DB_PATH: tmp.DB_PATH, DATA_DIR: tmp.DATA_DIR, HOST: '127.0.0.1', PORT: String(port) });++ delete process.env.LISTEN_FDS;++ let app = null;++ const realCreate = http.createServer;++ http.createServer = function (...a) { const h = a.find((x) => typeof x === 'function'); if (h && typeof h.handle === 'function' && typeof h.set === 'function') app = h; return realCreate.apply(this, a); };++ delete require.cache[require.resolve('../server/drill')];++ require('../server/index.js');++ http.createServer = realCreate;++ assert.ok(app, 'server/index.js created its HTTP server from the Express app');++ const base = `http://127.0.0.1:${port}`;++ const request = (method, p, { headers = {}, timeoutMs = 4000, body = null } = {}) => new Promise((resolve) => {++ let done = false;++ const finish = (r) => { if (!done) { done = true; resolve(r); } };++ let req;++ try { new URL(base + p); } catch (e) { finish({ status: 0, headers: {}, text: '', error: `bad path ${p}` }); return; }++ req = http.request(base + p, { method, headers: { ...(body ? { 'Content-Type': 'application/json' } : {}), ...headers } }, (res) => {++ let text = '';++ const timer = setTimeout(() => { res.destroy(); finish({ status: res.statusCode, headers: res.headers, text, timedOut: true }); }, timeoutMs);++ res.setEncoding('utf8');++ res.on('data', (c) => { if (text.length < 4 * 1024 * 1024) text += c; });++ res.on('end', () => { clearTimeout(timer); finish({ status: res.statusCode, headers: res.headers, text }); });++ res.on('error', () => { clearTimeout(timer); finish({ status: res.statusCode, headers: res.headers, text }); });++ });++ req.setTimeout(timeoutMs, () => { req.destroy(); finish({ status: 0, headers: {}, text: '', timedOut: true }); });++ req.on('error', (e) => finish({ status: 0, headers: {}, text: '', error: e.message }));++ if (body) req.write(typeof body === 'string' ? body : JSON.stringify(body));++ req.end();++ });++ let ready = null;++ for (let i = 0; i < 200 && !(ready && ready.status === 200); i++) {++ ready = await request('GET', '/api/ready', { timeoutMs: 1000 });++ if (ready.status !== 200) await new Promise((r) => setTimeout(r, 100));++ }++ assert.strictEqual(ready && ready.status, 200, `drill instance not ready: ${ready && ready.text}`);++ return { app, port, base, request, routes: listRoutes(app) };++}++++/** Capture console output (the logs a public request writes) until stop() is called. */++function captureLogs() {++ const lines = [];++ const orig = {};++ for (const m of ['log', 'info', 'warn', 'error', 'debug']) {++ orig[m] = console[m];++ console[m] = (...a) => { lines.push(a.map((x) => (typeof x === 'string' ? x : (() => { try { return x instanceof Error ? `${x.message}\n${x.stack}` : JSON.stringify(x); } catch { return String(x); } })())).join(' ')); };++ }++ const wOut = process.stdout.write.bind(process.stdout);++ const wErr = process.stderr.write.bind(process.stderr);++ process.stdout.write = (c, ...a) => { lines.push(String(c)); return true; };++ process.stderr.write = (c, ...a) => { lines.push(String(c)); return true; };++ return {++ lines,++ stop() { Object.assign(console, orig); process.stdout.write = wOut; process.stderr.write = wErr; return lines; },++ };++}++++/** Which of `needles` ({ label: value }) occur in a response's body or headers: [{ label, where }]. */++function leaks(res, needles) {++ const found = [];++ const headerText = Object.entries(res.headers || {}).map(([k, v]) => `${k}: ${[].concat(v).join(', ')}`).join('\n');++ for (const [label, value] of Object.entries(needles)) {++ if (!value) continue;++ if (res.text && res.text.includes(value)) found.push({ label, where: 'body' });++ if (headerText.includes(value)) found.push({ label, where: 'headers' });++ }++ return found;++}++++/**++ * Every GET path to crawl: each GET route (and path-mounted handler) expanded with `values(name)`,++ * each also with `query` appended when given, plus `extra` (pages the route list shows only as '*').++ */++function getPaths(srv, values, { query = '', extra = [] } = {}) {++ const paths = new Set();++ for (const r of srv.routes) {++ if (!r.methods.includes('get') && !r.methods.includes('_all')) continue;++ for (const p of expand(r.path, values)) {++ paths.add(p);++ if (query && !p.includes('?')) paths.add(`${p}?${query}`);++ }++ }++ for (const p of extra) paths.add(p);++ return [...paths];++}++++/**++ * GET every path as every person, a few requests at a time. `needlesFor(who)` names the values that++ * person must never see. Resolves { found: ['who: GET path → status carries label in its where'],++ * answered, timedOut: [paths], statuses: { '2xx': n, ... } }.++ */++async function crawlAll(srv, paths, people, needlesFor, { concurrency = 6, timeoutMs = 3000 } = {}) {++ const found = [];++ const timedOut = new Set();++ const statuses = {};++ let answered = 0;++ for (const [who, headers] of Object.entries(people)) {++ const needles = needlesFor(who);++ const queue = [...paths];++ await Promise.all(Array.from({ length: concurrency }, async () => {++ while (queue.length) {++ const p = queue.shift();++ const r = await srv.request('GET', p, { headers, timeoutMs });++ if (r.status) answered++;++ if (r.timedOut) timedOut.add(p);++ const cls = r.status ? `${String(r.status)[0]}xx` : 'none';++ statuses[cls] = (statuses[cls] || 0) + 1;++ for (const l of leaks(r, needles)) found.push(`${who}: GET ${p} → ${r.status} carries ${l.label} in its ${l.where}`);++ }++ }));++ }++ return { found, answered, timedOut: [...timedOut], statuses };++}++++module.exports = { REPO, ISSUER, tempEnv, seed, networkKeys, boot, listRoutes, expand, getPaths, crawlAll, captureLogs, leaks };nnew file mode 100644@@@ -0,0 +1,210 @@++'use strict';++/**++ * Live's own secrets never leave in a response (roadmap WS-R task 5, the internal-secret class).++ *++ * Boots the real server as a restore-drill instance (test/security-crawl.js) with a sentinel value in++ * every secret environment variable Live reads (the Network OAuth client secret, the internal API++ * keys, Media's key and webhook/event secrets, the Events and OpenRe subscription secrets, TURN,++ * PayPal, the admin password, GitHub tokens, the ops webhook) and in every secret site setting an++ * owner can store (GIF, TTS, AI, Twitch/Kick/YouTube, soundboard keys, the Discord and ops++ * webhooks), then requests every GET route Express knows, the pages, the probes (/api/ready,++ * /api/health, /metrics, /release.json) and the error paths (nonsense ids, unknown paths, refused++ * writes with broken bodies) as anonymous, a viewer, a streamer, a global mod, an admin and the++ * site owner. No body or header may contain an environment secret, whoever asks; the stored++ * settings are shown to the site owner only (the settings page is where they are edited), never++ * to an admin who is not the owner. The events Live sends (stream lifecycle, release) and the log++ * lines written while serving all of that may not contain them either.++ *++ * node test/security-secrets.test.js++ */++const assert = require('assert');++const crawl = require('./security-crawl');++++let failures = 0;++async function check(name, fn) {++ try { await fn(); console.log(` ✓ ${name}`); } catch (err) { failures++; console.log(` ✗ ${name}\n ${String(err.stack || err.message).split('\n').slice(0, 14).join('\n ')}`); }++}++++// Environment secrets: obviously fake, low entropy.++const ENV = {++ JWT_SECRET: 'sentinel-not-a-secret-jwt',++ OV_OAUTH_CLIENT_SECRET: 'sentinel-not-a-secret-oauth-client',++ INTERNAL_API_KEY: 'sentinel-not-a-secret-internal-api-key',++ OV_INTERNAL_KEY: 'sentinel-not-a-secret-ov-internal-key',++ MEDIA_API_KEY: 'sentinel-not-a-secret-media-api-key',++ MEDIA_WEBHOOK_SECRET: 'sentinel-not-a-secret-media-webhook',++ MEDIA_EVENTS_SECRET: 'sentinel-not-a-secret-media-events',++ LIVE_EVENTS_SECRET: 'sentinel-not-a-secret-live-events',++ OPENRE_EVENTS_SECRET: 'sentinel-not-a-secret-openre-events',++ TURN_AUTH_SECRET: 'sentinel-not-a-secret-turn-auth',++ PAYPAL_CLIENT_SECRET: 'sentinel-not-a-secret-paypal',++ ADMIN_PASSWORD: 'sentinel-not-a-secret-admin-password',++ GITHUB_TOKEN: 'sentinel-not-a-secret-github-token',++ GH_TOKEN: 'sentinel-not-a-secret-gh-token',++ OPS_ALERT_WEBHOOK_URL: 'https://hooks.example.test/sentinel-not-a-secret-ops-webhook',++};++// What makes the process production-like (the drill refuses every connection to these anyway).++const PROD_LIKE = {++ NODE_ENV: 'production',++ EVENTS_URL: 'http://127.0.0.1:9',++ MEDIA_URL: 'http://127.0.0.1:9',++ OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',++ OV_COMMUNITY_INTERNAL_URL: 'http://127.0.0.1:9',++ OV_CHAT_INTERNAL_URL: 'http://127.0.0.1:9',++ OV_OAUTH_CLIENT_ID: 'openvibe-live',++ PAYPAL_CLIENT_ID: 'paypal-client-id-public',++};++// Secret site settings (live.site_settings), stored the way the owner's settings page stores them.++const SETTINGS = {++ gif_giphy_api_key: 'sentinel-not-a-secret-giphy',++ gif_tenor_api_key: 'sentinel-not-a-secret-tenor',++ tts_aws_access_key_id: 'sentinel-not-a-secret-aws-id',++ tts_aws_secret_access_key: 'sentinel-not-a-secret-aws-key',++ tts_google_api_key: 'sentinel-not-a-secret-google-tts',++ tts_google_service_account: '{"private_key":"sentinel-not-a-secret-google-sa"}',++ twitch_client_secret: 'sentinel-not-a-secret-twitch',++ kick_client_secret: 'sentinel-not-a-secret-kick',++ youtube_api_key: 'sentinel-not-a-secret-youtube',++ soundboard_101_api_key: 'sentinel-not-a-secret-soundboard',++ ai_api_key: 'sentinel-not-a-secret-ai',++ discord_webhook_url: 'https://discord.example.test/api/webhooks/1/sentinel-not-a-secret-discord',++ ops_alert_webhook_url: 'https://hooks.example.test/sentinel-not-a-secret-ops-setting',++};++const SETTING_NEEDLES = Object.fromEntries(Object.entries(SETTINGS).map(([k, v]) => [k, v.includes('{') ? 'sentinel-not-a-secret-google-sa' : v]));++++const SEED = `++ const db = require('./server/db/database');++ db.initDb();++ const S = JSON.parse(process.env.SETTINGS_SEED);++ const mk = (u, role, owner) => {++ const id = Number(db.createUser({ username: u, password_hash: 'x', display_name: u, email: u + '@example.test', stream_key: null }).lastInsertRowid);++ db.run('UPDATE users SET role = ?, is_owner = ? WHERE id = ?', [role, owner ? 1 : 0, id]);++ db.ensureChannel(id);++ db.getDb().prepare("INSERT INTO linked_accounts (user_id, service, service_user_id, service_username) VALUES (?, 'network', ?, ?)").run(id, String(9000 + id), u);++ return id;++ };++ const ids = {++ streamer: mk('secstreamer', 'streamer'), viewer: mk('secviewer', 'user'), mod: mk('secmod', 'global_mod'),++ admin: mk('secadmin', 'admin'), owner: mk('secowner', 'admin', true),++ };++ for (const [k, v] of Object.entries(S)) db.setSetting(k, v);++ db.setSetting('tts_enabled', '1');++ const ch = db.getChannelByUserId(ids.streamer);++ ids.channel = ch.id;++ ids.live = Number(db.createStream({ user_id: ids.streamer, channel_id: ch.id, title: 'Live now', protocol: 'webrtc' }).lastInsertRowid);++ ids.ended = Number(db.createStream({ user_id: ids.streamer, channel_id: ch.id, title: 'Earlier', protocol: 'webrtc' }).lastInsertRowid);++ db.endStream(ids.ended);++ db.run("UPDATE streams SET last_heartbeat = datetime('now') WHERE id = ?", [ids.live]);++ const ev = require('./server/events/stream-events');++ const events = [ev.envelopeFor('started', ids.live), ev.envelopeFor('ended', ids.ended),++ require('./server/events/release-events').envelopeFor({ head: 'a'.repeat(40), previous: 'b'.repeat(40), commits: [] })];++ db.close();++ console.log(JSON.stringify({ ids, events }));++`;++++function hits(text, needles) {++ return Object.entries(needles).filter(([, v]) => v && String(text).includes(v)).map(([k]) => k);++}++++(async () => {++ const tmp = crawl.tempEnv('secrets');++ const seeded = crawl.seed(tmp, SEED, { ...PROD_LIKE, ...ENV, NODE_ENV: 'test', SETTINGS_SEED: JSON.stringify(SETTINGS) });++ assert.ok(seeded && seeded.ids, 'seed printed its ids');++ const { ids, events } = seeded;++++ console.log('secrets: what Live sends out');++ await check('stream lifecycle and release event envelopes carry no secret (built with every secret set)', () => {++ assert.strictEqual(events.length, 3);++ assert.deepStrictEqual(hits(JSON.stringify(events), { ...ENV, ...SETTING_NEEDLES }), []);++ });++++ const keys = crawl.networkKeys(tmp);++ const srv = await crawl.boot(tmp, { ...PROD_LIKE, ...ENV, ...keys.env });++ const bearer = (id, username, role) => ({ Authorization: `Bearer ${keys.sign({ sub: String(9000 + id), id: 9000 + id, username, role })}` });++ const people = {++ anonymous: {},++ viewer: bearer(ids.viewer, 'secviewer', 'user'),++ streamer: bearer(ids.streamer, 'secstreamer', 'streamer'),++ 'global mod': bearer(ids.mod, 'secmod', 'global_mod'),++ admin: bearer(ids.admin, 'secadmin', 'admin'),++ 'site owner': bearer(ids.owner, 'secowner', 'admin'),++ };++++ console.log('secrets: the sentinels are live');++ await check('the site owner\'s settings page shows the stored keys (so a crawl that finds none means something)', async () => {++ const r = await srv.request('GET', '/api/admin/settings', { headers: people['site owner'] });++ assert.strictEqual(r.status, 200, r.text.slice(0, 200));++ assert.ok(r.text.includes(SETTINGS.gif_tenor_api_key), 'owner sees a stored key');++ });++ await check('an admin who is not the owner gets them redacted', async () => {++ const r = await srv.request('GET', '/api/admin/settings', { headers: people.admin });++ assert.strictEqual(r.status, 200, r.text.slice(0, 200));++ assert.ok(!r.text.includes(SETTINGS.gif_tenor_api_key));++ assert.match(r.text, /gif_tenor_api_key/);++ });++ await check('each signed-in person is who the crawl says (GET /api/auth/me)', async () => {++ for (const [who, headers] of Object.entries(people)) {++ if (who === 'anonymous') continue;++ const r = await srv.request('GET', '/api/auth/me', { headers });++ assert.strictEqual(r.status, 200, `${who}: ${r.text.slice(0, 200)}`);++ }++ });++++ // Real ids first, then nonsense for the error paths.++ const nonsense = ['-1', '99999999999', "'\"<x>", 'x'.repeat(300), '..%2F..%2Fetc%2Fpasswd'];++ const numeric = [ids.live, ids.streamer, ids.channel, ids.ended, 1, ...nonsense];++ const byName = {++ username: ['secstreamer', ...nonsense], user: ['secstreamer'], userId: [ids.streamer, ...nonsense],++ platform: ['twitch', 'youtube', 'kick', 'nope'], namespace: ['live.site_settings', 'nope'], name: ['whip', 'nope'],++ file: ['x.mp3'], filename: ['x.png', '..%2F..%2F.env'], slug: ['secstreamer'], anonId: ['anon1'], kind: ['user'],++ type: ['vod', 'clip'], metric: ['viewers', 'nope'], ip: ['127.0.0.1'],++ };++ const values = (n) => byName[n] || numeric;++ const paths = crawl.getPaths(srv, values, {++ query: 'q=secstreamer&provider=tenor&id=%27&limit=-1',++ extra: ['/', '/@secstreamer', `/stream/${ids.live}`, '/dashboard', '/admin', '/settings', '/search?q=x', '/sitemap.xml',++ '/llms.txt', '/robots.txt', '/metrics', '/api/ready', '/api/health', '/release.json', '/api/nope', '/nope/nope',++ '/.env', '/api/%', '/api/chat/gif/trending?provider=tenor', '/api/chat/gif/search?provider=giphy&q=cats'],++ });++ const envNeedles = ENV;++ const allNeedles = { ...ENV, ...SETTING_NEEDLES };++++ console.log(`secrets: ${paths.length} GET paths from ${srv.routes.length} routes, as ${Object.keys(people).length} people`);++ const logs = crawl.captureLogs();++ let result;++ let writes = [];++ try {++ result = await crawl.crawlAll(srv, paths, people, (who) => (who === 'site owner' ? envNeedles : allNeedles));++ // Error paths of writes: a drill refuses them all, and the refusal (like a body-parser error)++ // must not echo anything.++ for (const [who, headers] of Object.entries({ anonymous: {}, admin: people.admin })) {++ for (const r of srv.routes.filter((x) => x.methods.some((m) => ['post', 'put', 'patch', 'delete'].includes(m))).slice(0, 400)) {++ const method = r.methods.find((m) => ['post', 'put', 'patch', 'delete'].includes(m)).toUpperCase();++ for (const p of crawl.expand(r.path, values).slice(0, 1)) {++ const res = await srv.request(method, p, { headers: { ...headers, 'Content-Type': 'application/json' }, body: '{"broken": ', timeoutMs: 3000 });++ for (const l of crawl.leaks(res, allNeedles)) writes.push(`${who}: ${method} ${p} → ${res.status} carries ${l.label} in its ${l.where}`);++ }++ }++ }++ } finally { logs.stop(); }++ console.log(` (answers: ${JSON.stringify(result.statuses)}; held open past the timeout: ${result.timedOut.join(', ') || 'none'})`);++++ await check('the crawl reached the server (most requests answered)', () => {++ assert.ok(result.answered > paths.length * 5, `${result.answered} answers for ${paths.length * 6} requests`);++ });++ await check('no GET route, page, probe or error shows an environment secret to anyone, or a stored key to anyone but the site owner', () => {++ assert.deepStrictEqual(result.found, []);++ });++ await check('refused writes with broken bodies echo no secret', () => {++ assert.deepStrictEqual(writes, []);++ });++ await check('no log line written while serving those requests contains a secret', () => {++ const bad = logs.lines.filter((l) => hits(l, allNeedles).length).map((l) => `${hits(l, allNeedles).join(',')}: ${l.slice(0, 200)}`);++ assert.deepStrictEqual(bad, []);++ });++++ tmp.cleanup();++ if (failures) { console.log(`\n${failures} failure(s)`); process.exit(1); }++ console.log('\nsecurity-secrets: all checks passed');++ process.exit(0);++})().catch((e) => { console.error(e); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,227 @@++'use strict';++/**++ * Stream keys never reach anyone but their owner (roadmap WS-R task 5, the stream-key class).++ *++ * Twice an ingest key went out to anonymous callers (the public channel endpoint's++ * managed_streams[].stream_key, then the stream detail's managed_stream_key; see++ * public-serializers.test.js and security-redaction.test.js), enough to publish to someone's slot++ * over RTMP or WHIP. Those tests pin the endpoints that leaked. This one covers the class: it seeds++ * a streamer whose every credential is a sentinel (account key, a Live slot key, an OpenRe slot's++ * rotated Live key, a restream key and SRT passphrase, RobotStreamer / platform / PowerChat tokens,++ * AI keys, control key and API token hashes, a camera password hash), boots the real server as a++ * restore-drill instance (test/security-crawl.js), and requests EVERY GET route Express knows,++ * plus the SPA pages, as five other people: anonymous, a viewer, another streamer, a global mod and++ * an admin. No response body or header, and no log line written while serving them, may contain++ * any sentinel. Then the owner's own key endpoints must show them (so the sentinels are live and a++ * crawl that found nothing means something), and the payloads Live sends out about a stream (the++ * live.stream.started/ended events, the search document, the go-live SSE and the Network go-live++ * call) must not carry them either.++ *++ * node test/security-stream-keys.test.js++ */++const assert = require('assert');++const crawl = require('./security-crawl');++++let failures = 0;++async function check(name, fn) {++ try { await fn(); console.log(` ✓ ${name}`); } catch (err) { failures++; console.log(` ✗ ${name}\n ${String(err.stack || err.message).split('\n').slice(0, 12).join('\n ')}`); }++}++++// Every credential the owner has. Low-entropy, obviously fake; hex where Live expects hex keys.++const K = {++ account: 'a1'.repeat(16),++ slotMain: 'b2'.repeat(20),++ slotOpenre: 'c3'.repeat(20),++ restreamKey: 'restream-sentinel-key-e5e5e5e5',++ srtPassphrase: 'srt-sentinel-passphrase-e6e6',++ rsToken: 'robotstreamer-sentinel-token-f6f6',++ platformAccess: 'platform-sentinel-access-a7a7',++ platformRefresh: 'platform-sentinel-refresh-b8b8',++ powerchatAccess: 'powerchat-sentinel-access-c8c8',++ powerchatRefresh: 'powerchat-sentinel-refresh-d8d8',++ aiBotToken: 'aibot-sentinel-token-c9c9',++ byoKey: 'sentinel-not-a-secret-byo',++ controlKeyHash: 'e1'.repeat(32),++ apiTokenHash: 'f2'.repeat(32),++ cameraPasswordHash: 'camera-sentinel-hash-a3a3',++ // The other people's own account keys: none of them may see another's, and the crawl must not++ // echo a caller's own key on a page about someone else either.++ fan: 'd4'.repeat(16),++ peer: 'e5'.repeat(16),++ mod: 'f6'.repeat(16),++ admin: 'a7'.repeat(16),++};++++const SEED = `++ const db = require('./server/db/database');++ db.initDb();++ const K = JSON.parse(process.env.SENTINELS);++ const mk = (u, role, key) => {++ const id = db.createUser({ username: u, password_hash: 'x', display_name: u, email: u + '@example.test', stream_key: key }).lastInsertRowid;++ db.run('UPDATE users SET role = ? WHERE id = ?', [role, id]);++ db.ensureChannel(id);++ db.getDb().prepare("INSERT INTO linked_accounts (user_id, service, service_user_id, service_username) VALUES (?, 'network', ?, ?)").run(id, String(9000 + Number(id)), u);++ return Number(id);++ };++ const owner = mk('keyowner', 'streamer', K.account);++ const fan = mk('keyfan', 'user', K.fan);++ const peer = mk('keypeer', 'streamer', K.peer);++ const mod = mk('keymod', 'global_mod', K.mod);++ const admin = mk('keyadmin', 'admin', K.admin);++ const ch = db.getChannelByUserId(owner);++ const slot = Number(db.createManagedStream({ user_id: owner, channel_id: ch.id, slug: 'main', title: 'Main slot', protocol: 'rtmp', stream_key: K.slotMain }).lastInsertRowid);++ const slot2 = Number(db.createManagedStream({ user_id: owner, channel_id: ch.id, slug: 'relay', title: 'OpenRe slot', protocol: 'rtmp', stream_key: K.slotOpenre }).lastInsertRowid);++ const cols = db.all('PRAGMA table_info(managed_streams)').map((c) => c.name);++ if (cols.includes('ingest_authority')) db.run("UPDATE managed_streams SET ingest_authority = 'openre', openre_stream_id = 'std_sentinel' WHERE id = ?", [slot2]);++ const live = Number(db.createStream({ user_id: owner, channel_id: ch.id, managed_stream_id: slot, title: 'Live now', protocol: 'rtmp' }).lastInsertRowid);++ const ended = Number(db.createStream({ user_id: owner, channel_id: ch.id, managed_stream_id: slot2, title: 'Earlier', protocol: 'rtmp' }).lastInsertRowid);++ db.endStream(ended);++ db.run("UPDATE streams SET last_heartbeat = datetime('now'), viewer_count = 3 WHERE id = ?", [live]);++ db.createRestreamDestination(owner, { platform: 'custom', name: 'Mirror', server_url: 'srt://ingest.example.test:9000', stream_key: K.restreamKey, srt_passphrase: K.srtPassphrase, managed_stream_id: slot });++ const ins = (sql, args) => db.getDb().prepare(sql).run(...args);++ ins('INSERT INTO robotstreamer_integrations (user_id, enabled, token, robot_id, owner_id, stream_name, owner_name) VALUES (?, 1, ?, ?, ?, ?, ?)', [owner, K.rsToken, 'r1', 'o1', 'keyowner', 'keyowner']);++ ins("INSERT INTO platform_connections (user_id, platform, platform_user_id, platform_username, access_token, refresh_token) VALUES (?, 'twitch', 't1', 'keyowner', ?, ?)", [owner, K.platformAccess, K.platformRefresh]);++ ins('INSERT INTO powerchat_connections (user_id, powerchat_username, powerchat_user_id, access_token, refresh_token) VALUES (?, ?, ?, ?, ?)', [owner, 'keyowner', 'p1', K.powerchatAccess, K.powerchatRefresh]);++ ins('INSERT INTO ai_chatbot_configs (user_id, enabled, api_token) VALUES (?, 1, ?)', [owner, K.aiBotToken]);++ ins('INSERT INTO channel_ai_config (user_id, enabled, use_shared_key, byo_key) VALUES (?, 1, 0, ?)', [owner, K.byoKey]);++ ins('INSERT INTO api_keys (user_id, key_hash, label) VALUES (?, ?, ?)', [owner, K.controlKeyHash, 'Robot']);++ ins('INSERT INTO api_tokens (user_id, token_hash, label, scopes) VALUES (?, ?, ?, ?)', [owner, K.apiTokenHash, 'Bot', '["chat","read"]']);++ const cam = Number(ins('INSERT INTO camera_profiles (user_id, stream_id, name, onvif_url, username, password_hash) VALUES (?, ?, ?, ?, ?, ?)', [owner, live, 'Desk cam', 'http://camera.example.test', 'admin', K.cameraPasswordHash]).lastInsertRowid);++ db.run('INSERT INTO follows (follower_id, streamer_id) VALUES (?, ?)', [fan, owner]);++ db.run("INSERT INTO chat_messages (stream_id, user_id, username, message) VALUES (?, ?, 'keyfan', 'hello')", [live, fan]);++++ // What Live sends out about the stream, built from these rows the way production builds it.++ (async () => {++ const out = {};++ const ev = require('./server/events/stream-events');++ out.started = ev.envelopeFor('started', live);++ out.ended = ev.envelopeFor('ended', ended);++ try { out.searchDocument = require('./server/events/search-documents').documentFor(owner); } catch (e) { out.searchDocumentError = e.message; }++ const raw = db.get('SELECT s.*, ms.stream_key AS managed_stream_key, ms.slug AS managed_stream_slug FROM streams s LEFT JOIN managed_streams ms ON ms.id = s.managed_stream_id WHERE s.id = ?', [live]);++ const streamer = db.getUserById(owner);++ const writes = [];++ const le = require('./server/streaming/live-events');++ le.subscribe({ on() {} }, { writeHead() {}, write(s) { writes.push(String(s)); } });++ le.announceGoLive(raw, streamer);++ out.sse = writes.join('');++ const posted = [];++ global.fetch = async (url, opts = {}) => { posted.push(String(opts.body || '')); return { ok: true, status: 200, json: async () => ({}) }; };++ require('./server/streaming/golive-notify').notifyFollowersGoLive(streamer, raw, { force: true });++ await new Promise((r) => setTimeout(r, 300));++ out.goLive = posted;++ out.rawHadKey = raw.managed_stream_key === K.slotMain && streamer.stream_key === K.account;++ db.close();++ console.log(JSON.stringify({ ids: { owner, fan, peer, mod, admin, channel: ch.id, slot, slot2, live, ended, cam }, out }));++ process.exit(0);++ })().catch((e) => { console.error(e); process.exit(1); });++`;++++function hits(text, needles) {++ return Object.entries(needles).filter(([, v]) => v && String(text).includes(v)).map(([k]) => k);++}++++(async () => {++ const tmp = crawl.tempEnv('stream-keys');++ const seeded = crawl.seed(tmp, SEED, { SENTINELS: JSON.stringify(K), INTERNAL_API_KEY: 'internal-sentinel-for-golive', OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9' });++ assert.ok(seeded && seeded.ids, 'seed printed its ids');++ const { ids, out } = seeded;++++ console.log('stream keys: what Live sends out about a stream');++ await check('the seeded rows really carry the keys (raw stream row and user row)', () => {++ assert.strictEqual(out.rawHadKey, true);++ });++ await check('live.stream.started / live.stream.ended envelopes carry no credential', () => {++ assert.ok(out.started && out.ended, 'both envelopes built');++ assert.deepStrictEqual(hits(JSON.stringify([out.started, out.ended]), K), []);++ });++ await check('the channel\'s search document carries no credential', () => {++ assert.ok(out.searchDocument || out.searchDocumentError === undefined, out.searchDocumentError);++ assert.deepStrictEqual(hits(JSON.stringify(out.searchDocument || {}), K), []);++ });++ await check('the go-live SSE event (every open page hears it) carries no credential, even from a raw row with the key', () => {++ assert.match(out.sse, /stream-live/);++ assert.deepStrictEqual(hits(out.sse, K), []);++ });++ await check('the go-live call to Network carries no credential', () => {++ assert.ok(out.goLive.length >= 1, 'Network was called');++ assert.deepStrictEqual(hits(out.goLive.join('\n'), K), []);++ });++++ // ── The booted server ──++ const keys = crawl.networkKeys(tmp);++ const srv = await crawl.boot(tmp, { ...keys.env, NODE_ENV: 'production' });++ const token = (id, username, role) => keys.sign({ sub: String(9000 + id), id: 9000 + id, username, role });++ const owner = { Authorization: `Bearer ${token(ids.owner, 'keyowner', 'streamer')}` };++ const people = {++ anonymous: {},++ viewer: { Authorization: `Bearer ${token(ids.fan, 'keyfan', 'user')}` },++ 'another streamer': { Authorization: `Bearer ${token(ids.peer, 'keypeer', 'streamer')}` },++ 'global mod': { Authorization: `Bearer ${token(ids.mod, 'keymod', 'global_mod')}` },++ admin: { Authorization: `Bearer ${token(ids.admin, 'keyadmin', 'admin')}` },++ };++++ console.log('stream keys: the owner sees their own keys (the sentinels are live)');++ await check('GET /api/auth/stream-key shows the owner their account key', async () => {++ const r = await srv.request('GET', '/api/auth/stream-key', { headers: owner });++ assert.strictEqual(r.status, 200, r.text.slice(0, 200));++ assert.ok(r.text.includes(K.account), 'account key shown to its owner');++ });++ await check('GET /api/streams/managed shows the owner their slot key', async () => {++ const r = await srv.request('GET', '/api/streams/managed', { headers: owner });++ assert.strictEqual(r.status, 200, r.text.slice(0, 200));++ assert.ok(r.text.includes(K.slotMain), 'slot key shown to its owner');++ });++ await check('each person is signed in as themselves (their own account key comes back to them)', async () => {++ for (const [who, headers] of Object.entries(people)) {++ if (who === 'anonymous') continue;++ const r = await srv.request('GET', '/api/auth/stream-key', { headers });++ assert.strictEqual(r.status, 200, `${who}: ${r.text.slice(0, 200)}`);++ }++ });++++ // Parameter values: every seeded id and name, so each route is asked about the owner's things.++ const numeric = [ids.live, ids.slot, ids.owner, ids.channel, ids.ended, ids.slot2, ids.cam, 1];++ const byName = {++ username: ['keyowner'], user: ['keyowner'], userId: [ids.owner], channelId: [ids.channel],++ streamId: [ids.live, ids.ended], managedStreamId: [ids.slot, ids.slot2], slotIdOrSlug: ['main', ids.slot, 'relay'],++ ref: ['main', ids.slot, 'relay'], platform: ['twitch', 'youtube', 'kick', 'custom'], anonId: ['anon1'], num: [1],++ file: ['x.mp3'], filename: ['x.png'], slug: ['keyowner'], stat: ['wins'], type: ['vod', 'clip', 'stream'],++ namespace: ['live.site_settings'], kind: ['user'], idOrSlug: ['default'], metric: ['viewers'], name: ['whip'],++ ip: ['127.0.0.1'], commentId: [1],++ };++ const values = (n) => byName[n] || numeric;++ const paths = crawl.getPaths(srv, values, {++ query: `q=keyowner&username=keyowner&user_id=${ids.owner}&stream_id=${ids.live}&managed_stream_id=${ids.slot}`,++ // The pages (SPA fallback + server-rendered SEO), which the route list shows only as '*'.++ extra: ['/', '/@keyowner', '/@keyowner/main', '/@keyowner/relay', `/stream/${ids.live}`, `/recap/${ids.live}`, '/dashboard',++ '/broadcast', '/broadcast/main', '/search?q=keyowner', '/content', '/moments', '/sitemap.xml', '/llms.txt', '/robots.txt',++ `/overlay/chat/keyowner/${ids.slot}`, '/obs/chat/keyowner', `/popout/${ids.live}`, '/kiosk'],++ });++++ console.log(`stream keys: ${paths.length} GET paths from ${srv.routes.length} routes, as ${Object.keys(people).length} people who are not the owner`);++ const own = { viewer: 'fan', 'another streamer': 'peer', 'global mod': 'mod', admin: 'admin' };++ const logs = crawl.captureLogs();++ let result;++ try {++ // Everyone else's own key is a secret to each person too, except their own.++ result = await crawl.crawlAll(srv, paths, people, (who) => Object.fromEntries(Object.entries(K).filter(([k]) => k !== own[who])));++ } finally { logs.stop(); }++ const { found, answered } = result;++ console.log(` (answers: ${JSON.stringify(result.statuses)}; held open past the timeout: ${result.timedOut.join(', ') || 'none'})`);++++ await check('the crawl reached the server (most requests answered)', () => {++ assert.ok(answered > paths.length * 4, `${answered} answers for ${paths.length * 5} requests`);++ });++ await check('no GET route or page shows the owner\'s credentials (or anyone else\'s) to anonymous, a viewer, another streamer, a mod or an admin', () => {++ assert.deepStrictEqual(found, []);++ });++ await check('no log line written while serving those requests contains a credential', () => {++ const bad = logs.lines.filter((l) => hits(l, K).length).map((l) => `${hits(l, K).join(',')}: ${l.slice(0, 200)}`);++ assert.deepStrictEqual(bad, []);++ });++++ tmp.cleanup();++ if (failures) { console.log(`\n${failures} failure(s)`); process.exit(1); }++ console.log('\nsecurity-stream-keys: all checks passed');++ process.exit(0);++})().catch((e) => { console.error(e); process.exit(1); });
@@ -21,7 +21,8 @@ const config = require('openvibe-shared/config');
@@ -21,7 +21,8 @@ const config = require('openvibe-shared/config');
const db = require('../db/database');
const db = require('../db/database');
const NOT_CONFIG = /^(arena_backfill_cursor_|star_streamer$|deploy_last_announced$|relay_users_backfilled$|bucks_bits_migration_done$|stats_vibes_reset_at$|storage_tier\.|money_writes_frozen)/;
const NOT_CONFIG = /^(arena_backfill_cursor_|star_streamer$|deploy_last_announced$|relay_users_backfilled$|bucks_bits_migration_done$|stats_vibes_reset_at$|storage_tier\.|money_writes_frozen)/;
-const SECRET = /(api[_-]?key|secret|token|password|client_secret|service_account|private[_-]?key|salt|access_key)/i;
+// A webhook URL is its own credential (anyone holding it can post), so it is secret-class too.
+const SECRET = /(api[_-]?key|secret|token|password|client_secret|service_account|private[_-]?key|salt|access[_-]?key|webhook)/i;
const PUBLIC_KEYS = new Set(['site_name', 'site_description', 'motd', 'registration_open', 'require_email', 'nsfw_enabled']);
const PUBLIC_KEYS = new Set(['site_name', 'site_description', 'motd', 'registration_open', 'require_email', 'nsfw_enabled']);
const SYSTEM = { type: 'service', id: 'live' };
const SYSTEM = { type: 'service', id: 'live' };
@@ -63,8 +63,10 @@ function canModerateContentOwner(actor, contentOwner) {
@@ -63,8 +63,10 @@ function canModerateContentOwner(actor, contentOwner) {
return false;
return false;
}
}
-// Settings whose values are secrets or control money/AI spend — owner-only to view/edit.
-const SENSITIVE_KEY_RE = /(api[_-]?key|secret|token|password|client_id|client_secret|service_account|private[_-]?key)/i;
+// Settings whose values are secrets or control money/AI spend — owner-only to view/edit. A webhook URL
+// carries its own credential (anyone holding a Discord webhook URL can post as it), and an AWS access
+// key id is half of a key pair.
+const SENSITIVE_KEY_RE = /(api[_-]?key|secret|token|password|client_id|client_secret|service_account|private[_-]?key|access[_-]?key|webhook)/i;
// money_writes_frozen (the Billing-cutover freeze) is owner-only too; every admin can READ it at
// money_writes_frozen (the Billing-cutover freeze) is owner-only too; every admin can READ it at
// GET /api/admin/money.
// GET /api/admin/money.
const SENSITIVE_KEY_PREFIXES = ['ai_', 'stripe_', 'ccbill_', 'crypto_', 'tts_google_', 'money_'];
const SENSITIVE_KEY_PREFIXES = ['ai_', 'stripe_', 'ccbill_', 'crypto_', 'tts_google_', 'money_'];
new file mode 100644
new file mode 100644
@@ -0,0 +1,239 @@
@@ -0,0 +1,239 @@
+'use strict';
+/**
+ * The crawler behind test/security-stream-keys.test.js and test/security-secrets.test.js (roadmap
+ * WS-R task 5). Not a test itself (no .test.js): it boots the real server/index.js in the calling
+ * process as a restore-drill instance (LIVE_DRILL, server/drill.js: HTTP on one loopback port, reads
+ * only, no outbound connection, no job, no socket server) against a temp database the caller seeded,
+ * lists every GET route Express knows after boot, and requests each one.
+ *
+ * Listing the routes from the booted app (not from a hand-kept list) is the point: a route added
+ * next month is crawled without anyone remembering to add it here.
+ *
+ * Sign-in is real: the caller gets an RSA key pair whose public half is the "Network" key Live
+ * verifies (OV_NETWORK_PUBLIC_KEY), and signs user session tokens with the private half.
+ */
+const assert = require('assert');
+const crypto = require('crypto');
+const fs = require('fs');
+const http = require('http');
+const net = require('net');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const REPO = path.resolve(__dirname, '..');
+const ISSUER = 'https://network.crawl.test';
+
+/** A temp directory with the drill's DB_PATH and DATA_DIR (both must lie outside the checkout). */
+function tempEnv(name) {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), `live-${name}-`));
+ const DB_PATH = path.join(dir, 'db', 'live.db');
+ const DATA_DIR = path.join(dir, 'data');
+ fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
+ fs.mkdirSync(DATA_DIR, { recursive: true });
+ return { dir, DB_PATH, DATA_DIR, cleanup: () => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* */ } } };
+}
+
+/**
+ * Run `script` in a child Node process in normal mode (not a drill) against the temp database, the
+ * way production would have written the rows a drill later reads. Its stdout's last line, if JSON,
+ * is returned parsed.
+ */
+function seed(tmp, script, env = {}) {
+ const r = spawnSync(process.execPath, ['-e', script], {
+ cwd: REPO, encoding: 'utf8',
+ env: { ...process.env, DB_PATH: tmp.DB_PATH, DATA_DIR: tmp.DATA_DIR, NODE_ENV: 'test', ...env },
+ });
+ assert.strictEqual(r.status, 0, `seeding failed:\n${r.stdout}\n${r.stderr}`);
+ const last = r.stdout.trim().split('\n').pop() || '';
+ try { return JSON.parse(last); } catch { return null; }
+}
+
+/** The "Network" signing key: its public half is what Live verifies session tokens with. */
+function networkKeys(tmp) {
+ const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 });
+ const publicKeyPath = path.join(tmp.dir, 'network-public.pem');
+ fs.writeFileSync(publicKeyPath, publicKey.export({ type: 'spki', format: 'pem' }));
+ const jwt = require('jsonwebtoken');
+ const sign = (claims) => jwt.sign({ ...claims }, privateKey, { algorithm: 'RS256', issuer: ISSUER, expiresIn: '1h' });
+ return { publicKeyPath, sign, env: { OV_NETWORK_PUBLIC_KEY: publicKeyPath, OV_NETWORK_URL: ISSUER } };
+}
+
+function freePort() {
+ return new Promise((resolve) => { const s = net.createServer().listen(0, '127.0.0.1', () => { const p = s.address().port; s.close(() => resolve(p)); }); });
+}
+
+/** The path an Express 4 layer is mounted at ('' for app-level middleware), or null if it is a pattern we cannot turn back into a path. */
+function mountPath(layer) {
+ if (!layer.regexp || layer.regexp.fast_slash) return '';
+ let src = layer.regexp.source.replace(/^\^/, '').replace(/\\\/\?\(\?=\\\/\|\$\)$/i, '').replace(/\/\?\(\?=\/\|\$\)$/i, '');
+ let i = 0;
+ src = src.replace(/\(\?:\(\[\^\\\/\]\+\?\)\)/g, () => `:${(layer.keys[i++] || {}).name || 'param'}`);
+ src = src.replace(/\\\//g, '/').replace(/\\\./g, '.').replace(/\\-/g, '-');
+ return /[\\^$()|[\]*+?]/.test(src) ? null : src;
+}
+
+/** Every route of an Express app: [{ path, methods }] with `path` a string template (':param'). */
+function listRoutes(app) {
+ const out = [];
+ const walk = (stack, prefix) => {
+ for (const layer of stack) {
+ if (layer.route) {
+ const methods = Object.keys(layer.route.methods).filter((m) => layer.route.methods[m]);
+ for (const p of [].concat(layer.route.path)) {
+ if (typeof p === 'string') out.push({ path: prefix + p, methods });
+ }
+ } else if (layer.handle && Array.isArray(layer.handle.stack)) {
+ const mp = mountPath(layer);
+ if (mp !== null) walk(layer.handle.stack, prefix + mp);
+ } else {
+ // A plain middleware mounted on a path (e.g. the "moved" answers): it answers everything under it.
+ const mp = mountPath(layer);
+ if (mp) out.push({ path: prefix + mp, methods: ['_all'] });
+ }
+ }
+ };
+ walk(app._router.stack, '');
+ const seen = new Set();
+ return out.filter((r) => { const k = `${r.methods.join(',')} ${r.path}`; if (seen.has(k)) return false; seen.add(k); return true; });
+}
+
+/**
+ * Concrete paths for a route template. `values(name)` gives the candidate values of a parameter;
+ * a template yields one path per candidate position (candidate i for every parameter), so a route
+ * with an unknown parameter is tried with each seeded id without multiplying out.
+ */
+function expand(template, values) {
+ const names = [];
+ const t = `/${template.replace(/^\/+/, '')}`.replace(/\*/g, 'x').replace(/:([A-Za-z0-9_]+)\??(\([^)]*\))?/g, (m, n) => { names.push(n); return `:${n}`; });
+ if (!names.length) return [t];
+ const lists = names.map((n) => values(n));
+ const width = Math.max(...lists.map((l) => l.length));
+ const paths = new Set();
+ for (let i = 0; i < width; i++) {
+ let j = 0;
+ paths.add(t.replace(/:([A-Za-z0-9_]+)/g, () => { const l = lists[j++]; return encodeURIComponent(String(l[Math.min(i, l.length - 1)])); }));
+ }
+ return [...paths];
+}
+
+/**
+ * Boot server/index.js in this process as a drill instance on a free loopback port. `env` is added
+ * to process.env first (the tests put their sentinel secrets there). Resolves with the app, its
+ * routes, and request(method, path, { headers, timeoutMs }).
+ */
+async function boot(tmp, env = {}) {
+ const port = await freePort();
+ Object.assign(process.env, env, { LIVE_DRILL: '1', DB_PATH: tmp.DB_PATH, DATA_DIR: tmp.DATA_DIR, HOST: '127.0.0.1', PORT: String(port) });
+ delete process.env.LISTEN_FDS;
+ let app = null;
+ const realCreate = http.createServer;
+ http.createServer = function (...a) { const h = a.find((x) => typeof x === 'function'); if (h && typeof h.handle === 'function' && typeof h.set === 'function') app = h; return realCreate.apply(this, a); };
+ delete require.cache[require.resolve('../server/drill')];
+ require('../server/index.js');
+ http.createServer = realCreate;
+ assert.ok(app, 'server/index.js created its HTTP server from the Express app');
+ const base = `http://127.0.0.1:${port}`;
+ const request = (method, p, { headers = {}, timeoutMs = 4000, body = null } = {}) => new Promise((resolve) => {
+ let done = false;
+ const finish = (r) => { if (!done) { done = true; resolve(r); } };
+ let req;
+ try { new URL(base + p); } catch (e) { finish({ status: 0, headers: {}, text: '', error: `bad path ${p}` }); return; }
+ req = http.request(base + p, { method, headers: { ...(body ? { 'Content-Type': 'application/json' } : {}), ...headers } }, (res) => {
+ let text = '';
+ const timer = setTimeout(() => { res.destroy(); finish({ status: res.statusCode, headers: res.headers, text, timedOut: true }); }, timeoutMs);
+ res.setEncoding('utf8');
+ res.on('data', (c) => { if (text.length < 4 * 1024 * 1024) text += c; });
+ res.on('end', () => { clearTimeout(timer); finish({ status: res.statusCode, headers: res.headers, text }); });
+ res.on('error', () => { clearTimeout(timer); finish({ status: res.statusCode, headers: res.headers, text }); });
+ });
+ req.setTimeout(timeoutMs, () => { req.destroy(); finish({ status: 0, headers: {}, text: '', timedOut: true }); });
+ req.on('error', (e) => finish({ status: 0, headers: {}, text: '', error: e.message }));
+ if (body) req.write(typeof body === 'string' ? body : JSON.stringify(body));
+ req.end();
+ });
+ let ready = null;
+ for (let i = 0; i < 200 && !(ready && ready.status === 200); i++) {
+ ready = await request('GET', '/api/ready', { timeoutMs: 1000 });
+ if (ready.status !== 200) await new Promise((r) => setTimeout(r, 100));
+ }
+ assert.strictEqual(ready && ready.status, 200, `drill instance not ready: ${ready && ready.text}`);
+ return { app, port, base, request, routes: listRoutes(app) };
+}
+
+/** Capture console output (the logs a public request writes) until stop() is called. */
+function captureLogs() {
+ const lines = [];
+ const orig = {};
+ for (const m of ['log', 'info', 'warn', 'error', 'debug']) {
+ orig[m] = console[m];
+ console[m] = (...a) => { lines.push(a.map((x) => (typeof x === 'string' ? x : (() => { try { return x instanceof Error ? `${x.message}\n${x.stack}` : JSON.stringify(x); } catch { return String(x); } })())).join(' ')); };
+ }
+ const wOut = process.stdout.write.bind(process.stdout);
+ const wErr = process.stderr.write.bind(process.stderr);
+ process.stdout.write = (c, ...a) => { lines.push(String(c)); return true; };
+ process.stderr.write = (c, ...a) => { lines.push(String(c)); return true; };
+ return {
+ lines,
+ stop() { Object.assign(console, orig); process.stdout.write = wOut; process.stderr.write = wErr; return lines; },
+ };
+}
+
+/** Which of `needles` ({ label: value }) occur in a response's body or headers: [{ label, where }]. */
+function leaks(res, needles) {
+ const found = [];
+ const headerText = Object.entries(res.headers || {}).map(([k, v]) => `${k}: ${[].concat(v).join(', ')}`).join('\n');
+ for (const [label, value] of Object.entries(needles)) {
+ if (!value) continue;
+ if (res.text && res.text.includes(value)) found.push({ label, where: 'body' });
+ if (headerText.includes(value)) found.push({ label, where: 'headers' });
+ }
+ return found;
+}
+
+/**
+ * Every GET path to crawl: each GET route (and path-mounted handler) expanded with `values(name)`,
+ * each also with `query` appended when given, plus `extra` (pages the route list shows only as '*').
+ */
+function getPaths(srv, values, { query = '', extra = [] } = {}) {
+ const paths = new Set();
+ for (const r of srv.routes) {
+ if (!r.methods.includes('get') && !r.methods.includes('_all')) continue;
+ for (const p of expand(r.path, values)) {
+ paths.add(p);
+ if (query && !p.includes('?')) paths.add(`${p}?${query}`);
+ }
+ }
+ for (const p of extra) paths.add(p);
+ return [...paths];
+}
+
+/**
+ * GET every path as every person, a few requests at a time. `needlesFor(who)` names the values that
+ * person must never see. Resolves { found: ['who: GET path → status carries label in its where'],
+ * answered, timedOut: [paths], statuses: { '2xx': n, ... } }.
+ */
+async function crawlAll(srv, paths, people, needlesFor, { concurrency = 6, timeoutMs = 3000 } = {}) {
+ const found = [];
+ const timedOut = new Set();
+ const statuses = {};
+ let answered = 0;
+ for (const [who, headers] of Object.entries(people)) {
+ const needles = needlesFor(who);
+ const queue = [...paths];
+ await Promise.all(Array.from({ length: concurrency }, async () => {
+ while (queue.length) {
+ const p = queue.shift();
+ const r = await srv.request('GET', p, { headers, timeoutMs });
+ if (r.status) answered++;
+ if (r.timedOut) timedOut.add(p);
+ const cls = r.status ? `${String(r.status)[0]}xx` : 'none';
+ statuses[cls] = (statuses[cls] || 0) + 1;
+ for (const l of leaks(r, needles)) found.push(`${who}: GET ${p} → ${r.status} carries ${l.label} in its ${l.where}`);
+ }
+ }));
+ }
+ return { found, answered, timedOut: [...timedOut], statuses };
+}
+
+module.exports = { REPO, ISSUER, tempEnv, seed, networkKeys, boot, listRoutes, expand, getPaths, crawlAll, captureLogs, leaks };
new file mode 100644
new file mode 100644
@@ -0,0 +1,210 @@
@@ -0,0 +1,210 @@
+'use strict';
+/**
+ * Live's own secrets never leave in a response (roadmap WS-R task 5, the internal-secret class).
+ *
+ * Boots the real server as a restore-drill instance (test/security-crawl.js) with a sentinel value in
+ * every secret environment variable Live reads (the Network OAuth client secret, the internal API
+ * keys, Media's key and webhook/event secrets, the Events and OpenRe subscription secrets, TURN,
+ * PayPal, the admin password, GitHub tokens, the ops webhook) and in every secret site setting an
+ * owner can store (GIF, TTS, AI, Twitch/Kick/YouTube, soundboard keys, the Discord and ops
+ * webhooks), then requests every GET route Express knows, the pages, the probes (/api/ready,
+ * /api/health, /metrics, /release.json) and the error paths (nonsense ids, unknown paths, refused
+ * writes with broken bodies) as anonymous, a viewer, a streamer, a global mod, an admin and the
+ * site owner. No body or header may contain an environment secret, whoever asks; the stored
+ * settings are shown to the site owner only (the settings page is where they are edited), never
+ * to an admin who is not the owner. The events Live sends (stream lifecycle, release) and the log
+ * lines written while serving all of that may not contain them either.
+ *
+ * node test/security-secrets.test.js
+ */
+const assert = require('assert');
+const crawl = require('./security-crawl');
+
+let failures = 0;
+async function check(name, fn) {
+ try { await fn(); console.log(` ✓ ${name}`); } catch (err) { failures++; console.log(` ✗ ${name}\n ${String(err.stack || err.message).split('\n').slice(0, 14).join('\n ')}`); }
+}
+
+// Environment secrets: obviously fake, low entropy.
+const ENV = {
+ JWT_SECRET: 'sentinel-not-a-secret-jwt',
+ OV_OAUTH_CLIENT_SECRET: 'sentinel-not-a-secret-oauth-client',
+ INTERNAL_API_KEY: 'sentinel-not-a-secret-internal-api-key',
+ OV_INTERNAL_KEY: 'sentinel-not-a-secret-ov-internal-key',
+ MEDIA_API_KEY: 'sentinel-not-a-secret-media-api-key',
+ MEDIA_WEBHOOK_SECRET: 'sentinel-not-a-secret-media-webhook',
+ MEDIA_EVENTS_SECRET: 'sentinel-not-a-secret-media-events',
+ LIVE_EVENTS_SECRET: 'sentinel-not-a-secret-live-events',
+ OPENRE_EVENTS_SECRET: 'sentinel-not-a-secret-openre-events',
+ TURN_AUTH_SECRET: 'sentinel-not-a-secret-turn-auth',
+ PAYPAL_CLIENT_SECRET: 'sentinel-not-a-secret-paypal',
+ ADMIN_PASSWORD: 'sentinel-not-a-secret-admin-password',
+ GITHUB_TOKEN: 'sentinel-not-a-secret-github-token',
+ GH_TOKEN: 'sentinel-not-a-secret-gh-token',
+ OPS_ALERT_WEBHOOK_URL: 'https://hooks.example.test/sentinel-not-a-secret-ops-webhook',
+};
+// What makes the process production-like (the drill refuses every connection to these anyway).
+const PROD_LIKE = {
+ NODE_ENV: 'production',
+ EVENTS_URL: 'http://127.0.0.1:9',
+ MEDIA_URL: 'http://127.0.0.1:9',
+ OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',
+ OV_COMMUNITY_INTERNAL_URL: 'http://127.0.0.1:9',
+ OV_CHAT_INTERNAL_URL: 'http://127.0.0.1:9',
+ OV_OAUTH_CLIENT_ID: 'openvibe-live',
+ PAYPAL_CLIENT_ID: 'paypal-client-id-public',
+};
+// Secret site settings (live.site_settings), stored the way the owner's settings page stores them.
+const SETTINGS = {
+ gif_giphy_api_key: 'sentinel-not-a-secret-giphy',
+ gif_tenor_api_key: 'sentinel-not-a-secret-tenor',
+ tts_aws_access_key_id: 'sentinel-not-a-secret-aws-id',
+ tts_aws_secret_access_key: 'sentinel-not-a-secret-aws-key',
+ tts_google_api_key: 'sentinel-not-a-secret-google-tts',
+ tts_google_service_account: '{"private_key":"sentinel-not-a-secret-google-sa"}',
+ twitch_client_secret: 'sentinel-not-a-secret-twitch',
+ kick_client_secret: 'sentinel-not-a-secret-kick',
+ youtube_api_key: 'sentinel-not-a-secret-youtube',
+ soundboard_101_api_key: 'sentinel-not-a-secret-soundboard',
+ ai_api_key: 'sentinel-not-a-secret-ai',
+ discord_webhook_url: 'https://discord.example.test/api/webhooks/1/sentinel-not-a-secret-discord',
+ ops_alert_webhook_url: 'https://hooks.example.test/sentinel-not-a-secret-ops-setting',
+};
+const SETTING_NEEDLES = Object.fromEntries(Object.entries(SETTINGS).map(([k, v]) => [k, v.includes('{') ? 'sentinel-not-a-secret-google-sa' : v]));
+
+const SEED = `
+ const db = require('./server/db/database');
+ db.initDb();
+ const S = JSON.parse(process.env.SETTINGS_SEED);
+ const mk = (u, role, owner) => {
+ const id = Number(db.createUser({ username: u, password_hash: 'x', display_name: u, email: u + '@example.test', stream_key: null }).lastInsertRowid);
+ db.run('UPDATE users SET role = ?, is_owner = ? WHERE id = ?', [role, owner ? 1 : 0, id]);
+ db.ensureChannel(id);
+ db.getDb().prepare("INSERT INTO linked_accounts (user_id, service, service_user_id, service_username) VALUES (?, 'network', ?, ?)").run(id, String(9000 + id), u);
+ return id;
+ };
+ const ids = {
+ streamer: mk('secstreamer', 'streamer'), viewer: mk('secviewer', 'user'), mod: mk('secmod', 'global_mod'),
+ admin: mk('secadmin', 'admin'), owner: mk('secowner', 'admin', true),
+ };
+ for (const [k, v] of Object.entries(S)) db.setSetting(k, v);
+ db.setSetting('tts_enabled', '1');
+ const ch = db.getChannelByUserId(ids.streamer);
+ ids.channel = ch.id;
+ ids.live = Number(db.createStream({ user_id: ids.streamer, channel_id: ch.id, title: 'Live now', protocol: 'webrtc' }).lastInsertRowid);
+ ids.ended = Number(db.createStream({ user_id: ids.streamer, channel_id: ch.id, title: 'Earlier', protocol: 'webrtc' }).lastInsertRowid);
+ db.endStream(ids.ended);
+ db.run("UPDATE streams SET last_heartbeat = datetime('now') WHERE id = ?", [ids.live]);
+ const ev = require('./server/events/stream-events');
+ const events = [ev.envelopeFor('started', ids.live), ev.envelopeFor('ended', ids.ended),
+ require('./server/events/release-events').envelopeFor({ head: 'a'.repeat(40), previous: 'b'.repeat(40), commits: [] })];
+ db.close();
+ console.log(JSON.stringify({ ids, events }));
+`;
+
+function hits(text, needles) {
+ return Object.entries(needles).filter(([, v]) => v && String(text).includes(v)).map(([k]) => k);
+}
+
+(async () => {
+ const tmp = crawl.tempEnv('secrets');
+ const seeded = crawl.seed(tmp, SEED, { ...PROD_LIKE, ...ENV, NODE_ENV: 'test', SETTINGS_SEED: JSON.stringify(SETTINGS) });
+ assert.ok(seeded && seeded.ids, 'seed printed its ids');
+ const { ids, events } = seeded;
+
+ console.log('secrets: what Live sends out');
+ await check('stream lifecycle and release event envelopes carry no secret (built with every secret set)', () => {
+ assert.strictEqual(events.length, 3);
+ assert.deepStrictEqual(hits(JSON.stringify(events), { ...ENV, ...SETTING_NEEDLES }), []);
+ });
+
+ const keys = crawl.networkKeys(tmp);
+ const srv = await crawl.boot(tmp, { ...PROD_LIKE, ...ENV, ...keys.env });
+ const bearer = (id, username, role) => ({ Authorization: `Bearer ${keys.sign({ sub: String(9000 + id), id: 9000 + id, username, role })}` });
+ const people = {
+ anonymous: {},
+ viewer: bearer(ids.viewer, 'secviewer', 'user'),
+ streamer: bearer(ids.streamer, 'secstreamer', 'streamer'),
+ 'global mod': bearer(ids.mod, 'secmod', 'global_mod'),
+ admin: bearer(ids.admin, 'secadmin', 'admin'),
+ 'site owner': bearer(ids.owner, 'secowner', 'admin'),
+ };
+
+ console.log('secrets: the sentinels are live');
+ await check('the site owner\'s settings page shows the stored keys (so a crawl that finds none means something)', async () => {
+ const r = await srv.request('GET', '/api/admin/settings', { headers: people['site owner'] });
+ assert.strictEqual(r.status, 200, r.text.slice(0, 200));
+ assert.ok(r.text.includes(SETTINGS.gif_tenor_api_key), 'owner sees a stored key');
+ });
+ await check('an admin who is not the owner gets them redacted', async () => {
+ const r = await srv.request('GET', '/api/admin/settings', { headers: people.admin });
+ assert.strictEqual(r.status, 200, r.text.slice(0, 200));
+ assert.ok(!r.text.includes(SETTINGS.gif_tenor_api_key));
+ assert.match(r.text, /gif_tenor_api_key/);
+ });
+ await check('each signed-in person is who the crawl says (GET /api/auth/me)', async () => {
+ for (const [who, headers] of Object.entries(people)) {
+ if (who === 'anonymous') continue;
+ const r = await srv.request('GET', '/api/auth/me', { headers });
+ assert.strictEqual(r.status, 200, `${who}: ${r.text.slice(0, 200)}`);
+ }
+ });
+
+ // Real ids first, then nonsense for the error paths.
+ const nonsense = ['-1', '99999999999', "'\"<x>", 'x'.repeat(300), '..%2F..%2Fetc%2Fpasswd'];
+ const numeric = [ids.live, ids.streamer, ids.channel, ids.ended, 1, ...nonsense];
+ const byName = {
+ username: ['secstreamer', ...nonsense], user: ['secstreamer'], userId: [ids.streamer, ...nonsense],
+ platform: ['twitch', 'youtube', 'kick', 'nope'], namespace: ['live.site_settings', 'nope'], name: ['whip', 'nope'],
+ file: ['x.mp3'], filename: ['x.png', '..%2F..%2F.env'], slug: ['secstreamer'], anonId: ['anon1'], kind: ['user'],
+ type: ['vod', 'clip'], metric: ['viewers', 'nope'], ip: ['127.0.0.1'],
+ };
+ const values = (n) => byName[n] || numeric;
+ const paths = crawl.getPaths(srv, values, {
+ query: 'q=secstreamer&provider=tenor&id=%27&limit=-1',
+ extra: ['/', '/@secstreamer', `/stream/${ids.live}`, '/dashboard', '/admin', '/settings', '/search?q=x', '/sitemap.xml',
+ '/llms.txt', '/robots.txt', '/metrics', '/api/ready', '/api/health', '/release.json', '/api/nope', '/nope/nope',
+ '/.env', '/api/%', '/api/chat/gif/trending?provider=tenor', '/api/chat/gif/search?provider=giphy&q=cats'],
+ });
+ const envNeedles = ENV;
+ const allNeedles = { ...ENV, ...SETTING_NEEDLES };
+
+ console.log(`secrets: ${paths.length} GET paths from ${srv.routes.length} routes, as ${Object.keys(people).length} people`);
+ const logs = crawl.captureLogs();
+ let result;
+ let writes = [];
+ try {
+ result = await crawl.crawlAll(srv, paths, people, (who) => (who === 'site owner' ? envNeedles : allNeedles));
+ // Error paths of writes: a drill refuses them all, and the refusal (like a body-parser error)
+ // must not echo anything.
+ for (const [who, headers] of Object.entries({ anonymous: {}, admin: people.admin })) {
+ for (const r of srv.routes.filter((x) => x.methods.some((m) => ['post', 'put', 'patch', 'delete'].includes(m))).slice(0, 400)) {
+ const method = r.methods.find((m) => ['post', 'put', 'patch', 'delete'].includes(m)).toUpperCase();
+ for (const p of crawl.expand(r.path, values).slice(0, 1)) {
+ const res = await srv.request(method, p, { headers: { ...headers, 'Content-Type': 'application/json' }, body: '{"broken": ', timeoutMs: 3000 });
+ for (const l of crawl.leaks(res, allNeedles)) writes.push(`${who}: ${method} ${p} → ${res.status} carries ${l.label} in its ${l.where}`);
+ }
+ }
+ }
+ } finally { logs.stop(); }
+ console.log(` (answers: ${JSON.stringify(result.statuses)}; held open past the timeout: ${result.timedOut.join(', ') || 'none'})`);
+
+ await check('the crawl reached the server (most requests answered)', () => {
+ assert.ok(result.answered > paths.length * 5, `${result.answered} answers for ${paths.length * 6} requests`);
+ });
+ await check('no GET route, page, probe or error shows an environment secret to anyone, or a stored key to anyone but the site owner', () => {
+ assert.deepStrictEqual(result.found, []);
+ });
+ await check('refused writes with broken bodies echo no secret', () => {
+ assert.deepStrictEqual(writes, []);
+ });
+ await check('no log line written while serving those requests contains a secret', () => {
+ const bad = logs.lines.filter((l) => hits(l, allNeedles).length).map((l) => `${hits(l, allNeedles).join(',')}: ${l.slice(0, 200)}`);
+ assert.deepStrictEqual(bad, []);
+ });
+
+ tmp.cleanup();
+ if (failures) { console.log(`\n${failures} failure(s)`); process.exit(1); }
+ console.log('\nsecurity-secrets: all checks passed');
+ process.exit(0);
+})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,227 @@
@@ -0,0 +1,227 @@
+'use strict';
+/**
+ * Stream keys never reach anyone but their owner (roadmap WS-R task 5, the stream-key class).
+ *
+ * Twice an ingest key went out to anonymous callers (the public channel endpoint's
+ * managed_streams[].stream_key, then the stream detail's managed_stream_key; see
+ * public-serializers.test.js and security-redaction.test.js), enough to publish to someone's slot
+ * over RTMP or WHIP. Those tests pin the endpoints that leaked. This one covers the class: it seeds
+ * a streamer whose every credential is a sentinel (account key, a Live slot key, an OpenRe slot's
+ * rotated Live key, a restream key and SRT passphrase, RobotStreamer / platform / PowerChat tokens,
+ * AI keys, control key and API token hashes, a camera password hash), boots the real server as a
+ * restore-drill instance (test/security-crawl.js), and requests EVERY GET route Express knows,
+ * plus the SPA pages, as five other people: anonymous, a viewer, another streamer, a global mod and
+ * an admin. No response body or header, and no log line written while serving them, may contain
+ * any sentinel. Then the owner's own key endpoints must show them (so the sentinels are live and a
+ * crawl that found nothing means something), and the payloads Live sends out about a stream (the
+ * live.stream.started/ended events, the search document, the go-live SSE and the Network go-live
+ * call) must not carry them either.
+ *
+ * node test/security-stream-keys.test.js
+ */
+const assert = require('assert');
+const crawl = require('./security-crawl');
+
+let failures = 0;
+async function check(name, fn) {
+ try { await fn(); console.log(` ✓ ${name}`); } catch (err) { failures++; console.log(` ✗ ${name}\n ${String(err.stack || err.message).split('\n').slice(0, 12).join('\n ')}`); }
+}
+
+// Every credential the owner has. Low-entropy, obviously fake; hex where Live expects hex keys.
+const K = {
+ account: 'a1'.repeat(16),
+ slotMain: 'b2'.repeat(20),
+ slotOpenre: 'c3'.repeat(20),
+ restreamKey: 'restream-sentinel-key-e5e5e5e5',
+ srtPassphrase: 'srt-sentinel-passphrase-e6e6',
+ rsToken: 'robotstreamer-sentinel-token-f6f6',
+ platformAccess: 'platform-sentinel-access-a7a7',
+ platformRefresh: 'platform-sentinel-refresh-b8b8',
+ powerchatAccess: 'powerchat-sentinel-access-c8c8',
+ powerchatRefresh: 'powerchat-sentinel-refresh-d8d8',
+ aiBotToken: 'aibot-sentinel-token-c9c9',
+ byoKey: 'sentinel-not-a-secret-byo',
+ controlKeyHash: 'e1'.repeat(32),
+ apiTokenHash: 'f2'.repeat(32),
+ cameraPasswordHash: 'camera-sentinel-hash-a3a3',
+ // The other people's own account keys: none of them may see another's, and the crawl must not
+ // echo a caller's own key on a page about someone else either.
+ fan: 'd4'.repeat(16),
+ peer: 'e5'.repeat(16),
+ mod: 'f6'.repeat(16),
+ admin: 'a7'.repeat(16),
+};
+
+const SEED = `
+ const db = require('./server/db/database');
+ db.initDb();
+ const K = JSON.parse(process.env.SENTINELS);
+ const mk = (u, role, key) => {
+ const id = db.createUser({ username: u, password_hash: 'x', display_name: u, email: u + '@example.test', stream_key: key }).lastInsertRowid;
+ db.run('UPDATE users SET role = ? WHERE id = ?', [role, id]);
+ db.ensureChannel(id);
+ db.getDb().prepare("INSERT INTO linked_accounts (user_id, service, service_user_id, service_username) VALUES (?, 'network', ?, ?)").run(id, String(9000 + Number(id)), u);
+ return Number(id);
+ };
+ const owner = mk('keyowner', 'streamer', K.account);
+ const fan = mk('keyfan', 'user', K.fan);
+ const peer = mk('keypeer', 'streamer', K.peer);
+ const mod = mk('keymod', 'global_mod', K.mod);
+ const admin = mk('keyadmin', 'admin', K.admin);
+ const ch = db.getChannelByUserId(owner);
+ const slot = Number(db.createManagedStream({ user_id: owner, channel_id: ch.id, slug: 'main', title: 'Main slot', protocol: 'rtmp', stream_key: K.slotMain }).lastInsertRowid);
+ const slot2 = Number(db.createManagedStream({ user_id: owner, channel_id: ch.id, slug: 'relay', title: 'OpenRe slot', protocol: 'rtmp', stream_key: K.slotOpenre }).lastInsertRowid);
+ const cols = db.all('PRAGMA table_info(managed_streams)').map((c) => c.name);
+ if (cols.includes('ingest_authority')) db.run("UPDATE managed_streams SET ingest_authority = 'openre', openre_stream_id = 'std_sentinel' WHERE id = ?", [slot2]);
+ const live = Number(db.createStream({ user_id: owner, channel_id: ch.id, managed_stream_id: slot, title: 'Live now', protocol: 'rtmp' }).lastInsertRowid);
+ const ended = Number(db.createStream({ user_id: owner, channel_id: ch.id, managed_stream_id: slot2, title: 'Earlier', protocol: 'rtmp' }).lastInsertRowid);
+ db.endStream(ended);
+ db.run("UPDATE streams SET last_heartbeat = datetime('now'), viewer_count = 3 WHERE id = ?", [live]);
+ db.createRestreamDestination(owner, { platform: 'custom', name: 'Mirror', server_url: 'srt://ingest.example.test:9000', stream_key: K.restreamKey, srt_passphrase: K.srtPassphrase, managed_stream_id: slot });
+ const ins = (sql, args) => db.getDb().prepare(sql).run(...args);
+ ins('INSERT INTO robotstreamer_integrations (user_id, enabled, token, robot_id, owner_id, stream_name, owner_name) VALUES (?, 1, ?, ?, ?, ?, ?)', [owner, K.rsToken, 'r1', 'o1', 'keyowner', 'keyowner']);
+ ins("INSERT INTO platform_connections (user_id, platform, platform_user_id, platform_username, access_token, refresh_token) VALUES (?, 'twitch', 't1', 'keyowner', ?, ?)", [owner, K.platformAccess, K.platformRefresh]);
+ ins('INSERT INTO powerchat_connections (user_id, powerchat_username, powerchat_user_id, access_token, refresh_token) VALUES (?, ?, ?, ?, ?)', [owner, 'keyowner', 'p1', K.powerchatAccess, K.powerchatRefresh]);
+ ins('INSERT INTO ai_chatbot_configs (user_id, enabled, api_token) VALUES (?, 1, ?)', [owner, K.aiBotToken]);
+ ins('INSERT INTO channel_ai_config (user_id, enabled, use_shared_key, byo_key) VALUES (?, 1, 0, ?)', [owner, K.byoKey]);
+ ins('INSERT INTO api_keys (user_id, key_hash, label) VALUES (?, ?, ?)', [owner, K.controlKeyHash, 'Robot']);
+ ins('INSERT INTO api_tokens (user_id, token_hash, label, scopes) VALUES (?, ?, ?, ?)', [owner, K.apiTokenHash, 'Bot', '["chat","read"]']);
+ const cam = Number(ins('INSERT INTO camera_profiles (user_id, stream_id, name, onvif_url, username, password_hash) VALUES (?, ?, ?, ?, ?, ?)', [owner, live, 'Desk cam', 'http://camera.example.test', 'admin', K.cameraPasswordHash]).lastInsertRowid);
+ db.run('INSERT INTO follows (follower_id, streamer_id) VALUES (?, ?)', [fan, owner]);
+ db.run("INSERT INTO chat_messages (stream_id, user_id, username, message) VALUES (?, ?, 'keyfan', 'hello')", [live, fan]);
+
+ // What Live sends out about the stream, built from these rows the way production builds it.
+ (async () => {
+ const out = {};
+ const ev = require('./server/events/stream-events');
+ out.started = ev.envelopeFor('started', live);
+ out.ended = ev.envelopeFor('ended', ended);
+ try { out.searchDocument = require('./server/events/search-documents').documentFor(owner); } catch (e) { out.searchDocumentError = e.message; }
+ const raw = db.get('SELECT s.*, ms.stream_key AS managed_stream_key, ms.slug AS managed_stream_slug FROM streams s LEFT JOIN managed_streams ms ON ms.id = s.managed_stream_id WHERE s.id = ?', [live]);
+ const streamer = db.getUserById(owner);
+ const writes = [];
+ const le = require('./server/streaming/live-events');
+ le.subscribe({ on() {} }, { writeHead() {}, write(s) { writes.push(String(s)); } });
+ le.announceGoLive(raw, streamer);
+ out.sse = writes.join('');
+ const posted = [];
+ global.fetch = async (url, opts = {}) => { posted.push(String(opts.body || '')); return { ok: true, status: 200, json: async () => ({}) }; };
+ require('./server/streaming/golive-notify').notifyFollowersGoLive(streamer, raw, { force: true });
+ await new Promise((r) => setTimeout(r, 300));
+ out.goLive = posted;
+ out.rawHadKey = raw.managed_stream_key === K.slotMain && streamer.stream_key === K.account;
+ db.close();
+ console.log(JSON.stringify({ ids: { owner, fan, peer, mod, admin, channel: ch.id, slot, slot2, live, ended, cam }, out }));
+ process.exit(0);
+ })().catch((e) => { console.error(e); process.exit(1); });
+`;
+
+function hits(text, needles) {
+ return Object.entries(needles).filter(([, v]) => v && String(text).includes(v)).map(([k]) => k);
+}
+
+(async () => {
+ const tmp = crawl.tempEnv('stream-keys');
+ const seeded = crawl.seed(tmp, SEED, { SENTINELS: JSON.stringify(K), INTERNAL_API_KEY: 'internal-sentinel-for-golive', OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9' });
+ assert.ok(seeded && seeded.ids, 'seed printed its ids');
+ const { ids, out } = seeded;
+
+ console.log('stream keys: what Live sends out about a stream');
+ await check('the seeded rows really carry the keys (raw stream row and user row)', () => {
+ assert.strictEqual(out.rawHadKey, true);
+ });
+ await check('live.stream.started / live.stream.ended envelopes carry no credential', () => {
+ assert.ok(out.started && out.ended, 'both envelopes built');
+ assert.deepStrictEqual(hits(JSON.stringify([out.started, out.ended]), K), []);
+ });
+ await check('the channel\'s search document carries no credential', () => {
+ assert.ok(out.searchDocument || out.searchDocumentError === undefined, out.searchDocumentError);
+ assert.deepStrictEqual(hits(JSON.stringify(out.searchDocument || {}), K), []);
+ });
+ await check('the go-live SSE event (every open page hears it) carries no credential, even from a raw row with the key', () => {
+ assert.match(out.sse, /stream-live/);
+ assert.deepStrictEqual(hits(out.sse, K), []);
+ });
+ await check('the go-live call to Network carries no credential', () => {
+ assert.ok(out.goLive.length >= 1, 'Network was called');
+ assert.deepStrictEqual(hits(out.goLive.join('\n'), K), []);
+ });
+
+ // ── The booted server ──
+ const keys = crawl.networkKeys(tmp);
+ const srv = await crawl.boot(tmp, { ...keys.env, NODE_ENV: 'production' });
+ const token = (id, username, role) => keys.sign({ sub: String(9000 + id), id: 9000 + id, username, role });
+ const owner = { Authorization: `Bearer ${token(ids.owner, 'keyowner', 'streamer')}` };
+ const people = {
+ anonymous: {},
+ viewer: { Authorization: `Bearer ${token(ids.fan, 'keyfan', 'user')}` },
+ 'another streamer': { Authorization: `Bearer ${token(ids.peer, 'keypeer', 'streamer')}` },
+ 'global mod': { Authorization: `Bearer ${token(ids.mod, 'keymod', 'global_mod')}` },
+ admin: { Authorization: `Bearer ${token(ids.admin, 'keyadmin', 'admin')}` },
+ };
+
+ console.log('stream keys: the owner sees their own keys (the sentinels are live)');
+ await check('GET /api/auth/stream-key shows the owner their account key', async () => {
+ const r = await srv.request('GET', '/api/auth/stream-key', { headers: owner });
+ assert.strictEqual(r.status, 200, r.text.slice(0, 200));
+ assert.ok(r.text.includes(K.account), 'account key shown to its owner');
+ });
+ await check('GET /api/streams/managed shows the owner their slot key', async () => {
+ const r = await srv.request('GET', '/api/streams/managed', { headers: owner });
+ assert.strictEqual(r.status, 200, r.text.slice(0, 200));
+ assert.ok(r.text.includes(K.slotMain), 'slot key shown to its owner');
+ });
+ await check('each person is signed in as themselves (their own account key comes back to them)', async () => {
+ for (const [who, headers] of Object.entries(people)) {
+ if (who === 'anonymous') continue;
+ const r = await srv.request('GET', '/api/auth/stream-key', { headers });
+ assert.strictEqual(r.status, 200, `${who}: ${r.text.slice(0, 200)}`);
+ }
+ });
+
+ // Parameter values: every seeded id and name, so each route is asked about the owner's things.
+ const numeric = [ids.live, ids.slot, ids.owner, ids.channel, ids.ended, ids.slot2, ids.cam, 1];
+ const byName = {
+ username: ['keyowner'], user: ['keyowner'], userId: [ids.owner], channelId: [ids.channel],
+ streamId: [ids.live, ids.ended], managedStreamId: [ids.slot, ids.slot2], slotIdOrSlug: ['main', ids.slot, 'relay'],
+ ref: ['main', ids.slot, 'relay'], platform: ['twitch', 'youtube', 'kick', 'custom'], anonId: ['anon1'], num: [1],
+ file: ['x.mp3'], filename: ['x.png'], slug: ['keyowner'], stat: ['wins'], type: ['vod', 'clip', 'stream'],
+ namespace: ['live.site_settings'], kind: ['user'], idOrSlug: ['default'], metric: ['viewers'], name: ['whip'],
+ ip: ['127.0.0.1'], commentId: [1],
+ };
+ const values = (n) => byName[n] || numeric;
+ const paths = crawl.getPaths(srv, values, {
+ query: `q=keyowner&username=keyowner&user_id=${ids.owner}&stream_id=${ids.live}&managed_stream_id=${ids.slot}`,
+ // The pages (SPA fallback + server-rendered SEO), which the route list shows only as '*'.
+ extra: ['/', '/@keyowner', '/@keyowner/main', '/@keyowner/relay', `/stream/${ids.live}`, `/recap/${ids.live}`, '/dashboard',
+ '/broadcast', '/broadcast/main', '/search?q=keyowner', '/content', '/moments', '/sitemap.xml', '/llms.txt', '/robots.txt',
+ `/overlay/chat/keyowner/${ids.slot}`, '/obs/chat/keyowner', `/popout/${ids.live}`, '/kiosk'],
+ });
+
+ console.log(`stream keys: ${paths.length} GET paths from ${srv.routes.length} routes, as ${Object.keys(people).length} people who are not the owner`);
+ const own = { viewer: 'fan', 'another streamer': 'peer', 'global mod': 'mod', admin: 'admin' };
+ const logs = crawl.captureLogs();
+ let result;
+ try {
+ // Everyone else's own key is a secret to each person too, except their own.
+ result = await crawl.crawlAll(srv, paths, people, (who) => Object.fromEntries(Object.entries(K).filter(([k]) => k !== own[who])));
+ } finally { logs.stop(); }
+ const { found, answered } = result;
+ console.log(` (answers: ${JSON.stringify(result.statuses)}; held open past the timeout: ${result.timedOut.join(', ') || 'none'})`);
+
+ await check('the crawl reached the server (most requests answered)', () => {
+ assert.ok(answered > paths.length * 4, `${answered} answers for ${paths.length * 5} requests`);
+ });
+ await check('no GET route or page shows the owner\'s credentials (or anyone else\'s) to anonymous, a viewer, another streamer, a mod or an admin', () => {
+ assert.deepStrictEqual(found, []);
+ });
+ await check('no log line written while serving those requests contains a credential', () => {
+ const bad = logs.lines.filter((l) => hits(l, K).length).map((l) => `${hits(l, K).join(',')}: ${l.slice(0, 200)}`);
+ assert.deepStrictEqual(bad, []);
+ });
+
+ tmp.cleanup();
+ if (failures) { console.log(`\n${failures} failure(s)`); process.exit(1); }
+ console.log('\nsecurity-stream-keys: all checks passed');
+ process.exit(0);
+})().catch((e) => { console.error(e); process.exit(1); });