openvibers/openvibe.live Commits
Broadcast: make audio changes reach viewers, VODs and every restream
Changes
@@@ -2220,6 +2220,7 @@ function cleanupStream(streamId) {_cleanupComposite(ss);if (ss._screenStream) { ss._screenStream.getTracks().forEach(t => t.stop()); ss._screenStream = null; }if (ss.localStream) { ss.localStream.getTracks().forEach(t => t.stop()); ss.localStream = null; }++ _teardownAudioMixer(ss);stopRobotStreamerRestream(streamId, { quiet: true }).catch(() => {});_cleanupSfuProduce(streamId);@@@ -2823,30 +2824,15 @@ async function startMediaCapture(streamId, opts = {}) {}ss._micEnabled = forceAudio !== null;-- // Mix desktop audio + mic audio if both are available++ // Always publish the mixer's output, even when only one source exists right now:++ // the mic can be toggled at any point during the stream, and a stable output track++ // is what makes that invisible to the VOD recorder, the SFU producer, the++ // RobotStreamer producer and every viewer connection.const screenAudioTracks = screenStream.getAudioTracks();-- let finalAudioTrack = null;-- if (micStream && screenAudioTracks.length > 0) {-- // Mix both via AudioContext-- try {-- const mixCtx = new AudioContext();-- const dest = mixCtx.createMediaStreamDestination();-- const desktopSource = mixCtx.createMediaStreamSource(new MediaStream(screenAudioTracks));-- desktopSource.connect(dest);-- const micSource = mixCtx.createMediaStreamSource(micStream);-- micSource.connect(dest);-- finalAudioTrack = dest.stream.getAudioTracks()[0];-- ss._mixAudioContext = mixCtx;-- ss._micStream = micStream;-- } catch {-- finalAudioTrack = micStream.getAudioTracks()[0] || screenAudioTracks[0];-- }-- } else if (micStream) {-- finalAudioTrack = micStream.getAudioTracks()[0];-- ss._micStream = micStream;-- } else if (screenAudioTracks.length > 0) {-- finalAudioTrack = screenAudioTracks[0];-- }++ _ensureAudioMixer(ss);++ if (screenAudioTracks.length > 0) _mixSetSource(ss, 'desktop', new MediaStream(screenAudioTracks));++ if (micStream) { _mixSetSource(ss, 'mic', micStream); ss._micStream = micStream; }++ const finalAudioTrack = _mixOutputTrack(ss);// If camera overlay is enabled, capture camera too and composite onto canvasif (ss._cameraOverlayEnabled) {@@@ -4270,11 +4256,7 @@ async function _rebuildScreenShareAudio(audioId, streamId) {ss._micStream.getTracks().forEach(t => t.stop());ss._micStream = null;}-- // Close existing AudioContext mix-- if (ss._mixAudioContext) {-- try { ss._mixAudioContext.close(); } catch {}-- ss._mixAudioContext = null;-- }++ // The mixer keeps its output track across a device swap — see _ensureAudioMixer.// Acquire the new mic (if audioId is not null and mic is enabled)let newMicStream = null;@@@ -4297,40 +4279,28 @@ async function _rebuildScreenShareAudio(audioId, streamId) {}}-- // Rebuild mix-- let finalAudioTrack = null;-- if (newMicStream && screenAudioTracks.length > 0) {-- try {-- const mixCtx = new AudioContext();-- const dest = mixCtx.createMediaStreamDestination();-- mixCtx.createMediaStreamSource(new MediaStream(screenAudioTracks)).connect(dest);-- mixCtx.createMediaStreamSource(newMicStream).connect(dest);-- finalAudioTrack = dest.stream.getAudioTracks()[0];-- ss._mixAudioContext = mixCtx;-- ss._micStream = newMicStream;-- } catch {-- finalAudioTrack = newMicStream.getAudioTracks()[0] || screenAudioTracks[0];-- }-- } else if (newMicStream) {-- finalAudioTrack = newMicStream.getAudioTracks()[0];-- ss._micStream = newMicStream;-- } else if (screenAudioTracks.length > 0) {-- finalAudioTrack = screenAudioTracks[0];-- }++ // Point the mixer at the new device. Its OUTPUT track is unchanged, so viewers, the++ // VOD recorder and every restream carry on with the track they already hold — a mic++ // swap mid-stream is now inaudible as a glitch rather than a silent failure.++ _ensureAudioMixer(ss);++ if (screenAudioTracks.length > 0) _mixSetSource(ss, 'desktop', new MediaStream(screenAudioTracks));++ _mixSetSource(ss, 'mic', newMicStream);++ ss._micStream = newMicStream;++ const finalAudioTrack = _mixOutputTrack(ss);if (!finalAudioTrack) {if (!newMicStream) toast('Could not acquire microphone for screen share', 'error');return;}-- // Swap audio track in localStream-- const oldAudio = ss.localStream.getAudioTracks()[0];-- if (oldAudio) { oldAudio.stop(); ss.localStream.removeTrack(oldAudio); }-- ss.localStream.addTrack(finalAudioTrack);---- // Propagate to all outputs: viewer PCs, RobotStreamer, SFU-- _replaceAllViewerTracks(streamId);-- toast('Microphone updated (screen share mode)', 'success');++ // Only touches anything on the one-off migration from a pre-mixer session.++ const published = ss.localStream?.getAudioTracks()[0] || null;++ if (published !== finalAudioTrack && ss.localStream) {++ if (published) ss.localStream.removeTrack(published);++ ss.localStream.addTrack(finalAudioTrack);++ _republishAudioTrack(streamId, finalAudioTrack);++ }++ toast(newMicStream ? 'Microphone updated (screen share mode)' : 'Microphone off (screen share mode)', 'success');}async function handleSignalingMessage(streamId, msg) {@@@ -4954,6 +4924,74 @@ function _replaceAllViewerTracks(streamId) {* Start the PiP composite — draws screen + camera overlay onto a canvas.* Camera position is controlled by ss._pipX / _pipY / _pipW (fractions of canvas).*/++++/* ── Stable audio mixer ───────────────────────────────────────────────────────++ Every consumer of the broadcast audio binds to a track ONCE and keeps it:++ MediaRecorder snapshots its tracks at construction and never notices later++ ones, mediasoup Producers wrap a specific track, and viewer RTCPeerConnection++ senders hold their own reference. So the old approach — removeTrack/addTrack++ on ss.localStream whenever the mic toggled — changed nothing for anybody++ downstream: viewers kept hearing the previous track and the VOD kept++ recording it, silently, for the rest of the session.++++ Instead, route all audio through one MediaStreamAudioDestinationNode. Its++ output track is created once and stays valid for the life of the context, so++ toggling the microphone is just connecting or disconnecting a source node and++ nothing downstream has to be told anything at all.++ ──────────────────────────────────────────────────────────────────────────── */++++function _ensureAudioMixer(ss) {++ if (ss._audioMix) return ss._audioMix;++ const Ctx = window.AudioContext || window.webkitAudioContext;++ const ctx = new Ctx();++ if (ctx.state === 'suspended') ctx.resume().catch(() => {});++ const dest = ctx.createMediaStreamDestination();++ // A zero-offset constant source keeps the graph running when nothing else is++ // connected, so the output track stays live (and silent) rather than ending++ // when the last real input goes away.++ const silence = ctx.createConstantSource();++ silence.offset.value = 0;++ silence.connect(dest);++ try { silence.start(); } catch { /* already started */ }++ ss._audioMix = { ctx, dest, silence, mic: null, desktop: null };++ return ss._audioMix;++}++++/** Connect or disconnect one named input ('mic' | 'desktop'). Output track is untouched. */++function _mixSetSource(ss, key, stream) {++ const mix = _ensureAudioMixer(ss);++ if (mix[key]) {++ try { mix[key].node.disconnect(); } catch { /* */ }++ mix[key] = null;++ }++ if (stream && stream.getAudioTracks && stream.getAudioTracks().length) {++ try {++ const node = mix.ctx.createMediaStreamSource(stream);++ node.connect(mix.dest);++ mix[key] = { node, stream };++ } catch (err) {++ console.warn('[Broadcast] Could not add', key, 'to the audio mix:', err.message);++ }++ }++ return _mixOutputTrack(ss);++}++++/** The one audio track the whole broadcast should ever publish. */++function _mixOutputTrack(ss) {++ try { return ss._audioMix?.dest.stream.getAudioTracks()[0] || null; } catch { return null; }++}++++function _teardownAudioMixer(ss) {++ if (!ss?._audioMix) return;++ const mix = ss._audioMix;++ for (const key of ['mic', 'desktop']) {++ if (mix[key]) { try { mix[key].node.disconnect(); } catch { /* */ } }++ }++ try { mix.silence.stop(); } catch { /* */ }++ try { mix.ctx.close(); } catch { /* */ }++ ss._audioMix = null;++}++function _startPipComposite(ss, screenStream, screenTrack, camStream, finalAudioTrack, res) {const canvas = document.createElement('canvas');canvas.width = res.w; canvas.height = res.h;@@@ -5055,83 +5093,40 @@ function _syncScreenShareLiveControls(ss) {/** Toggle mic on/off during screen share */async function toggleScreenShareMic() {const ss = getActiveStreamState();-- const streamId = broadcastState.activeStreamId;if (!ss || !broadcastState.settings.screenShare) return;if (ss._micEnabled) {-- // Turn mic OFF — disconnect mic from audio mix++ // Disconnect the mic from the mix and release the device. The published track is++ // the mixer's output, which does not change — so viewers, the VOD recorder and++ // every restream keep the exact same track and simply stop hearing the mic.++ _mixSetSource(ss, 'mic', null);if (ss._micStream) {ss._micStream.getTracks().forEach(t => t.stop());ss._micStream = null;}-- // Rebuild audio without mic — just desktop audio-- if (ss._mixAudioContext) {-- try { ss._mixAudioContext.close(); } catch {}-- ss._mixAudioContext = null;-- }-- // Replace audio track with desktop-only audio-- if (ss._screenStream) {-- const desktopAudioTracks = ss._screenStream.getAudioTracks();-- if (desktopAudioTracks.length > 0 && ss.localStream) {-- const oldAudio = ss.localStream.getAudioTracks()[0];-- if (oldAudio) ss.localStream.removeTrack(oldAudio);-- ss.localStream.addTrack(desktopAudioTracks[0]);-- _replaceAllViewerTracks(streamId);-- }-- }ss._micEnabled = false;toast('Mic muted', 'info');} else {-- // Turn mic ON — get mic stream and mix with desktop audioconst s = broadcastState.settings;-- const audioConstraints = buildAudioConstraints(s, s.forceAudio);let micStream;try {-- micStream = await _getUserMediaWithTimeout({ audio: audioConstraints, video: false });++ micStream = await _getUserMediaWithTimeout({ audio: buildAudioConstraints(s, s.forceAudio), video: false });} catch (err) {toast('Could not access microphone: ' + err.message, 'error');return;}ss._micStream = micStream;---- // Mix desktop + mic via AudioContext-- const desktopAudioTracks = ss._screenStream ? ss._screenStream.getAudioTracks() : [];-- if (desktopAudioTracks.length > 0) {-- try {-- const mixCtx = new AudioContext();-- const dest = mixCtx.createMediaStreamDestination();-- mixCtx.createMediaStreamSource(new MediaStream(desktopAudioTracks)).connect(dest);-- mixCtx.createMediaStreamSource(micStream).connect(dest);-- const mixedTrack = dest.stream.getAudioTracks()[0];-- ss._mixAudioContext = mixCtx;---- if (ss.localStream) {-- const oldAudio = ss.localStream.getAudioTracks()[0];-- if (oldAudio) ss.localStream.removeTrack(oldAudio);-- ss.localStream.addTrack(mixedTrack);-- _replaceAllViewerTracks(streamId);-- }-- } catch {-- // Fallback: just use mic track-- if (ss.localStream) {-- const oldAudio = ss.localStream.getAudioTracks()[0];-- if (oldAudio) ss.localStream.removeTrack(oldAudio);-- const micTrack = micStream.getAudioTracks()[0];-- if (micTrack) ss.localStream.addTrack(micTrack);-- _replaceAllViewerTracks(streamId);-- }-- }-- } else {-- // No desktop audio — just add mic-- if (ss.localStream) {-- const micTrack = micStream.getAudioTracks()[0];-- if (micTrack) {-- const oldAudio = ss.localStream.getAudioTracks()[0];-- if (oldAudio) ss.localStream.removeTrack(oldAudio);-- ss.localStream.addTrack(micTrack);-- _replaceAllViewerTracks(streamId);-- }-- }++ _mixSetSource(ss, 'mic', micStream);++++ // If capture began before a mixer existed (older session, or a path that published++ // a raw track), the published track is NOT the mixer output — swap it in once, and++ // tell the consumers that hold their own reference.++ const mixed = _mixOutputTrack(ss);++ const published = ss.localStream?.getAudioTracks()[0] || null;++ if (mixed && published !== mixed && ss.localStream) {++ if (published) ss.localStream.removeTrack(published);++ ss.localStream.addTrack(mixed);++ _republishAudioTrack(broadcastState.activeStreamId, mixed);}ss._micEnabled = true;toast('Mic unmuted', 'info');@@@ -5139,11 +5134,57 @@ async function toggleScreenShareMic() {_syncScreenShareLiveControls(ss);}++/**++ * Push a new audio track to everything that captured its own reference.++ *++ * Only needed when the published track genuinely changes identity — with the mixer in++ * place that is a one-off migration, not the normal path. MediaRecorder cannot swap++ * tracks at all, so the VOD segment is rolled instead; the chunked uploader already++ * treats segments as independent, so this costs a segment boundary and nothing else.++ */++function _republishAudioTrack(streamId, track) {++ const ss = getStreamState(streamId);++ if (!ss || !track) return;++++ // Local SFU producer++ try {++ const st = _sfuProduceStates.get(streamId);++ st?.audioProducer?.replaceTrack({ track }).catch((e) => console.warn('[Broadcast] SFU audio replaceTrack failed:', e.message));++ } catch { /* */ }++++ // RobotStreamer producer++ try {++ ss.robotStreamer?.audioProducer?.replaceTrack({ track }).catch((e) => console.warn('[RS Restream] audio replaceTrack failed:', e.message));++ } catch { /* */ }++++ // Direct viewer peer connections++ try {++ ss.viewerConnections?.forEach((pc) => {++ const sender = pc.getSenders?.().find(x => x.track && x.track.kind === 'audio');++ sender?.replaceTrack(track).catch(() => {});++ });++ } catch { /* */ }++++ // VOD: MediaRecorder is bound to the tracks it was constructed with, so roll the++ // segment. uploadVodRecording({finalizeStream:false}) flushes what has been captured++ // so far without ending the recording server-side; startVodRecording() then opens a++ // fresh segment around the new track. The chunk uploader already treats segments as++ // independent units, so the cost is one segment boundary.++ (async () => {++ try {++ if (!ss.vodRecorder || ss.vodRecorder.state === 'inactive') return;++ console.log('[VOD] Rolling segment to pick up the new audio track');++ await uploadVodRecording(streamId, { finalizeStream: false });++ startVodRecording(streamId);++ } catch (e) { console.warn('[VOD] Could not roll segment for audio change:', e.message); }++ })();++}++/** Cleanup composite canvas + camera overlay resources */function _cleanupComposite(ss) {if (ss._compositeAnimFrame) { cancelAnimationFrame(ss._compositeAnimFrame); ss._compositeAnimFrame = null; }if (ss._cameraOverlayStream) { ss._cameraOverlayStream.getTracks().forEach(t => t.stop()); ss._cameraOverlayStream = null; }-- if (ss._mixAudioContext) { try { ss._mixAudioContext.close(); } catch {} ss._mixAudioContext = null; }++ _teardownAudioMixer(ss);if (ss._micStream) { ss._micStream.getTracks().forEach(t => t.stop()); ss._micStream = null; }ss._compositeCanvas = null;ss._compositeCtx = null;@@@ -357,6 +357,7 @@ class RsPassthroughRelay {_teardown(session) {if (session.statsTimer) { clearInterval(session.statsTimer); session.statsTimer = null; }if (session.ingestWatchdog) { clearInterval(session.ingestWatchdog); session.ingestWatchdog = null; }++ if (session._audioWatch) { try { session._audioWatch(); } catch { /* */ } session._audioWatch = null; }const sfu = require('../streaming/webrtc-sfu');for (const ing of session.ingests) {try { ing.socket.removeAllListeners('message'); ing.socket.close(); } catch {}@@@ -375,10 +376,35 @@ class RsPassthroughRelay {const sid = session.streamId;// 1) Source producers on our SFU (wait for video; audio optional).++ //++ // Audio is sampled ONCE here, and the werift peer is built around whatever++ // exists at this instant. A broadcaster who goes live video-only and switches++ // the microphone on afterwards would otherwise stay silent on RobotStreamer for++ // the whole session, because nothing revisits this decision. Give audio a short++ // grace period so a mic that is a moment behind the video producer still makes++ // it into the session, and watch for a later one so we can rebuild.const videoProd = await sfu.waitForProducer(session.roomId, 'video', 30000);-- const audioProd = sfu.findProducerByKind(session.roomId, 'audio');++ let audioProd = sfu.findProducerByKind(session.roomId, 'audio');++ if (!audioProd) {++ try { audioProd = await sfu.waitForProducer(session.roomId, 'audio', 4000); }++ catch { audioProd = null; } // genuinely a video-only stream++ }log(sid, `source producers: video=${videoProd.id}${audioProd ? ` audio=${audioProd.id}` : ' (no audio)'}`);++ // If we ended up without audio, restart the passthrough the moment one appears.++ if (!audioProd) {++ const onAudio = ({ roomId, kind }) => {++ if (session.stopped || kind !== 'audio' || roomId !== session.roomId) return;++ sfu.removeListener?.('producer-added', onAudio);++ log(sid, 'audio producer appeared after start — restarting passthrough to include it');++ const st = { id: sid }; const integ = { robot_id: session.robotId, token: session.token };++ this._teardown(session);++ this._scheduleRestart(session, st, integ, 'audio producer added after a video-only start');++ };++ sfu.on?.('producer-added', onAudio);++ session._audioWatch = () => sfu.removeListener?.('producer-added', onAudio);++ }++// 2) Encoded-RTP ingest (PlainTransport → localhost UDP socket).const videoIn = await openPlainIngest(sfu, session.roomId, videoProd.id);session.ingests.push(videoIn);@@@ -162,6 +162,46 @@ class RestreamManager extends EventEmitter {if (kind !== 'video') return; // Only restart on video producer lossthis._handleProducerRemoved(roomId, producerId);});++ // An AUDIO producer appearing after we started matters just as much as one++ // going away. FFmpeg's argument list is built once, and `hasAudio` is decided++ // from whether an audio producer existed at that moment — so a broadcaster who++ // goes live video-only and then switches the microphone on (screen-share mic++ // toggle, or a mic granted after the fact) would keep pushing SILENT video to++ // Twitch/Kick for the rest of the session, with nothing in the logs to say why.++ // Restart the session so the SDP and the -map arguments are rebuilt with audio.++ webrtcSFU.on('producer-added', ({ roomId, kind }) => {++ if (kind !== 'audio') return;++ this._handleAudioProducerAdded(roomId);++ });++ }++++ /**++ * An audio producer appeared. Restart any running WebRTC restream for that stream++ * that was started without audio; leave the ones that already have it alone, since++ * restarting a healthy session would drop frames on the destination for no reason.++ */++ _handleAudioProducerAdded(roomId) {++ const m = String(roomId).match(/^stream-(\d+)$/);++ if (!m) return;++ const streamId = parseInt(m[1], 10);++ let restarted = 0;++ for (const [key, session] of this.sessions) {++ if (session.streamId !== streamId) continue;++ if (session.streamInfo?.protocol !== 'webrtc') continue;++ if (session.status === 'stopped') continue;++ if (session.webrtcState?.hasAudio) continue; // already carrying audio++ console.log(`[Restream] Audio producer appeared in ${roomId} — restarting ${key} to include it`);++ this._killProcess(session);++ session.status = 'error';++ session.lastError = 'Audio source added';++ session.restartAttempts = 0;++ session.restartDelay = RESTART_BASE_DELAY;++ this._scheduleRestart(session);++ restarted++;++ }++ if (restarted > 0) {++ console.log(`[Restream] Restarting ${restarted} restream(s) for stream ${streamId} to pick up audio`);++ }}/**@@@ -506,6 +546,9 @@ class RestreamManager extends EventEmitter {sdpPath,videoTransportId: videoConsumer.transportId,audioTransportId: audioConsumer?.transportId || null,++ // Whether THIS ffmpeg invocation was built with an audio mapping. Read by++ // _handleAudioProducerAdded to avoid restarting sessions that already have it.++ hasAudio: !!audioConsumer,};// Step 5: Spawn FFmpeg with SDP input → RTMP outputnnew file mode 100644@@@ -0,0 +1,86 @@++/**++ * Tranche 2 contract: mic/desktop audio must reach every downstream consumer, and keep++ * reaching them when the streamer changes something mid-broadcast.++ *++ * Every consumer of the broadcast audio binds to a track ONCE and keeps its own++ * reference — MediaRecorder snapshots its tracks at construction, mediasoup Producers++ * wrap a specific track, viewer RTCPeerConnection senders hold theirs. So the old++ * removeTrack/addTrack dance on ss.localStream changed nothing for anyone: toggling the++ * mic mid-stream left viewers hearing the previous track and the VOD recording it,++ * silently, for the rest of the session.++ *++ * Server side, the same shape of bug existed across process boundaries: ffmpeg's++ * argument list and the RS relay's werift peer are both built once from whichever++ * producers existed at that instant, so a broadcaster who went live video-only and then++ * switched the mic on kept pushing silent video to Twitch/Kick/RobotStreamer.++ */++const assert = require('assert');++const fs = require('fs');++const path = require('path');++++const root = path.join(__dirname, '..');++const read = (p) => fs.readFileSync(path.join(root, p), 'utf8');++const bc = read('public/js/broadcast.js');++const restream = read('server/streaming/restream-manager.js');++const relay = read('server/integrations/rs-passthrough-relay.js');++++// ── A: one mixer, one stable output track ────────────────────────────────────────────++assert.ok(/function _ensureAudioMixer\(ss\)/.test(bc), 'a shared audio mixer must exist');++assert.ok(/createMediaStreamDestination\(\)/.test(bc), 'the mixer must publish a MediaStreamDestination track');++assert.ok(/createConstantSource\(\)/.test(bc),++ 'the mixer needs a silent keep-alive source, or its output track ends when the last input is removed');++assert.ok(/function _mixOutputTrack\(ss\)/.test(bc), 'consumers need one accessor for the published track');++console.log('OK A: mixer exists and its output track is kept alive independently of its inputs');++++// ── B: capture publishes the mixer output, not a raw device track ────────────────────++const captureIdx = bc.indexOf('const finalAudioTrack = _mixOutputTrack(ss);');++assert.ok(captureIdx > 0, 'screen-share capture must publish the mixer output');++console.log('OK B: screen-share capture publishes the mixer output');++++// ── C: toggling the mic only touches the mixer ───────────────────────────────────────++const toggle = bc.slice(bc.indexOf('async function toggleScreenShareMic()'),++ bc.indexOf('function _republishAudioTrack('));++assert.ok(toggle.includes("_mixSetSource(ss, 'mic', null)"), 'muting must disconnect the mic from the mix');++assert.ok(toggle.includes("_mixSetSource(ss, 'mic', micStream)"), 'unmuting must connect the mic to the mix');++assert.ok(!/localStream\.removeTrack\([\s\S]{0,40}\)\s*;\s*[\s\S]{0,80}addTrack\(desktopAudioTracks/.test(toggle),++ 'the old remove/add track dance must be gone — it never reached any consumer');++console.log('OK C: mic toggle is a mixer operation, invisible to downstream consumers');++++// ── D: when the track identity DOES change, every holder is told ─────────────────────++const republish = bc.slice(bc.indexOf('function _republishAudioTrack('));++for (const [what, re] of [++ ['the local SFU producer', /audioProducer\?\.replaceTrack\(\{ track \}\)/],++ ['the RobotStreamer producer', /robotStreamer\?\.audioProducer\?\.replaceTrack/],++ ['viewer peer connections', /getSenders\?\.\(\)[\s\S]{0,120}replaceTrack\(track\)/],++ ['the VOD recorder', /uploadVodRecording\(streamId, \{ finalizeStream: false \}\)[\s\S]{0,120}startVodRecording\(streamId\)/],++]) {++ assert.ok(re.test(republish), `_republishAudioTrack must update ${what}`);++}++console.log('OK D: SFU, RobotStreamer, viewer PCs and the VOD segment are all updated on a real track change');++++// ── E: the mixer is torn down with the capture ───────────────────────────────────────++assert.ok(/_teardownAudioMixer\(ss\)/.test(bc), 'the mixer must be closed when capture stops');++assert.strictEqual((bc.match(/_mixAudioContext/g) || []).length, 0,++ 'the old per-toggle AudioContext must be fully retired — two mixers would double the audio');++console.log('OK E: mixer torn down on stop, legacy per-toggle context fully removed');++++// ── F: RTMP restreams rebuild when audio shows up late ───────────────────────────────++assert.ok(/webrtcSFU\.on\('producer-added'/.test(restream),++ 'the restream manager must watch for producers being ADDED, not only removed');++assert.ok(/_handleAudioProducerAdded\(roomId\)/.test(restream), 'audio arriving late needs a handler');++const handler = restream.slice(restream.indexOf('_handleAudioProducerAdded(roomId) {'));++assert.ok(/webrtcState\?\.hasAudio\) continue/.test(handler),++ 'sessions that already carry audio must be left alone — restarting them drops frames for nothing');++assert.ok(/hasAudio: !!audioConsumer/.test(restream),++ 'the session must record whether its ffmpeg was built with an audio mapping');++console.log('OK F: Twitch/Kick restreams restart to pick up a microphone enabled after going live');++++// ── G: the RobotStreamer relay does the same ─────────────────────────────────────────++assert.ok(/waitForProducer\(session\.roomId, 'audio', \d+\)/.test(relay),++ 'the relay should give a slightly-late mic a grace period before declaring the stream video-only');++assert.ok(/producer-added/.test(relay) && /audio producer appeared after start/.test(relay),++ 'the relay must restart when audio appears after a video-only start');++assert.ok(/session\._audioWatch/.test(relay), 'the watcher must be tracked so teardown can remove it');++console.log('OK G: RobotStreamer relay waits for a late mic, then rebuilds if one appears');++++console.log('✅ broadcast audio harmony test passed');
@@ -2220,6 +2220,7 @@ function cleanupStream(streamId) {
@@ -2220,6 +2220,7 @@ function cleanupStream(streamId) {
_cleanupComposite(ss);
_cleanupComposite(ss);
if (ss._screenStream) { ss._screenStream.getTracks().forEach(t => t.stop()); ss._screenStream = null; }
if (ss._screenStream) { ss._screenStream.getTracks().forEach(t => t.stop()); ss._screenStream = null; }
if (ss.localStream) { ss.localStream.getTracks().forEach(t => t.stop()); ss.localStream = null; }
if (ss.localStream) { ss.localStream.getTracks().forEach(t => t.stop()); ss.localStream = null; }
+ _teardownAudioMixer(ss);
stopRobotStreamerRestream(streamId, { quiet: true }).catch(() => {});
stopRobotStreamerRestream(streamId, { quiet: true }).catch(() => {});
_cleanupSfuProduce(streamId);
_cleanupSfuProduce(streamId);
@@ -2823,30 +2824,15 @@ async function startMediaCapture(streamId, opts = {}) {
@@ -2823,30 +2824,15 @@ async function startMediaCapture(streamId, opts = {}) {
}
}
ss._micEnabled = forceAudio !== null;
ss._micEnabled = forceAudio !== null;
- // Mix desktop audio + mic audio if both are available
+ // Always publish the mixer's output, even when only one source exists right now:
+ // the mic can be toggled at any point during the stream, and a stable output track
+ // is what makes that invisible to the VOD recorder, the SFU producer, the
+ // RobotStreamer producer and every viewer connection.
const screenAudioTracks = screenStream.getAudioTracks();
const screenAudioTracks = screenStream.getAudioTracks();
- let finalAudioTrack = null;
- if (micStream && screenAudioTracks.length > 0) {
- // Mix both via AudioContext
- try {
- const mixCtx = new AudioContext();
- const dest = mixCtx.createMediaStreamDestination();
- const desktopSource = mixCtx.createMediaStreamSource(new MediaStream(screenAudioTracks));
- desktopSource.connect(dest);
- const micSource = mixCtx.createMediaStreamSource(micStream);
- micSource.connect(dest);
- finalAudioTrack = dest.stream.getAudioTracks()[0];
- ss._mixAudioContext = mixCtx;
- ss._micStream = micStream;
- } catch {
- finalAudioTrack = micStream.getAudioTracks()[0] || screenAudioTracks[0];
- }
- } else if (micStream) {
- finalAudioTrack = micStream.getAudioTracks()[0];
- ss._micStream = micStream;
- } else if (screenAudioTracks.length > 0) {
- finalAudioTrack = screenAudioTracks[0];
- }
+ _ensureAudioMixer(ss);
+ if (screenAudioTracks.length > 0) _mixSetSource(ss, 'desktop', new MediaStream(screenAudioTracks));
+ if (micStream) { _mixSetSource(ss, 'mic', micStream); ss._micStream = micStream; }
+ const finalAudioTrack = _mixOutputTrack(ss);
// If camera overlay is enabled, capture camera too and composite onto canvas
// If camera overlay is enabled, capture camera too and composite onto canvas
if (ss._cameraOverlayEnabled) {
if (ss._cameraOverlayEnabled) {
@@ -4270,11 +4256,7 @@ async function _rebuildScreenShareAudio(audioId, streamId) {
@@ -4270,11 +4256,7 @@ async function _rebuildScreenShareAudio(audioId, streamId) {
ss._micStream.getTracks().forEach(t => t.stop());
ss._micStream.getTracks().forEach(t => t.stop());
ss._micStream = null;
ss._micStream = null;
}
}
- // Close existing AudioContext mix
- if (ss._mixAudioContext) {
- try { ss._mixAudioContext.close(); } catch {}
- ss._mixAudioContext = null;
- }
+ // The mixer keeps its output track across a device swap — see _ensureAudioMixer.
// Acquire the new mic (if audioId is not null and mic is enabled)
// Acquire the new mic (if audioId is not null and mic is enabled)
let newMicStream = null;
let newMicStream = null;
@@ -4297,40 +4279,28 @@ async function _rebuildScreenShareAudio(audioId, streamId) {
@@ -4297,40 +4279,28 @@ async function _rebuildScreenShareAudio(audioId, streamId) {
}
}
}
}
- // Rebuild mix
- let finalAudioTrack = null;
- if (newMicStream && screenAudioTracks.length > 0) {
- try {
- const mixCtx = new AudioContext();
- const dest = mixCtx.createMediaStreamDestination();
- mixCtx.createMediaStreamSource(new MediaStream(screenAudioTracks)).connect(dest);
- mixCtx.createMediaStreamSource(newMicStream).connect(dest);
- finalAudioTrack = dest.stream.getAudioTracks()[0];
- ss._mixAudioContext = mixCtx;
- ss._micStream = newMicStream;
- } catch {
- finalAudioTrack = newMicStream.getAudioTracks()[0] || screenAudioTracks[0];
- }
- } else if (newMicStream) {
- finalAudioTrack = newMicStream.getAudioTracks()[0];
- ss._micStream = newMicStream;
- } else if (screenAudioTracks.length > 0) {
- finalAudioTrack = screenAudioTracks[0];
- }
+ // Point the mixer at the new device. Its OUTPUT track is unchanged, so viewers, the
+ // VOD recorder and every restream carry on with the track they already hold — a mic
+ // swap mid-stream is now inaudible as a glitch rather than a silent failure.
+ _ensureAudioMixer(ss);
+ if (screenAudioTracks.length > 0) _mixSetSource(ss, 'desktop', new MediaStream(screenAudioTracks));
+ _mixSetSource(ss, 'mic', newMicStream);
+ ss._micStream = newMicStream;
+ const finalAudioTrack = _mixOutputTrack(ss);
if (!finalAudioTrack) {
if (!finalAudioTrack) {
if (!newMicStream) toast('Could not acquire microphone for screen share', 'error');
if (!newMicStream) toast('Could not acquire microphone for screen share', 'error');
return;
return;
}
}
- // Swap audio track in localStream
- const oldAudio = ss.localStream.getAudioTracks()[0];
- if (oldAudio) { oldAudio.stop(); ss.localStream.removeTrack(oldAudio); }
- ss.localStream.addTrack(finalAudioTrack);
-
- // Propagate to all outputs: viewer PCs, RobotStreamer, SFU
- _replaceAllViewerTracks(streamId);
- toast('Microphone updated (screen share mode)', 'success');
+ // Only touches anything on the one-off migration from a pre-mixer session.
+ const published = ss.localStream?.getAudioTracks()[0] || null;
+ if (published !== finalAudioTrack && ss.localStream) {
+ if (published) ss.localStream.removeTrack(published);
+ ss.localStream.addTrack(finalAudioTrack);
+ _republishAudioTrack(streamId, finalAudioTrack);
+ }
+ toast(newMicStream ? 'Microphone updated (screen share mode)' : 'Microphone off (screen share mode)', 'success');
}
}
async function handleSignalingMessage(streamId, msg) {
async function handleSignalingMessage(streamId, msg) {
@@ -4954,6 +4924,74 @@ function _replaceAllViewerTracks(streamId) {
@@ -4954,6 +4924,74 @@ function _replaceAllViewerTracks(streamId) {
* Start the PiP composite — draws screen + camera overlay onto a canvas.
* Start the PiP composite — draws screen + camera overlay onto a canvas.
* Camera position is controlled by ss._pipX / _pipY / _pipW (fractions of canvas).
* Camera position is controlled by ss._pipX / _pipY / _pipW (fractions of canvas).
*/
*/
+
+/* ── Stable audio mixer ───────────────────────────────────────────────────────
+ Every consumer of the broadcast audio binds to a track ONCE and keeps it:
+ MediaRecorder snapshots its tracks at construction and never notices later
+ ones, mediasoup Producers wrap a specific track, and viewer RTCPeerConnection
+ senders hold their own reference. So the old approach — removeTrack/addTrack
+ on ss.localStream whenever the mic toggled — changed nothing for anybody
+ downstream: viewers kept hearing the previous track and the VOD kept
+ recording it, silently, for the rest of the session.
+
+ Instead, route all audio through one MediaStreamAudioDestinationNode. Its
+ output track is created once and stays valid for the life of the context, so
+ toggling the microphone is just connecting or disconnecting a source node and
+ nothing downstream has to be told anything at all.
+ ──────────────────────────────────────────────────────────────────────────── */
+
+function _ensureAudioMixer(ss) {
+ if (ss._audioMix) return ss._audioMix;
+ const Ctx = window.AudioContext || window.webkitAudioContext;
+ const ctx = new Ctx();
+ if (ctx.state === 'suspended') ctx.resume().catch(() => {});
+ const dest = ctx.createMediaStreamDestination();
+ // A zero-offset constant source keeps the graph running when nothing else is
+ // connected, so the output track stays live (and silent) rather than ending
+ // when the last real input goes away.
+ const silence = ctx.createConstantSource();
+ silence.offset.value = 0;
+ silence.connect(dest);
+ try { silence.start(); } catch { /* already started */ }
+ ss._audioMix = { ctx, dest, silence, mic: null, desktop: null };
+ return ss._audioMix;
+}
+
+/** Connect or disconnect one named input ('mic' | 'desktop'). Output track is untouched. */
+function _mixSetSource(ss, key, stream) {
+ const mix = _ensureAudioMixer(ss);
+ if (mix[key]) {
+ try { mix[key].node.disconnect(); } catch { /* */ }
+ mix[key] = null;
+ }
+ if (stream && stream.getAudioTracks && stream.getAudioTracks().length) {
+ try {
+ const node = mix.ctx.createMediaStreamSource(stream);
+ node.connect(mix.dest);
+ mix[key] = { node, stream };
+ } catch (err) {
+ console.warn('[Broadcast] Could not add', key, 'to the audio mix:', err.message);
+ }
+ }
+ return _mixOutputTrack(ss);
+}
+
+/** The one audio track the whole broadcast should ever publish. */
+function _mixOutputTrack(ss) {
+ try { return ss._audioMix?.dest.stream.getAudioTracks()[0] || null; } catch { return null; }
+}
+
+function _teardownAudioMixer(ss) {
+ if (!ss?._audioMix) return;
+ const mix = ss._audioMix;
+ for (const key of ['mic', 'desktop']) {
+ if (mix[key]) { try { mix[key].node.disconnect(); } catch { /* */ } }
+ }
+ try { mix.silence.stop(); } catch { /* */ }
+ try { mix.ctx.close(); } catch { /* */ }
+ ss._audioMix = null;
+}
+
function _startPipComposite(ss, screenStream, screenTrack, camStream, finalAudioTrack, res) {
function _startPipComposite(ss, screenStream, screenTrack, camStream, finalAudioTrack, res) {
const canvas = document.createElement('canvas');
const canvas = document.createElement('canvas');
canvas.width = res.w; canvas.height = res.h;
canvas.width = res.w; canvas.height = res.h;
@@ -5055,83 +5093,40 @@ function _syncScreenShareLiveControls(ss) {
@@ -5055,83 +5093,40 @@ function _syncScreenShareLiveControls(ss) {
/** Toggle mic on/off during screen share */
/** Toggle mic on/off during screen share */
async function toggleScreenShareMic() {
async function toggleScreenShareMic() {
const ss = getActiveStreamState();
const ss = getActiveStreamState();
- const streamId = broadcastState.activeStreamId;
if (!ss || !broadcastState.settings.screenShare) return;
if (!ss || !broadcastState.settings.screenShare) return;
if (ss._micEnabled) {
if (ss._micEnabled) {
- // Turn mic OFF — disconnect mic from audio mix
+ // Disconnect the mic from the mix and release the device. The published track is
+ // the mixer's output, which does not change — so viewers, the VOD recorder and
+ // every restream keep the exact same track and simply stop hearing the mic.
+ _mixSetSource(ss, 'mic', null);
if (ss._micStream) {
if (ss._micStream) {
ss._micStream.getTracks().forEach(t => t.stop());
ss._micStream.getTracks().forEach(t => t.stop());
ss._micStream = null;
ss._micStream = null;
}
}
- // Rebuild audio without mic — just desktop audio
- if (ss._mixAudioContext) {
- try { ss._mixAudioContext.close(); } catch {}
- ss._mixAudioContext = null;
- }
- // Replace audio track with desktop-only audio
- if (ss._screenStream) {
- const desktopAudioTracks = ss._screenStream.getAudioTracks();
- if (desktopAudioTracks.length > 0 && ss.localStream) {
- const oldAudio = ss.localStream.getAudioTracks()[0];
- if (oldAudio) ss.localStream.removeTrack(oldAudio);
- ss.localStream.addTrack(desktopAudioTracks[0]);
- _replaceAllViewerTracks(streamId);
- }
- }
ss._micEnabled = false;
ss._micEnabled = false;
toast('Mic muted', 'info');
toast('Mic muted', 'info');
} else {
} else {
- // Turn mic ON — get mic stream and mix with desktop audio
const s = broadcastState.settings;
const s = broadcastState.settings;
- const audioConstraints = buildAudioConstraints(s, s.forceAudio);
let micStream;
let micStream;
try {
try {
- micStream = await _getUserMediaWithTimeout({ audio: audioConstraints, video: false });
+ micStream = await _getUserMediaWithTimeout({ audio: buildAudioConstraints(s, s.forceAudio), video: false });
} catch (err) {
} catch (err) {
toast('Could not access microphone: ' + err.message, 'error');
toast('Could not access microphone: ' + err.message, 'error');
return;
return;
}
}
ss._micStream = micStream;
ss._micStream = micStream;
-
- // Mix desktop + mic via AudioContext
- const desktopAudioTracks = ss._screenStream ? ss._screenStream.getAudioTracks() : [];
- if (desktopAudioTracks.length > 0) {
- try {
- const mixCtx = new AudioContext();
- const dest = mixCtx.createMediaStreamDestination();
- mixCtx.createMediaStreamSource(new MediaStream(desktopAudioTracks)).connect(dest);
- mixCtx.createMediaStreamSource(micStream).connect(dest);
- const mixedTrack = dest.stream.getAudioTracks()[0];
- ss._mixAudioContext = mixCtx;
-
- if (ss.localStream) {
- const oldAudio = ss.localStream.getAudioTracks()[0];
- if (oldAudio) ss.localStream.removeTrack(oldAudio);
- ss.localStream.addTrack(mixedTrack);
- _replaceAllViewerTracks(streamId);
- }
- } catch {
- // Fallback: just use mic track
- if (ss.localStream) {
- const oldAudio = ss.localStream.getAudioTracks()[0];
- if (oldAudio) ss.localStream.removeTrack(oldAudio);
- const micTrack = micStream.getAudioTracks()[0];
- if (micTrack) ss.localStream.addTrack(micTrack);
- _replaceAllViewerTracks(streamId);
- }
- }
- } else {
- // No desktop audio — just add mic
- if (ss.localStream) {
- const micTrack = micStream.getAudioTracks()[0];
- if (micTrack) {
- const oldAudio = ss.localStream.getAudioTracks()[0];
- if (oldAudio) ss.localStream.removeTrack(oldAudio);
- ss.localStream.addTrack(micTrack);
- _replaceAllViewerTracks(streamId);
- }
- }
+ _mixSetSource(ss, 'mic', micStream);
+
+ // If capture began before a mixer existed (older session, or a path that published
+ // a raw track), the published track is NOT the mixer output — swap it in once, and
+ // tell the consumers that hold their own reference.
+ const mixed = _mixOutputTrack(ss);
+ const published = ss.localStream?.getAudioTracks()[0] || null;
+ if (mixed && published !== mixed && ss.localStream) {
+ if (published) ss.localStream.removeTrack(published);
+ ss.localStream.addTrack(mixed);
+ _republishAudioTrack(broadcastState.activeStreamId, mixed);
}
}
ss._micEnabled = true;
ss._micEnabled = true;
toast('Mic unmuted', 'info');
toast('Mic unmuted', 'info');
@@ -5139,11 +5134,57 @@ async function toggleScreenShareMic() {
@@ -5139,11 +5134,57 @@ async function toggleScreenShareMic() {
_syncScreenShareLiveControls(ss);
_syncScreenShareLiveControls(ss);
}
}
+/**
+ * Push a new audio track to everything that captured its own reference.
+ *
+ * Only needed when the published track genuinely changes identity — with the mixer in
+ * place that is a one-off migration, not the normal path. MediaRecorder cannot swap
+ * tracks at all, so the VOD segment is rolled instead; the chunked uploader already
+ * treats segments as independent, so this costs a segment boundary and nothing else.
+ */
+function _republishAudioTrack(streamId, track) {
+ const ss = getStreamState(streamId);
+ if (!ss || !track) return;
+
+ // Local SFU producer
+ try {
+ const st = _sfuProduceStates.get(streamId);
+ st?.audioProducer?.replaceTrack({ track }).catch((e) => console.warn('[Broadcast] SFU audio replaceTrack failed:', e.message));
+ } catch { /* */ }
+
+ // RobotStreamer producer
+ try {
+ ss.robotStreamer?.audioProducer?.replaceTrack({ track }).catch((e) => console.warn('[RS Restream] audio replaceTrack failed:', e.message));
+ } catch { /* */ }
+
+ // Direct viewer peer connections
+ try {
+ ss.viewerConnections?.forEach((pc) => {
+ const sender = pc.getSenders?.().find(x => x.track && x.track.kind === 'audio');
+ sender?.replaceTrack(track).catch(() => {});
+ });
+ } catch { /* */ }
+
+ // VOD: MediaRecorder is bound to the tracks it was constructed with, so roll the
+ // segment. uploadVodRecording({finalizeStream:false}) flushes what has been captured
+ // so far without ending the recording server-side; startVodRecording() then opens a
+ // fresh segment around the new track. The chunk uploader already treats segments as
+ // independent units, so the cost is one segment boundary.
+ (async () => {
+ try {
+ if (!ss.vodRecorder || ss.vodRecorder.state === 'inactive') return;
+ console.log('[VOD] Rolling segment to pick up the new audio track');
+ await uploadVodRecording(streamId, { finalizeStream: false });
+ startVodRecording(streamId);
+ } catch (e) { console.warn('[VOD] Could not roll segment for audio change:', e.message); }
+ })();
+}
+
/** Cleanup composite canvas + camera overlay resources */
/** Cleanup composite canvas + camera overlay resources */
function _cleanupComposite(ss) {
function _cleanupComposite(ss) {
if (ss._compositeAnimFrame) { cancelAnimationFrame(ss._compositeAnimFrame); ss._compositeAnimFrame = null; }
if (ss._compositeAnimFrame) { cancelAnimationFrame(ss._compositeAnimFrame); ss._compositeAnimFrame = null; }
if (ss._cameraOverlayStream) { ss._cameraOverlayStream.getTracks().forEach(t => t.stop()); ss._cameraOverlayStream = null; }
if (ss._cameraOverlayStream) { ss._cameraOverlayStream.getTracks().forEach(t => t.stop()); ss._cameraOverlayStream = null; }
- if (ss._mixAudioContext) { try { ss._mixAudioContext.close(); } catch {} ss._mixAudioContext = null; }
+ _teardownAudioMixer(ss);
if (ss._micStream) { ss._micStream.getTracks().forEach(t => t.stop()); ss._micStream = null; }
if (ss._micStream) { ss._micStream.getTracks().forEach(t => t.stop()); ss._micStream = null; }
ss._compositeCanvas = null;
ss._compositeCanvas = null;
ss._compositeCtx = null;
ss._compositeCtx = null;
@@ -357,6 +357,7 @@ class RsPassthroughRelay {
@@ -357,6 +357,7 @@ class RsPassthroughRelay {
_teardown(session) {
_teardown(session) {
if (session.statsTimer) { clearInterval(session.statsTimer); session.statsTimer = null; }
if (session.statsTimer) { clearInterval(session.statsTimer); session.statsTimer = null; }
if (session.ingestWatchdog) { clearInterval(session.ingestWatchdog); session.ingestWatchdog = null; }
if (session.ingestWatchdog) { clearInterval(session.ingestWatchdog); session.ingestWatchdog = null; }
+ if (session._audioWatch) { try { session._audioWatch(); } catch { /* */ } session._audioWatch = null; }
const sfu = require('../streaming/webrtc-sfu');
const sfu = require('../streaming/webrtc-sfu');
for (const ing of session.ingests) {
for (const ing of session.ingests) {
try { ing.socket.removeAllListeners('message'); ing.socket.close(); } catch {}
try { ing.socket.removeAllListeners('message'); ing.socket.close(); } catch {}
@@ -375,10 +376,35 @@ class RsPassthroughRelay {
@@ -375,10 +376,35 @@ class RsPassthroughRelay {
const sid = session.streamId;
const sid = session.streamId;
// 1) Source producers on our SFU (wait for video; audio optional).
// 1) Source producers on our SFU (wait for video; audio optional).
+ //
+ // Audio is sampled ONCE here, and the werift peer is built around whatever
+ // exists at this instant. A broadcaster who goes live video-only and switches
+ // the microphone on afterwards would otherwise stay silent on RobotStreamer for
+ // the whole session, because nothing revisits this decision. Give audio a short
+ // grace period so a mic that is a moment behind the video producer still makes
+ // it into the session, and watch for a later one so we can rebuild.
const videoProd = await sfu.waitForProducer(session.roomId, 'video', 30000);
const videoProd = await sfu.waitForProducer(session.roomId, 'video', 30000);
- const audioProd = sfu.findProducerByKind(session.roomId, 'audio');
+ let audioProd = sfu.findProducerByKind(session.roomId, 'audio');
+ if (!audioProd) {
+ try { audioProd = await sfu.waitForProducer(session.roomId, 'audio', 4000); }
+ catch { audioProd = null; } // genuinely a video-only stream
+ }
log(sid, `source producers: video=${videoProd.id}${audioProd ? ` audio=${audioProd.id}` : ' (no audio)'}`);
log(sid, `source producers: video=${videoProd.id}${audioProd ? ` audio=${audioProd.id}` : ' (no audio)'}`);
+ // If we ended up without audio, restart the passthrough the moment one appears.
+ if (!audioProd) {
+ const onAudio = ({ roomId, kind }) => {
+ if (session.stopped || kind !== 'audio' || roomId !== session.roomId) return;
+ sfu.removeListener?.('producer-added', onAudio);
+ log(sid, 'audio producer appeared after start — restarting passthrough to include it');
+ const st = { id: sid }; const integ = { robot_id: session.robotId, token: session.token };
+ this._teardown(session);
+ this._scheduleRestart(session, st, integ, 'audio producer added after a video-only start');
+ };
+ sfu.on?.('producer-added', onAudio);
+ session._audioWatch = () => sfu.removeListener?.('producer-added', onAudio);
+ }
+
// 2) Encoded-RTP ingest (PlainTransport → localhost UDP socket).
// 2) Encoded-RTP ingest (PlainTransport → localhost UDP socket).
const videoIn = await openPlainIngest(sfu, session.roomId, videoProd.id);
const videoIn = await openPlainIngest(sfu, session.roomId, videoProd.id);
session.ingests.push(videoIn);
session.ingests.push(videoIn);
@@ -162,6 +162,46 @@ class RestreamManager extends EventEmitter {
@@ -162,6 +162,46 @@ class RestreamManager extends EventEmitter {
if (kind !== 'video') return; // Only restart on video producer loss
if (kind !== 'video') return; // Only restart on video producer loss
this._handleProducerRemoved(roomId, producerId);
this._handleProducerRemoved(roomId, producerId);
});
});
+ // An AUDIO producer appearing after we started matters just as much as one
+ // going away. FFmpeg's argument list is built once, and `hasAudio` is decided
+ // from whether an audio producer existed at that moment — so a broadcaster who
+ // goes live video-only and then switches the microphone on (screen-share mic
+ // toggle, or a mic granted after the fact) would keep pushing SILENT video to
+ // Twitch/Kick for the rest of the session, with nothing in the logs to say why.
+ // Restart the session so the SDP and the -map arguments are rebuilt with audio.
+ webrtcSFU.on('producer-added', ({ roomId, kind }) => {
+ if (kind !== 'audio') return;
+ this._handleAudioProducerAdded(roomId);
+ });
+ }
+
+ /**
+ * An audio producer appeared. Restart any running WebRTC restream for that stream
+ * that was started without audio; leave the ones that already have it alone, since
+ * restarting a healthy session would drop frames on the destination for no reason.
+ */
+ _handleAudioProducerAdded(roomId) {
+ const m = String(roomId).match(/^stream-(\d+)$/);
+ if (!m) return;
+ const streamId = parseInt(m[1], 10);
+ let restarted = 0;
+ for (const [key, session] of this.sessions) {
+ if (session.streamId !== streamId) continue;
+ if (session.streamInfo?.protocol !== 'webrtc') continue;
+ if (session.status === 'stopped') continue;
+ if (session.webrtcState?.hasAudio) continue; // already carrying audio
+ console.log(`[Restream] Audio producer appeared in ${roomId} — restarting ${key} to include it`);
+ this._killProcess(session);
+ session.status = 'error';
+ session.lastError = 'Audio source added';
+ session.restartAttempts = 0;
+ session.restartDelay = RESTART_BASE_DELAY;
+ this._scheduleRestart(session);
+ restarted++;
+ }
+ if (restarted > 0) {
+ console.log(`[Restream] Restarting ${restarted} restream(s) for stream ${streamId} to pick up audio`);
+ }
}
}
/**
/**
@@ -506,6 +546,9 @@ class RestreamManager extends EventEmitter {
@@ -506,6 +546,9 @@ class RestreamManager extends EventEmitter {
sdpPath,
sdpPath,
videoTransportId: videoConsumer.transportId,
videoTransportId: videoConsumer.transportId,
audioTransportId: audioConsumer?.transportId || null,
audioTransportId: audioConsumer?.transportId || null,
+ // Whether THIS ffmpeg invocation was built with an audio mapping. Read by
+ // _handleAudioProducerAdded to avoid restarting sessions that already have it.
+ hasAudio: !!audioConsumer,
};
};
// Step 5: Spawn FFmpeg with SDP input → RTMP output
// Step 5: Spawn FFmpeg with SDP input → RTMP output
new file mode 100644
new file mode 100644
@@ -0,0 +1,86 @@
@@ -0,0 +1,86 @@
+/**
+ * Tranche 2 contract: mic/desktop audio must reach every downstream consumer, and keep
+ * reaching them when the streamer changes something mid-broadcast.
+ *
+ * Every consumer of the broadcast audio binds to a track ONCE and keeps its own
+ * reference — MediaRecorder snapshots its tracks at construction, mediasoup Producers
+ * wrap a specific track, viewer RTCPeerConnection senders hold theirs. So the old
+ * removeTrack/addTrack dance on ss.localStream changed nothing for anyone: toggling the
+ * mic mid-stream left viewers hearing the previous track and the VOD recording it,
+ * silently, for the rest of the session.
+ *
+ * Server side, the same shape of bug existed across process boundaries: ffmpeg's
+ * argument list and the RS relay's werift peer are both built once from whichever
+ * producers existed at that instant, so a broadcaster who went live video-only and then
+ * switched the mic on kept pushing silent video to Twitch/Kick/RobotStreamer.
+ */
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+
+const root = path.join(__dirname, '..');
+const read = (p) => fs.readFileSync(path.join(root, p), 'utf8');
+const bc = read('public/js/broadcast.js');
+const restream = read('server/streaming/restream-manager.js');
+const relay = read('server/integrations/rs-passthrough-relay.js');
+
+// ── A: one mixer, one stable output track ────────────────────────────────────────────
+assert.ok(/function _ensureAudioMixer\(ss\)/.test(bc), 'a shared audio mixer must exist');
+assert.ok(/createMediaStreamDestination\(\)/.test(bc), 'the mixer must publish a MediaStreamDestination track');
+assert.ok(/createConstantSource\(\)/.test(bc),
+ 'the mixer needs a silent keep-alive source, or its output track ends when the last input is removed');
+assert.ok(/function _mixOutputTrack\(ss\)/.test(bc), 'consumers need one accessor for the published track');
+console.log('OK A: mixer exists and its output track is kept alive independently of its inputs');
+
+// ── B: capture publishes the mixer output, not a raw device track ────────────────────
+const captureIdx = bc.indexOf('const finalAudioTrack = _mixOutputTrack(ss);');
+assert.ok(captureIdx > 0, 'screen-share capture must publish the mixer output');
+console.log('OK B: screen-share capture publishes the mixer output');
+
+// ── C: toggling the mic only touches the mixer ───────────────────────────────────────
+const toggle = bc.slice(bc.indexOf('async function toggleScreenShareMic()'),
+ bc.indexOf('function _republishAudioTrack('));
+assert.ok(toggle.includes("_mixSetSource(ss, 'mic', null)"), 'muting must disconnect the mic from the mix');
+assert.ok(toggle.includes("_mixSetSource(ss, 'mic', micStream)"), 'unmuting must connect the mic to the mix');
+assert.ok(!/localStream\.removeTrack\([\s\S]{0,40}\)\s*;\s*[\s\S]{0,80}addTrack\(desktopAudioTracks/.test(toggle),
+ 'the old remove/add track dance must be gone — it never reached any consumer');
+console.log('OK C: mic toggle is a mixer operation, invisible to downstream consumers');
+
+// ── D: when the track identity DOES change, every holder is told ─────────────────────
+const republish = bc.slice(bc.indexOf('function _republishAudioTrack('));
+for (const [what, re] of [
+ ['the local SFU producer', /audioProducer\?\.replaceTrack\(\{ track \}\)/],
+ ['the RobotStreamer producer', /robotStreamer\?\.audioProducer\?\.replaceTrack/],
+ ['viewer peer connections', /getSenders\?\.\(\)[\s\S]{0,120}replaceTrack\(track\)/],
+ ['the VOD recorder', /uploadVodRecording\(streamId, \{ finalizeStream: false \}\)[\s\S]{0,120}startVodRecording\(streamId\)/],
+]) {
+ assert.ok(re.test(republish), `_republishAudioTrack must update ${what}`);
+}
+console.log('OK D: SFU, RobotStreamer, viewer PCs and the VOD segment are all updated on a real track change');
+
+// ── E: the mixer is torn down with the capture ───────────────────────────────────────
+assert.ok(/_teardownAudioMixer\(ss\)/.test(bc), 'the mixer must be closed when capture stops');
+assert.strictEqual((bc.match(/_mixAudioContext/g) || []).length, 0,
+ 'the old per-toggle AudioContext must be fully retired — two mixers would double the audio');
+console.log('OK E: mixer torn down on stop, legacy per-toggle context fully removed');
+
+// ── F: RTMP restreams rebuild when audio shows up late ───────────────────────────────
+assert.ok(/webrtcSFU\.on\('producer-added'/.test(restream),
+ 'the restream manager must watch for producers being ADDED, not only removed');
+assert.ok(/_handleAudioProducerAdded\(roomId\)/.test(restream), 'audio arriving late needs a handler');
+const handler = restream.slice(restream.indexOf('_handleAudioProducerAdded(roomId) {'));
+assert.ok(/webrtcState\?\.hasAudio\) continue/.test(handler),
+ 'sessions that already carry audio must be left alone — restarting them drops frames for nothing');
+assert.ok(/hasAudio: !!audioConsumer/.test(restream),
+ 'the session must record whether its ffmpeg was built with an audio mapping');
+console.log('OK F: Twitch/Kick restreams restart to pick up a microphone enabled after going live');
+
+// ── G: the RobotStreamer relay does the same ─────────────────────────────────────────
+assert.ok(/waitForProducer\(session\.roomId, 'audio', \d+\)/.test(relay),
+ 'the relay should give a slightly-late mic a grace period before declaring the stream video-only');
+assert.ok(/producer-added/.test(relay) && /audio producer appeared after start/.test(relay),
+ 'the relay must restart when audio appears after a video-only start');
+assert.ok(/session\._audioWatch/.test(relay), 'the watcher must be tracked so teardown can remove it');
+console.log('OK G: RobotStreamer relay waits for a late mic, then rebuilds if one appears');
+
+console.log('✅ broadcast audio harmony test passed');