openvibers/openvibe.live Commits

A late camera still reaches viewers and the VOD; without one, the sound is recorded (#467)

OP OpenVibers committed 13 days ago · 4e77dc6749 · 3 files, +124 −25

Stream 2827 (moosely, 2026-10-30): she saw her own camera, but the browser produced only audio (the camera was not
in the stream yet when the server asked to produce, and a track that arrived later was only ever swapped, never
produced). The recorder waited 60 s for video, gave up, and the stream had no VOD at all.
- broadcast.js: producing waits up to 8 s for a camera that is still starting; a camera or microphone that arrives
  later is produced into the same transport (_syncSfuProducerTracks); the streamer is told when viewers are not
  getting the camera; the produce status reports the track counts.
- recorder.js: without a camera after 20 s, the sound is recorded alone (Media #240), and the recording rolls to a
  part with the picture when the camera arrives.

Changes

ddiff --git a/public/js/broadcast.js b/public/js/broadcast.js
iindex d8835750..531b42cd 100644
---- a/public/js/broadcast.js
++++ b/public/js/broadcast.js
@@@ -4078,6 +4078,10 @@ function _waitForSfuMessage(streamId, type, timeoutMs = 15000) {
});
}
++// How long producing waits for a camera that is still starting before it sends audio alone (the camera is produced
++// later as soon as it arrives: _syncSfuProducerTracks).
++const SFU_VIDEO_WAIT_MS = 8000;
++
async function _handleSfuProduceRequest(streamId) {
const ss = getStreamState(streamId);
if (!ss?.localStream || !ss?.signalingWs) {
@@@ -4111,6 +4115,7 @@ async function _handleSfuProduceRequest(streamId) {
const device = new Device();
await device.load({ routerRtpCapabilities: capsMsg.rtpCapabilities });
++ state.device = device;
// Step 3: Create send transport
sendBroadcastSignal(ss, { type: 'sfu-create-transport' });
@@@ -4200,9 +4205,22 @@ async function _handleSfuProduceRequest(streamId) {
}
});
-- // Step 4: Produce video + audio tracks
-- const videoTrack = ss.localStream.getVideoTracks()[0];
-- const audioTrack = ss.localStream.getAudioTracks()[0];
++ // Step 4: Produce video + audio tracks. The server asks for this a second after the broadcaster connects; a
++ // camera still starting then (or re-acquired by the recovery handlers) used to leave the stream audio only for
++ // good: stream 2827 (2026-10-30) showed the streamer her own camera while viewers and the recording had no
++ // picture. So wait a few seconds for a video track, and a track that arrives later is produced by
++ // _syncSfuProducerTracks.
++ const liveStream = () => (getStreamState(streamId) || ss).localStream;
++ let videoTrack = liveStream() && liveStream().getVideoTracks()[0];
++ if (!videoTrack) {
++ console.warn('[SFU Produce] No video track yet: waiting up to ' + (SFU_VIDEO_WAIT_MS / 1000) + ' s for the camera');
++ const until = Date.now() + SFU_VIDEO_WAIT_MS;
++ while (!videoTrack && state.active && Date.now() < until) {
++ await new Promise((r) => setTimeout(r, 250));
++ videoTrack = liveStream() && liveStream().getVideoTracks()[0];
++ }
++ }
++ const audioTrack = liveStream() && liveStream().getAudioTracks()[0];
if (videoTrack) {
const opts = sfuVideoProduceOptions(device, videoTrack);
@@@ -4248,7 +4266,12 @@ async function _handleSfuProduceRequest(streamId) {
console.log('[SFU Produce] Could not log post-produce PC state:', e.message);
}
-- sendBroadcastSignal(ss, { type: 'sfu-produce-status', status: 'ok', detail: `video=${!!state.videoProducer} audio=${!!state.audioProducer}` });
++ const tracks = liveStream() ? `tracks=v${liveStream().getVideoTracks().length}a${liveStream().getAudioTracks().length}` : 'tracks=none';
++ sendBroadcastSignal(ss, { type: 'sfu-produce-status', status: 'ok', detail: `video=${!!state.videoProducer} audio=${!!state.audioProducer} ${tracks}` });
++ if (!state.videoProducer) {
++ // The streamer must know: their preview may still show the camera while viewers see none.
++ toast('Viewers are not getting your camera yet. It will start as soon as the camera does; if it does not, switch the camera off and on.', 'warning');
++ }
} catch (err) {
console.error('[SFU Produce] Failed:', err.message);
sendBroadcastSignal(ss, { type: 'sfu-produce-status', status: 'error', error: err.message });
@@@ -4286,6 +4309,32 @@ async function _syncSfuProducerTracks(streamId) {
if (!state?.active) return;
const videoTrack = ss.localStream.getVideoTracks()[0] || null;
const audioTrack = ss.localStream.getAudioTracks()[0] || null;
++ // A kind with no producer yet (the camera or microphone arrived after producing started) is produced now, so it
++ // reaches viewers and the recording (the server's recorder rolls to a part that has it).
++ if (!state.videoProducer && videoTrack && state.transport && state.device && !state.transport.closed) {
++ try {
++ const opts = sfuVideoProduceOptions(state.device, videoTrack);
++ try { state.videoProducer = await state.transport.produce(opts); } catch (err) {
++ if (!opts.codec) throw err;
++ delete opts.codec;
++ state.videoProducer = await state.transport.produce(opts);
++ }
++ await applyVideoSenderPrefs(state.videoProducer);
++ console.log('[SFU] Video producer created for a camera that arrived late:', state.videoProducer.id);
++ sendBroadcastSignal(ss, { type: 'sfu-produce-status', status: 'ok', detail: 'video=true audio=' + !!state.audioProducer + ' late-video' });
++ toast('Your camera is live now', 'success');
++ } catch (e) {
++ console.warn('[SFU] Late video produce failed:', e.message);
++ }
++ }
++ if (!state.audioProducer && audioTrack && state.transport && !state.transport.closed) {
++ try {
++ state.audioProducer = await state.transport.produce({ track: audioTrack, stopTracks: false });
++ console.log('[SFU] Audio producer created for a microphone that arrived late:', state.audioProducer.id);
++ } catch (e) {
++ console.warn('[SFU] Late audio produce failed:', e.message);
++ }
++ }
if (state.videoProducer && videoTrack) {
try {
await state.videoProducer.replaceTrack({ track: videoTrack });
ddiff --git a/server/streaming/recorder.js b/server/streaming/recorder.js
iindex fc51692f..5fa82064 100644
---- a/server/streaming/recorder.js
++++ b/server/streaming/recorder.js
@@@ -74,7 +74,13 @@ class StreamRecorder {
if (!m) return;
const streamId = Number(m[1]);
const rec = this.activeRecordings.get(streamId);
-- if (!rec || rec.type !== 'rtp' || !rec.videoProducerId || rec._rolling) return;
++ if (!rec || rec.type !== 'rtp' || rec._rolling) return;
++ if (rec.audioOnly) {
++ // The recording has only sound so far: the camera that just arrived rolls it to a part with the picture.
++ if (kind === 'video' && rec.vodId) this._rollRecording(streamId, rec, 'a camera arrived while the recording had only sound');
++ return;
++ }
++ if (!rec.videoProducerId) return;
const room = webrtcSFU.rooms?.get(roomId);
const alive = (id) => { const e = id && room?.producers?.get(id); return !!(e && e.producer && !e.producer.closed); };
if (kind === 'audio') {
@@@ -163,6 +169,9 @@ class StreamRecorder {
}
static get AUDIO_WAIT_MS() { return Number(process.env.VOD_AUDIO_WAIT_MS) || 5000; }
++ // How long a recording waits for the camera before it records the sound alone (it rolls to a part with the picture
++ // when the camera arrives). Was 60 s with no fallback, so a stream whose camera never arrived had no VOD at all.
++ static get VIDEO_WAIT_MS() { return Number(process.env.VOD_VIDEO_WAIT_MS) || 20000; }
/** How often a WebRTC recording asks the broadcaster for a keyframe (webrtc-sfu.js createPlainConsumer); 0 turns it off. */
static get KEYFRAME_SEC() { const v = process.env.VOD_KEYFRAME_SEC; const n = Number(v); return v != null && v !== '' && Number.isFinite(n) && n >= 0 ? n : 4; }
static get DISK_RETRY_DELAY_MS() { return 5 * 60 * 1000; }
@@@ -233,17 +242,26 @@ class StreamRecorder {
const webrtcSFU = require('./webrtc-sfu');
const roomId = `stream-${streamId}`;
-- // Wait for the broadcaster's video producer (up to 60s — same as the old recorder).
-- let videoProducer;
++ // Wait for the broadcaster's video producer. Without one, a stream that has sound records its sound (stream
++ // 2827, 2026-10-30, had no VOD at all: the camera never reached the server, and this gave up), and the
++ // recording rolls to a part with the picture when a camera arrives (_installProducerWatch).
++ let videoProducer = null;
try {
-- videoProducer = await webrtcSFU.waitForProducer(roomId, 'video', 60000);
++ videoProducer = await webrtcSFU.waitForProducer(roomId, 'video', StreamRecorder.VIDEO_WAIT_MS);
} catch {
-- console.warn(`[VOD] No video producer for stream ${streamId} within timeout — recording not started`);
++ videoProducer = null;
++ }
++ if (rec._cancel || !this.activeRecordings.has(streamId)) return;
++ if (!videoProducer && !webrtcSFU.findProducerByKind(roomId, 'audio')) {
++ console.warn(`[VOD] No video or audio producer for stream ${streamId} within timeout: recording not started`);
this.activeRecordings.delete(streamId);
return;
}
-- if (rec._cancel || !this.activeRecordings.has(streamId)) return;
-- rec.videoProducerId = videoProducer.id;
++ if (!videoProducer) {
++ console.warn(`[VOD] No video producer for stream ${streamId} within timeout: recording its sound alone until a camera arrives`);
++ rec.audioOnly = true;
++ }
++ rec.videoProducerId = videoProducer ? videoProducer.id : null;
// A browser produces its audio a moment after its video. Asked once, the instant the video arrived, the answer
// was "no audio" and the whole recording was silent (VOD 5986, 2026-10-25: both producers in the same second).
// So wait a few seconds for it: a stream with no microphone then records video only, and a microphone that
@@@ -275,31 +293,38 @@ class StreamRecorder {
...(kind === 'audio' && pCodec.channels ? { channels: pCodec.channels } : {}),
};
};
-- const video = codecInfo(videoProducer, 'video');
++ const video = videoProducer ? codecInfo(videoProducer, 'video') : null;
const audio = codecInfo(audioProducer, 'audio');
-- if (!video) {
++ if (videoProducer && !video) {
console.warn(`[VOD] Could not resolve video codec info for stream ${streamId} — recording not started`);
this.activeRecordings.delete(streamId);
return;
}
++ if (!video && !audio) {
++ console.warn(`[VOD] Stream ${streamId} has neither a picture nor sound to record — recording not started`);
++ this.activeRecordings.delete(streamId);
++ return;
++ }
await this._createVod(streamId, stream, rec, rec);
if (rec._cancel) return this._abort(streamId, rec);
-- const { videoPort, audioPort } = await media.ingestRtpStart(rec.vodId, { video, audio: audio || undefined });
++ const { videoPort, audioPort } = await media.ingestRtpStart(rec.vodId, { video: video || undefined, audio: audio || undefined });
if (rec._cancel) return this._abort(streamId, rec);
const webrtcState = { roomId, videoTransportId: null, audioTransportId: null };
try {
// Media listens on 127.0.0.1:<port> (RTCP on port+1 — Media allocates its
// ingest pool in rtp/rtcp pairs, mirroring the inherited recorder).
-- const videoConsumer = await webrtcSFU.createPlainConsumer(
-- roomId, videoProducer.id, '127.0.0.1', videoPort, videoPort + 1,
-- { keyframeEverySec: StreamRecorder.KEYFRAME_SEC }
-- );
-- webrtcState.videoTransportId = videoConsumer.transportId;
-- if (video.payloadType !== videoConsumer.payloadType) {
-- console.warn(`[VOD] stream ${streamId}: predicted video PT ${video.payloadType} != consumer PT ${videoConsumer.payloadType} — Media ingest may mis-map`);
++ if (videoProducer && videoPort) {
++ const videoConsumer = await webrtcSFU.createPlainConsumer(
++ roomId, videoProducer.id, '127.0.0.1', videoPort, videoPort + 1,
++ { keyframeEverySec: StreamRecorder.KEYFRAME_SEC }
++ );
++ webrtcState.videoTransportId = videoConsumer.transportId;
++ if (video.payloadType !== videoConsumer.payloadType) {
++ console.warn(`[VOD] stream ${streamId}: predicted video PT ${video.payloadType} != consumer PT ${videoConsumer.payloadType} — Media ingest may mis-map`);
++ }
}
if (audio && audioPort) {
const audioConsumer = await webrtcSFU.createPlainConsumer(
@@@ -317,7 +342,7 @@ class StreamRecorder {
rec.webrtcState = webrtcState;
this._partStarted(stream, rec);
-- console.log(`[VOD] Media RTP ingest started: stream ${streamId} → vod ${rec.vodId} (v:${videoPort}${audioPort ? ` a:${audioPort}` : ''}, ${rec.mode})`);
++ console.log(`[VOD] Media RTP ingest started: stream ${streamId} → vod ${rec.vodId} (${videoPort ? `v:${videoPort}` : 'audio only'}${audioPort ? ` a:${audioPort}` : ''}, ${rec.mode})`);
}
/**
ddiff --git a/test/recorder-audio.test.js b/test/recorder-audio.test.js
iindex 073c5faa..f9bae123 100644
---- a/test/recorder-audio.test.js
++++ b/test/recorder-audio.test.js
@@@ -12,6 +12,7 @@ const path = require('path');
const { EventEmitter } = require('events');
process.env.VOD_AUDIO_WAIT_MS = '300';
++process.env.VOD_VIDEO_WAIT_MS = '300';
// A small SFU: rooms of producers, the real findProducerByKind/waitForProducer behaviour, plain consumers recorded.
const sfu = new EventEmitter();
@@@ -53,7 +54,7 @@ db.resolveStreamVodVisibility = async () => 'public';
let vodSeq = 6000;
const ingests = [];
media.createVod = async () => ({ id: ++vodSeq });
--media.ingestRtpStart = async (vodId, { video, audio }) => { ingests.push({ vodId, video, audio }); return { videoPort: 12000, audioPort: audio ? 12002 : null }; };
++media.ingestRtpStart = async (vodId, { video, audio }) => { ingests.push({ vodId, video, audio }); return { videoPort: video ? 12000 : null, audioPort: audio ? 12002 : null }; };
media.ingestRtpStop = async () => {};
media.finalizeVod = async () => {};
const wait = (ms) => new Promise((r) => setTimeout(r, ms));
@@@ -101,7 +102,31 @@ const until = async (fn, ms = 3000) => { const end = Date.now() + ms; while (!fn
assert.strictEqual(recorder.activeRecordings.get(102).part, 3);
console.log('✅ a new microphone after the recorded one died rolls too');
-- for (const id of [101, 102]) recorder.stopRecording(id);
-- console.log('\n✅ recorder audio: waits for the microphone, video-only without one, rolls when one comes later');
++ // 5. Stream 2827 (2026-10-30): the camera never reached the server. The sound is recorded on its own instead of
++ // nothing at all, and when the camera arrives the recording rolls to a part with the picture.
++ sfu.produce('stream-103', 'audio');
++ recorder.startRecording(103, 'webrtc', {}, { mode: 'vod' });
++ await until(() => ingests.length === 5, 3000);
++ assert.strictEqual(ingests[4].video, undefined, 'no camera: no video section asked of Media');
++ assert.ok(ingests[4].audio && ingests[4].audio.codec === 'opus', 'but the sound is recorded');
++ assert.strictEqual(recorder.activeRecordings.get(103).audioOnly, true);
++ assert.deepStrictEqual(sfu.consumers.filter((c) => c.roomId === 'stream-103').map((c) => c.producerId.split('-')[0]), ['audio'], 'one plain consumer, for the sound');
++ console.log('✅ no camera: the sound is recorded instead of nothing');
++ sfu.produce('stream-103', 'video');
++ await until(() => ingests.length === 6, 4000);
++ assert.ok(ingests[5].video && ingests[5].audio, 'the camera arrived: the next part has the picture and the sound');
++ assert.strictEqual(recorder.activeRecordings.get(103).part, 2);
++ assert.ok(!recorder.activeRecordings.get(103).audioOnly, 'and is no longer sound only');
++ console.log('✅ a camera that arrives late rolls the recording to a part with the picture');
++
++ // 6. Neither picture nor sound: nothing is recorded, and nothing is left waiting.
++ recorder.startRecording(104, 'webrtc', {}, { mode: 'vod' });
++ await wait(500);
++ assert.strictEqual(ingests.length, 6, 'nothing to record, no VOD');
++ assert.ok(!recorder.activeRecordings.has(104));
++ console.log('✅ neither picture nor sound: no recording');
++
++ for (const id of [101, 102, 103]) recorder.stopRecording(id);
++ console.log('\n✅ recorder audio: waits for the microphone, video-only without one, sound-only without a camera, rolls when either comes later');
process.exit(0);
})().catch((err) => { console.error(err); process.exit(1); });