openvibers/openvibe.bot Commits

Bot: the live video runs on pages without a robot panel (the machine screen page never played) (#116)

OP OpenVibers committed 29 days ago · dae93dea53 · 2 files, +289 −174

Fix for the live screen (Bot#115), found by the production proof. On `/machines/:id/screen` the player stayed on "Waiting for the machine…" even though the machine was publishing and `screen.json` already answered `live` with a viewer URL.

**Cause:** `public/panel.js` stops at `if (!main) return;` on a page without `#panel`, and the live-video code (the mediasoup loader, `viewerWait`, `liveCamera`) came after that line. Only a robot's panel has `#panel`, so the screen page's poller never ran.

**Fix:** the live-video block moves above the `#panel` check, unchanged. It uses only `$`/`$$`, so robot panels and embeds play their cameras as before.

**Test:** `test/panel-screen-client.test.js` runs the real panel.js in a VM against a minimal DOM with no `#panel`:
- the tile polls `screen.json`, then opens the viewer socket to the session's signaling URL with a video element;
- an offline machine says so;
- a machine that stops says the screen is no longer shown, hides the video, closes the socket and stops polling.

The test fails on the old panel.js and passes now. The panel, embed-panel, embed-drive, try-sim, device-safety and machine-screen tests pass.

Changes

ddiff --git a/public/panel.js b/public/panel.js
iindex ed762eb..193655e 100644
---- a/public/panel.js
++++ b/public/panel.js
@@@ -75,6 +75,182 @@
return;
}
++ // The live video runs on any page with a tile (a robot's panel and embed, a machine's live screen), so it comes
++ // before the panel's own wiring, which needs #panel.
++ // ── Live camera: the robot's OpenRestream WebRTC session ────────────────────────────────────────
++ // A tile the server marked with a live session's viewer signaling URL plays it: the mediasoup-client
++ // `sfu-viewer-*` flow OpenVibe.Live's watch page runs against OpenRestream's /w/<session id> signaling
++ // (watch → ready → recv transport → consume). mediasoup-client's package ships no browser bundle, so
++ // the pinned ESM build Live loads comes from esm.sh — the panel's own CSP allows exactly that origin.
++ // A tile with no URL is left alone; one whose session ends keeps its placeholder and reconnects with
++ // the same backoff as /control.
++ const MEDIASOUP_URL = 'https://esm.sh/[email protected]';
++ let mediasoup = null;
++ const loadMediasoup = () => (mediasoup ||= import(MEDIASOUP_URL).then(
++ (m) => m.Device || (m.default && m.default.Device),
++ (e) => { mediasoup = null; throw e; }));
++
++ /** The first `type` frame on an OpenRestream viewer socket; a viewer error, a close or silence rejects. */
++ function viewerWait(ws, type, timeoutMs = 15000) {
++ return new Promise((resolve, reject) => {
++ if (ws.readyState !== WebSocket.OPEN) return reject(new Error('the camera signaling socket closed'));
++ let timer;
++ const done = (finish) => {
++ clearTimeout(timer);
++ ws.removeEventListener('message', onMessage);
++ ws.removeEventListener('close', onClose);
++ finish();
++ };
++ const onMessage = (e) => {
++ let m; try { m = JSON.parse(e.data); } catch { return; }
++ if (m.type === type) done(() => resolve(m));
++ else if (m.type === 'watch-queued') done(() => reject(new Error('nothing is publishing yet')));
++ else if (m.type === 'sfu-error' || m.type === 'sfu-viewer-error') done(() => reject(new Error(m.error || m.type)));
++ };
++ const onClose = () => done(() => reject(new Error('the camera signaling socket closed')));
++ timer = setTimeout(() => done(() => reject(new Error(`no ${type} from the camera`))), timeoutMs);
++ ws.addEventListener('message', onMessage);
++ ws.addEventListener('close', onClose);
++ });
++ }
++
++ function liveCamera(tile) {
++ const url = tile.dataset.signalingUrl;
++ const screen = $('[data-camera-screen]', tile);
++ if (!url || !screen) return;
++ const placeholder = [...screen.children];
++ const video = document.createElement('video');
++ video.muted = true;
++ video.autoplay = true;
++ video.playsInline = true;
++ video.hidden = true;
++ screen.appendChild(video);
++ let current = null; // the session being connected: { ws, transport, consumers }
++ let timer = null; // the pending reconnect
++ let attempt = 0;
++ let stopped = false; // stop(): no more reconnects (a machine's screen that ended)
++
++ /** The tile shows the video or its placeholder, never both. */
++ function showLive(on) {
++ video.hidden = !on;
++ for (const el of placeholder) el.hidden = on;
++ }
++ function teardown() {
++ const session = current;
++ current = null;
++ video.srcObject = null;
++ if (!session) return;
++ for (const c of session.consumers) { try { c.close(); } catch { /* gone */ } }
++ if (session.transport) { try { session.transport.close(); } catch { /* gone */ } }
++ if (session.ws) { try { session.ws.close(); } catch { /* gone */ } }
++ }
++ /** The placeholder returns, the session closes, and the next attempt is a backoff away. */
++ function retry() {
++ if (timer || stopped) return;
++ showLive(false);
++ teardown();
++ timer = setTimeout(() => { timer = null; connect(); }, Math.min(10000, 500 * 2 ** attempt++));
++ }
++
++ async function connect() {
++ const session = { ws: null, transport: null, consumers: [] };
++ const stale = () => current !== session;
++ current = session;
++ try {
++ const ws = new WebSocket(url);
++ session.ws = ws;
++ await new Promise((open, fail) => {
++ ws.addEventListener('open', open, { once: true });
++ ws.addEventListener('close', () => fail(new Error('the camera signaling socket closed')), { once: true });
++ });
++ if (stale()) return;
++ ws.send(JSON.stringify({ type: 'watch' }));
++ const ready = await viewerWait(ws, 'sfu-viewer-ready');
++ if (stale()) return;
++ if (!ready.producers || !ready.producers.length) throw new Error('the camera has no live track yet');
++ const Device = await loadMediasoup();
++ if (stale()) return;
++ const device = new Device();
++ await device.load({ routerRtpCapabilities: ready.rtpCapabilities });
++ if (stale()) return;
++ ws.send(JSON.stringify({ type: 'sfu-viewer-create-transport' }));
++ const created = await viewerWait(ws, 'sfu-viewer-transport-created');
++ if (stale()) return;
++ const transport = device.createRecvTransport({
++ id: created.id, iceParameters: created.iceParameters, iceCandidates: created.iceCandidates,
++ dtlsParameters: created.dtlsParameters, iceServers: created.iceServers || [],
++ });
++ session.transport = transport;
++ transport.on('connect', ({ dtlsParameters }, cb, errb) => {
++ if (stale()) return errb(new Error('the camera session is gone'));
++ ws.send(JSON.stringify({ type: 'sfu-viewer-connect-transport', transportId: transport.id, dtlsParameters }));
++ viewerWait(ws, 'sfu-viewer-transport-connected').then(() => cb(), () => errb(new Error('the camera transport did not connect')));
++ });
++ const media = new MediaStream();
++ for (const producer of ready.producers) {
++ ws.send(JSON.stringify({ type: 'sfu-viewer-consume', transportId: transport.id, producerId: producer.id, rtpCapabilities: device.rtpCapabilities }));
++ const consumed = await viewerWait(ws, 'sfu-viewer-consumed');
++ if (stale()) return;
++ const consumer = await transport.consume({ id: consumed.id, producerId: consumed.producerId, kind: consumed.kind, rtpParameters: consumed.rtpParameters });
++ session.consumers.push(consumer);
++ media.addTrack(consumer.track);
++ }
++ if (stale()) return;
++ video.srcObject = media;
++ showLive(true);
++ attempt = 0;
++ await video.play().catch(() => { /* autoplay holds the track anyway */ });
++ ws.addEventListener('close', () => { if (!stale()) retry(); });
++ } catch (e) {
++ if (!stale()) retry();
++ }
++ }
++ connect();
++ return {
++ stop() {
++ stopped = true;
++ if (timer) { clearTimeout(timer); timer = null; }
++ showLive(false);
++ teardown();
++ },
++ };
++ }
++ for (const tile of $$('[data-signaling-url]')) liveCamera(tile);
++
++ // ── A machine's live screen (/machines/:id/screen): wait until the machine publishes, then play it as a camera ──
++ for (const time of $$('time[data-local-time]')) {
++ const at = new Date(time.getAttribute('datetime'));
++ if (!Number.isNaN(at.getTime())) time.textContent = at.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
++ }
++ for (const tile of $$('[data-screen-state]')) {
++ const status = $('[data-screen-status]', tile);
++ let playing = null;
++ let misses = 0;
++ const say = (text) => { if (status) status.textContent = text; };
++ const poll = async () => {
++ let s = null;
++ try {
++ const r = await fetch(tile.dataset.screenState, { headers: { Accept: 'application/json' }, credentials: 'same-origin', cache: 'no-store' });
++ s = r.ok ? await r.json() : null;
++ } catch { s = null; }
++ if (s && s.signaling_url && !playing) {
++ tile.dataset.signalingUrl = s.signaling_url;
++ say('Connecting to the screen…');
++ playing = liveCamera(tile);
++ } else if (playing && s && s.state === 'off') {
++ // The machine stopped (its time is up, Stop, or its kill switch): leave the last word on the page.
++ playing.stop();
++ say('Your screen is no longer shown.');
++ return;
++ } else if (!playing) {
++ misses += 1;
++ say(s && s.online === false ? 'The machine is offline.' : misses > 20 ? 'Still waiting for the machine. Is the Node running in your desktop session?' : 'Waiting for the machine…');
++ }
++ setTimeout(poll, playing ? 5000 : 1500);
++ };
++ if (status && !$('.error', document)) poll();
++ }
++
const main = document.getElementById('panel');
if (!main) return;
const ROBOT_ID = main.dataset.robotId;
@@@ -504,180 +680,6 @@
send('point', { x: (e.clientX - left) / width, y: (e.clientY - top) / height });
});
-- // ── Live camera: the robot's OpenRestream WebRTC session ────────────────────────────────────────
-- // A tile the server marked with a live session's viewer signaling URL plays it: the mediasoup-client
-- // `sfu-viewer-*` flow OpenVibe.Live's watch page runs against OpenRestream's /w/<session id> signaling
-- // (watch → ready → recv transport → consume). mediasoup-client's package ships no browser bundle, so
-- // the pinned ESM build Live loads comes from esm.sh — the panel's own CSP allows exactly that origin.
-- // A tile with no URL is left alone; one whose session ends keeps its placeholder and reconnects with
-- // the same backoff as /control.
-- const MEDIASOUP_URL = 'https://esm.sh/[email protected]';
-- let mediasoup = null;
-- const loadMediasoup = () => (mediasoup ||= import(MEDIASOUP_URL).then(
-- (m) => m.Device || (m.default && m.default.Device),
-- (e) => { mediasoup = null; throw e; }));
--
-- /** The first `type` frame on an OpenRestream viewer socket; a viewer error, a close or silence rejects. */
-- function viewerWait(ws, type, timeoutMs = 15000) {
-- return new Promise((resolve, reject) => {
-- if (ws.readyState !== WebSocket.OPEN) return reject(new Error('the camera signaling socket closed'));
-- let timer;
-- const done = (finish) => {
-- clearTimeout(timer);
-- ws.removeEventListener('message', onMessage);
-- ws.removeEventListener('close', onClose);
-- finish();
-- };
-- const onMessage = (e) => {
-- let m; try { m = JSON.parse(e.data); } catch { return; }
-- if (m.type === type) done(() => resolve(m));
-- else if (m.type === 'watch-queued') done(() => reject(new Error('nothing is publishing yet')));
-- else if (m.type === 'sfu-error' || m.type === 'sfu-viewer-error') done(() => reject(new Error(m.error || m.type)));
-- };
-- const onClose = () => done(() => reject(new Error('the camera signaling socket closed')));
-- timer = setTimeout(() => done(() => reject(new Error(`no ${type} from the camera`))), timeoutMs);
-- ws.addEventListener('message', onMessage);
-- ws.addEventListener('close', onClose);
-- });
-- }
--
-- function liveCamera(tile) {
-- const url = tile.dataset.signalingUrl;
-- const screen = $('[data-camera-screen]', tile);
-- if (!url || !screen) return;
-- const placeholder = [...screen.children];
-- const video = document.createElement('video');
-- video.muted = true;
-- video.autoplay = true;
-- video.playsInline = true;
-- video.hidden = true;
-- screen.appendChild(video);
-- let current = null; // the session being connected: { ws, transport, consumers }
-- let timer = null; // the pending reconnect
-- let attempt = 0;
-- let stopped = false; // stop(): no more reconnects (a machine's screen that ended)
--
-- /** The tile shows the video or its placeholder, never both. */
-- function showLive(on) {
-- video.hidden = !on;
-- for (const el of placeholder) el.hidden = on;
-- }
-- function teardown() {
-- const session = current;
-- current = null;
-- video.srcObject = null;
-- if (!session) return;
-- for (const c of session.consumers) { try { c.close(); } catch { /* gone */ } }
-- if (session.transport) { try { session.transport.close(); } catch { /* gone */ } }
-- if (session.ws) { try { session.ws.close(); } catch { /* gone */ } }
-- }
-- /** The placeholder returns, the session closes, and the next attempt is a backoff away. */
-- function retry() {
-- if (timer || stopped) return;
-- showLive(false);
-- teardown();
-- timer = setTimeout(() => { timer = null; connect(); }, Math.min(10000, 500 * 2 ** attempt++));
-- }
--
-- async function connect() {
-- const session = { ws: null, transport: null, consumers: [] };
-- const stale = () => current !== session;
-- current = session;
-- try {
-- const ws = new WebSocket(url);
-- session.ws = ws;
-- await new Promise((open, fail) => {
-- ws.addEventListener('open', open, { once: true });
-- ws.addEventListener('close', () => fail(new Error('the camera signaling socket closed')), { once: true });
-- });
-- if (stale()) return;
-- ws.send(JSON.stringify({ type: 'watch' }));
-- const ready = await viewerWait(ws, 'sfu-viewer-ready');
-- if (stale()) return;
-- if (!ready.producers || !ready.producers.length) throw new Error('the camera has no live track yet');
-- const Device = await loadMediasoup();
-- if (stale()) return;
-- const device = new Device();
-- await device.load({ routerRtpCapabilities: ready.rtpCapabilities });
-- if (stale()) return;
-- ws.send(JSON.stringify({ type: 'sfu-viewer-create-transport' }));
-- const created = await viewerWait(ws, 'sfu-viewer-transport-created');
-- if (stale()) return;
-- const transport = device.createRecvTransport({
-- id: created.id, iceParameters: created.iceParameters, iceCandidates: created.iceCandidates,
-- dtlsParameters: created.dtlsParameters, iceServers: created.iceServers || [],
-- });
-- session.transport = transport;
-- transport.on('connect', ({ dtlsParameters }, cb, errb) => {
-- if (stale()) return errb(new Error('the camera session is gone'));
-- ws.send(JSON.stringify({ type: 'sfu-viewer-connect-transport', transportId: transport.id, dtlsParameters }));
-- viewerWait(ws, 'sfu-viewer-transport-connected').then(() => cb(), () => errb(new Error('the camera transport did not connect')));
-- });
-- const media = new MediaStream();
-- for (const producer of ready.producers) {
-- ws.send(JSON.stringify({ type: 'sfu-viewer-consume', transportId: transport.id, producerId: producer.id, rtpCapabilities: device.rtpCapabilities }));
-- const consumed = await viewerWait(ws, 'sfu-viewer-consumed');
-- if (stale()) return;
-- const consumer = await transport.consume({ id: consumed.id, producerId: consumed.producerId, kind: consumed.kind, rtpParameters: consumed.rtpParameters });
-- session.consumers.push(consumer);
-- media.addTrack(consumer.track);
-- }
-- if (stale()) return;
-- video.srcObject = media;
-- showLive(true);
-- attempt = 0;
-- await video.play().catch(() => { /* autoplay holds the track anyway */ });
-- ws.addEventListener('close', () => { if (!stale()) retry(); });
-- } catch (e) {
-- if (!stale()) retry();
-- }
-- }
-- connect();
-- return {
-- stop() {
-- stopped = true;
-- if (timer) { clearTimeout(timer); timer = null; }
-- showLive(false);
-- teardown();
-- },
-- };
-- }
-- for (const tile of $$('[data-signaling-url]')) liveCamera(tile);
--
-- // ── A machine's live screen (/machines/:id/screen): wait until the machine publishes, then play it as a camera ──
-- for (const time of $$('time[data-local-time]')) {
-- const at = new Date(time.getAttribute('datetime'));
-- if (!Number.isNaN(at.getTime())) time.textContent = at.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
-- }
-- for (const tile of $$('[data-screen-state]')) {
-- const status = $('[data-screen-status]', tile);
-- let playing = null;
-- let misses = 0;
-- const say = (text) => { if (status) status.textContent = text; };
-- const poll = async () => {
-- let s = null;
-- try {
-- const r = await fetch(tile.dataset.screenState, { headers: { Accept: 'application/json' }, credentials: 'same-origin', cache: 'no-store' });
-- s = r.ok ? await r.json() : null;
-- } catch { s = null; }
-- if (s && s.signaling_url && !playing) {
-- tile.dataset.signalingUrl = s.signaling_url;
-- say('Connecting to the screen…');
-- playing = liveCamera(tile);
-- } else if (playing && s && s.state === 'off') {
-- // The machine stopped (its time is up, Stop, or its kill switch): leave the last word on the page.
-- playing.stop();
-- say('Your screen is no longer shown.');
-- return;
-- } else if (!playing) {
-- misses += 1;
-- say(s && s.online === false ? 'The machine is offline.' : misses > 20 ? 'Still waiting for the machine. Is the Node running in your desktop session?' : 'Waiting for the machine…');
-- }
-- setTimeout(poll, playing ? 5000 : 1500);
-- };
-- if (status && !$('.error', document)) poll();
-- }
--
// ── State ─────────────────────────────────────────────────────────────────────────────────────────
function enableControls() {
for (const section of $$('[data-command]')) {
ddiff --git a/test/panel-screen-client.test.js b/test/panel-screen-client.test.js
nnew file mode 100644
iindex 0000000..2f1971c
---- /dev/null
++++ b/test/panel-screen-client.test.js
@@@ -0,0 +1,113 @@
++'use strict';
++// public/panel.js on a page with no robot panel (a machine's live screen, /machines/:id/screen): the tile polls the
++// machine's screen.json until the machine publishes, then opens the session's viewer signaling socket; once the
++// machine stops, it says so and stops reconnecting. The live video must not wait for #panel, which only a robot's
++// panel has (2026-10-15: it did, and the screen page stayed on "Waiting for the machine…").
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const vm = require('vm');
++const { check, done } = require('./helpers/app');
++
++const SOURCE = fs.readFileSync(path.join(__dirname, '..', 'public', 'panel.js'), 'utf8');
++
++/** Just enough DOM to run panel.js on the screen page: one tile, its placeholder lines and its status line. */
++function screenPage(states) {
++ const el = (tag, attrs = {}) => ({
++ tagName: tag.toUpperCase(), dataset: { ...attrs }, hidden: false, textContent: '', children: [], parentElement: null,
++ appendChild(c) { this.children.push(c); c.parentElement = this; return c; },
++ remove() { if (this.parentElement) this.parentElement.children = this.parentElement.children.filter((x) => x !== this); },
++ addEventListener() {}, setAttribute() {}, removeAttribute() {}, querySelector() { return null; }, querySelectorAll() { return []; },
++ play() { return Promise.resolve(); },
++ });
++ const status = el('p');
++ status.textContent = 'Waiting for the machine…';
++ const screen = el('div');
++ screen.appendChild(el('svg'));
++ screen.appendChild(status);
++ const tile = el('figure', { screenState: '/machines/dev_1/screen.json' });
++ tile.appendChild(screen);
++ const find = (root, sel) => {
++ if (sel === '[data-camera-screen]') return screen;
++ if (sel === '[data-screen-status]') return status;
++ if (sel === 'video') return screen.children.find((c) => c.tagName === 'VIDEO') || null;
++ return null;
++ };
++ tile.querySelector = (sel) => find(tile, sel);
++ const document = {
++ getElementById: () => null,
++ querySelector: (sel) => (sel === '[data-screen-state]' ? tile : find(document, sel)),
++ querySelectorAll: (sel) => (sel === '[data-screen-state]' ? [tile] : []),
++ createElement: (tag) => el(tag),
++ addEventListener() {}, hidden: false,
++ };
++ const sockets = [];
++ class WebSocket {
++ constructor(url) { this.url = url; this.readyState = 0; this.listeners = {}; sockets.push(this); }
++ addEventListener(type, fn) { (this.listeners[type] ||= []).push(fn); }
++ send() {}
++ close() { this.closed = true; }
++ }
++ WebSocket.OPEN = 1;
++ const polled = [];
++ const timers = [];
++ const ctx = {
++ document, WebSocket, console, Promise, JSON, Math, Date, URL, Number, String, Object, Array, Set, Map, Error,
++ location: { protocol: 'https:', host: 'openvibe.bot' }, navigator: {},
++ MediaStream: class {}, window: {},
++ fetch: async (url) => { polled.push(url); const s = states.shift() || { live: false, state: 'off', online: true }; return { ok: true, json: async () => s }; },
++ setTimeout: (fn, ms) => { timers.push({ fn, ms }); return timers.length; },
++ clearTimeout: () => {},
++ addEventListener() {}, requestAnimationFrame() {},
++ };
++ ctx.window = ctx;
++ vm.createContext(ctx);
++ vm.runInContext(SOURCE, ctx, { filename: 'panel.js' });
++ const settle = () => new Promise((r) => setImmediate(r));
++ // tick: the next scheduled poll runs (the page's own timers fire only here), then its fetch settles.
++ const tick = async () => { const due = timers.splice(0); for (const t of due) t.fn(); await settle(); await settle(); };
++ return { status, sockets, polled, tick, settle, screen };
++}
++
++(async () => {
++ await check('the screen page waits, then opens the session\'s viewer socket — with no #panel on the page', async () => {
++ const page = screenPage([
++ { live: false, signaling_url: null, state: 'off', online: true },
++ { live: true, signaling_url: 'wss://ingest.openre.stream/w/ses_X', state: 'live', online: true },
++ ]);
++ await page.settle();
++ await page.settle();
++ assert.deepStrictEqual(page.polled, ['/machines/dev_1/screen.json'], 'it asks at once');
++ assert.strictEqual(page.status.textContent, 'Waiting for the machine…');
++ assert.strictEqual(page.sockets.length, 0);
++ await page.tick();
++ assert.strictEqual(page.polled.length, 2);
++ assert.strictEqual(page.status.textContent, 'Connecting to the screen…');
++ assert.deepStrictEqual(page.sockets.map((s) => s.url), ['wss://ingest.openre.stream/w/ses_X'], 'the viewer socket, to the session\'s signaling URL');
++ assert.ok(page.screen.children.some((c) => c.tagName === 'VIDEO'), 'a video element in the tile');
++ });
++
++ await check('an offline machine says so; one that stops says the screen is no longer shown and stops polling', async () => {
++ const offline = screenPage([{ live: false, signaling_url: null, state: 'off', online: false }]);
++ await offline.settle();
++ await offline.settle();
++ assert.strictEqual(offline.status.textContent, 'The machine is offline.');
++ const page = screenPage([
++ { live: true, signaling_url: 'wss://ingest.openre.stream/w/ses_Y', state: 'live', online: true },
++ { live: false, signaling_url: null, state: 'off', online: true },
++ ]);
++ await page.settle();
++ await page.settle();
++ assert.strictEqual(page.status.textContent, 'Connecting to the screen…');
++ await page.tick();
++ assert.strictEqual(page.status.textContent, 'Your screen is no longer shown.');
++ const asked = page.polled.length;
++ await page.tick();
++ await page.tick();
++ assert.strictEqual(page.polled.length, asked, 'no more polling');
++ assert.ok(page.screen.children.filter((c) => c.tagName === 'VIDEO').every((c) => c.hidden), 'the video is hidden, the placeholder back');
++ assert.ok(page.sockets.every((s) => s.closed), 'the viewer socket is closed');
++ });
++
++ done();
++})();