openvibers/openvibe.bot Commits
Watch and take over an agent's desk (Desks plan D4): a task's link opens its desk on the owner's screen page, and the owner's mouse and keyboard go to the desk shown, as the owner (#177)
Changes
@@@ -7,6 +7,9 @@const main = document.querySelector('main[data-share]');if (!main || main.dataset.control !== '1') return;const base = main.dataset.share;++ // The owner's own screen page (data-owner): signed in by cookie, same site; a share link: the token only.++ const credentials = main.dataset.owner === '1' ? 'same-origin' : 'omit';++ const desk = new URLSearchParams(location.search).get('desk');const fig = main.querySelector('[data-screen-state]');const use = main.querySelector('[data-share-use]');const startBtn = main.querySelector('[data-share-start]');@@@ -15,7 +18,7 @@const say = (t) => { if (note) note.textContent = t || ''; };async function post(path, body) {-- const r = await fetch(base + path, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify(body || {}), credentials: 'omit' });++ const r = await fetch(base + path, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify(body || {}), credentials });const j = await r.json().catch(() => ({}));if (!r.ok) throw new Error(j.detail || ('the computer answered ' + r.status));return j;@@@ -24,7 +27,7 @@// Start when the screen is off; the state comes from the same JSON panel.js reads.async function refresh() {try {-- const r = await fetch(base + '/state.json', { headers: { Accept: 'application/json' }, cache: 'no-store', credentials: 'omit' });++ const r = await fetch(base + '/state.json', { headers: { Accept: 'application/json' }, cache: 'no-store', credentials });const s = r.ok ? await r.json() : null;if (startBtn) startBtn.hidden = !s || s.live || !s.online;if (s && !s.online) say('The computer is offline.');@@@ -34,7 +37,7 @@setInterval(refresh, 5000);if (startBtn) startBtn.addEventListener('click', async () => {startBtn.disabled = true; say('Starting the screen…');-- try { await post('/start'); say('Starting: the picture appears in a few seconds.'); } catch (e) { say(e.message); } finally { startBtn.disabled = false; refresh(); }++ try { await post('/start', desk ? { desk } : {}); say('Starting: the picture appears in a few seconds.'); } catch (e) { say(e.message); } finally { startBtn.disabled = false; refresh(); }});if (stopBtn) stopBtn.addEventListener('click', async () => {try { await post('/stop'); say('The screen is stopped.'); } catch (e) { say(e.message); } finally { refresh(); }@@@ -20,6 +20,26 @@const { BotError } = require('./util');const SECONDS = { min: 30, max: 1800, default: 600 };++const DESK = /^(main|private|desk-[a-z0-9][a-z0-9-]{0,23})$/;++const ACTIONS = new Set(['click', 'double_click', 'right_click', 'move', 'scroll', 'type', 'key']);++++/** A person's mouse or keyboard action on a shown screen, as bot.command@1 input takes it (share links and the owner). */++function inputValue(value) {++ const v = value && typeof value === 'object' ? value : {};++ if (!ACTIONS.has(v.action)) throw new BotError(422, 'bot.invalid_input', `action is one of ${[...ACTIONS].join(', ')}`);++ const out = { action: v.action };++ if (['click', 'double_click', 'right_click', 'move', 'scroll'].includes(v.action) && v.x != null) {++ for (const k of ['x', 'y', 'width', 'height']) {++ const n = Math.round(Number(v[k]));++ if (!Number.isFinite(n) || n < 0 || n > 10000) throw new BotError(422, 'bot.invalid_input', `${k} is a position in the picture`);++ out[k] = n;++ }++ }++ if (v.action === 'scroll') out.dy = Math.max(-50, Math.min(50, Math.round(Number(v.dy) || 0)));++ if (v.action === 'type') out.text = String(v.text || '').slice(0, 500);++ if (v.action === 'key') out.keys = String(v.keys || '').slice(0, 40);++ return out;++}const FPS = { min: 1, max: 5, default: 2 };function whole(v, { min, max, default: dflt }, name) {@@@ -31,14 +51,19 @@ function whole(v, { min, max, default: dflt }, name) {function createMachineScreen({ domain, hub, now = () => Date.now() }) {const unavailable = () => new BotError(503, 'bot.unavailable', 'machines are not connected here');++ // The desk each machine's live screen shows (this process): the owner's input goes there, never to another screen.++ const shown = new Map();++ const owner = (subject) => ({ kind: 'user', sub: subject });++ const rate = new Map();return {++ DESK,limits: { SECONDS, FPS },async start(device, subject, principal, opts = {}) {if (!hub) throw unavailable();const seconds = whole(opts.seconds, SECONDS, 'seconds');const fps = whole(opts.fps, FPS, 'fps');const desk = opts.desk == null || opts.desk === '' ? 'main' : String(opts.desk);-- if (desk !== 'main' && !/^(private|desk-[a-z0-9][a-z0-9-]{0,23})$/.test(desk)) throw new BotError(422, 'bot.invalid_input', 'desk is main, private or desk-<name>');++ if (!DESK.test(desk)) throw new BotError(422, 'bot.invalid_input', 'desk is main, private or desk-<name>');if (!hub.isOnline(device.id)) throw new BotError(409, 'bot.device_offline', 'the machine is offline');// Before any key is made: the machine says whether it can (Node v0.12.0's computer plugin, a capture tool).const offered = hub.computerOf(device.id);@@@ -60,13 +85,32 @@ function createMachineScreen({ domain, hub, now = () => Date.now() }) {}const data = r.data && typeof r.data === 'object' ? r.data : {};const until = Number.isFinite(data.until_ms) ? new Date(data.until_ms).toISOString() : new Date(now() + seconds * 1000).toISOString();++ shown.set(device.id, desk);return { streaming: true, seconds, fps, until, desk };},async state(device, subject) {const sensors = hub ? ((hub.deviceState(device.id).telemetry || {}).sensors || {}) : {};const state = sensors.computer_screen_stream === 'live' ? 'live' : 'off';const url = state === 'live' ? await domain.machines.screen.live(device, subject) : null;-- return { live: !!url, signaling_url: url, state, online: !!(hub && hub.isOnline(device.id)) };++ return { live: !!url, signaling_url: url, state, online: !!(hub && hub.isOnline(device.id)), desk: shown.get(device.id) || 'main' };++ },++ /**++ * The owner's own mouse or keyboard on the screen shown now (Desks plan D4, "Take over"): sent as the owner, so++ * Bot's person-first rule holds an agent off that screen while they use it, and it goes on when they stop.++ */++ async input(device, subject, value) {++ if (!hub) throw unavailable();++ const out = inputValue(value);++ const desk = shown.get(device.id) || 'main';++ if (desk !== 'main') out.desk = desk;++ const recent = (rate.get(device.id) || []).filter((x) => now() - x < 1000);++ if (recent.length >= 8) throw new BotError(429, 'bot.rate_limited', 'slow down a little');++ recent.push(now());++ rate.set(device.id, recent);++ const r = await hub.machineCommand(device.id, owner(subject), subject, { kind: 'input', value: out, ms: 15000 });++ if (r && r.result === 'ack') return { done: true, desk };++ const code = (r && r.code) || 'bot.device_refused';++ throw new BotError(r && r.result === 'expired' ? 504 : 409, code.startsWith('bot.') ? code : 'bot.device_refused', `the computer did not do it: ${(r && (r.reason || r.result)) || 'no answer'}`.slice(0, 300));},async stop(device, subject, principal) {let told = false;@@@ -75,9 +119,10 @@ function createMachineScreen({ domain, hub, now = () => Date.now() }) {told = r.result === 'ack';}const ended = await domain.machines.screen.end(device, subject);++ shown.delete(device.id);return { stopped: told || ended };},};}--module.exports = { createMachineScreen, SECONDS, FPS };++module.exports = { createMachineScreen, inputValue, ACTIONS, SECONDS, FPS, DESK };@@@ -23,9 +23,9 @@* move, scroll, type, key) → the machine's answer*/const { BotError, prefixedId, sha256, token: newToken } = require('./util');++const { inputValue, ACTIONS } = require('./machine-screen');const DAYS = { min: 1, max: 365, default: 30 };--const ACTIONS = new Set(['click', 'double_click', 'right_click', 'move', 'scroll', 'type', 'key']);const PER_SECOND = 8; // a person clicks and types a few times a second, not morefunction createScreenShares({ db, domain, hub, machineScreen, now = () => Date.now() }) {@@@ -106,19 +106,7 @@ function createScreenShares({ db, domain, hub, machineScreen, now = () => Date.nasync function input(t, value = {}) {const s = await must(t, { control: true });if (!hub) throw new BotError(503, 'bot.unavailable', 'machines are not connected here');-- const v = value && typeof value === 'object' ? value : {};-- if (!ACTIONS.has(v.action)) throw new BotError(422, 'bot.invalid_input', `action is one of ${[...ACTIONS].join(', ')}`);-- const out = { action: v.action };-- if (['click', 'double_click', 'right_click', 'move', 'scroll'].includes(v.action) && v.x != null) {-- for (const k of ['x', 'y', 'width', 'height']) {-- const n = Math.round(Number(v[k]));-- if (!Number.isFinite(n) || n < 0 || n > 10000) throw new BotError(422, 'bot.invalid_input', `${k} is a position in the picture`);-- out[k] = n;-- }-- }-- if (v.action === 'scroll') out.dy = Math.max(-50, Math.min(50, Math.round(Number(v.dy) || 0)));-- if (v.action === 'type') out.text = String(v.text || '').slice(0, 500);-- if (v.action === 'key') out.keys = String(v.keys || '').slice(0, 40);++ const out = inputValue(value);if (s.share.desk === 'private') out.desk = 'private'; // a desk link drives the desk, never the owner's screenconst recent = (rate.get(s.share.id) || []).filter((x) => now() - x < 1000);if (recent.length >= PER_SECOND) throw new BotError(429, 'bot.rate_limited', 'slow down a little');@@@ -822,16 +822,22 @@ ${control ? `<div class="share-controls">return page(`${name}: shared screen`, body, { scripts: ['/panel/panel.js', ...(control ? ['/panel/screen-share.js'] : [])], bodyClass: 'share-body', description: 'A shared computer screen on OpenVibe.Bot.' });}--function renderMachineScreenPage({ machine, until = null, error = null, idle = false, shares = [], madeLink = null, madeShare = null, hasDesk = false }) {++function renderMachineScreenPage({ machine, until = null, error = null, idle = false, shares = [], madeLink = null, madeShare = null, hasDesk = false, desk = 'main' }) {const name = machine.name || machine.id;const stateUrl = `/machines/${encodeURIComponent(machine.id)}/screen.json`;++ const base = `/machines/${encodeURIComponent(machine.id)}/screen`;++ // An agent task's own desk (desk-<name>, Desks plan D1) is what a task's Watch link opens.++ const agentDesk = /^desk-/.test(desk);++ const deskWords = desk === 'main' ? 'your own screen' : desk === 'private' ? 'the private desk' : 'your agent\'s desk';// Idle (nothing is shown, as from a link): a start button instead of waiting for a stream nobody started.if (idle && !error) {const body = `${topbar(`<a class="crumb" href="/machines">Machines</a><span class="crumb">${esc(name)}</span>`)}<main class="page"><h1>${esc(name)}: screen</h1>++${agentDesk ? `<p>Your agent works on a desk of its own on this computer. Watch it live, and take it over whenever you like: while you use the mouse and keyboard, the agent waits; when you stop, it goes on.</p>++<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><input type="hidden" name="desk" value="${esc(desk)}"><button type="submit" class="primary">Watch your agent's desk</button></form>` : ''}<p class="muted">Show this computer's screen here, live, to you only, for 10 minutes. Nothing is recorded, and the computer says on its desktop that its screen is live.</p>--<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><button type="submit" class="primary">Show my screen</button></form>++<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><button type="submit" class="${agentDesk ? 'button' : 'primary'}">Show my screen</button></form>${hasDesk ? `<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><input type="hidden" name="desk" value="private"><button type="submit" class="button">Show the private desk</button></form><p class="muted">The private desk is a screen of its own on this computer, where your agent works without touching yours.</p>` : ''}${sharesBlock(machine, shares, madeLink, madeShare, hasDesk)}@@@ -839,22 +845,30 @@ ${sharesBlock(machine, shares, madeLink, madeShare, hasDesk)}return page(`${name}: screen`, body, { description: 'Your machine\'s screen, live, for you only.', canonical: '/machines' });}const ends = until ? `<p class="muted">Shown until <time datetime="${esc(until)}" data-local-time>${esc(until)}</time>, to you only. Nothing is recorded.</p>` : '';++ // Take over (Desks plan D4): the share page's own controls (public/screen-share.js) in owner mode, on what is shown.++ const takeOver = error ? '' : `<div class="share-controls">++<button type="button" class="primary" data-share-start hidden>Start the screen</button>++<label class="check share-use"><input type="checkbox" data-share-use> ${desk === 'main' ? 'Use the mouse and keyboard' : 'Take over: use the mouse and keyboard'}</label>++<span class="muted small" data-share-note role="status"></span></div>++${desk !== 'main' ? '<p class="muted small">While you use it, the agent waits on this desk; uncheck to give it back, and it goes on a moment later.</p>' : ''}`;const body = `${topbar(`<a class="crumb" href="/machines">Machines</a><span class="crumb">${esc(name)}</span>`)}--<main class="page">++<main class="page" data-share="${esc(base)}" data-control="${error ? '0' : '1'}" data-owner="1">++<p class="muted small">Showing ${esc(deskWords)}${agentDesk ? ` (<code>${esc(desk)}</code>)` : ''}.</p><h1>${esc(name)}: screen</h1>${error ? `<p class="error" role="alert">${esc(error)}</p>` : ends}--<figure class="camera machine-screen" data-screen-state="${esc(stateUrl)}">++<figure class="camera machine-screen" data-screen-state="${esc(stateUrl)}" tabindex="0" aria-label="The screen: turn on the mouse and keyboard to click and type on it"><div class="camera-screen" data-camera-screen>${icon('plug', 'icon camera-icon')}<p class="camera-title" data-screen-status role="status">${error ? 'The screen is not shown.' : 'Waiting for the machine…'}</p><p class="camera-sub">The picture comes from the machine itself, a few frames a second.</p></div></figure>++${takeOver}<div class="machine-screen-actions">--<form method="post" action="/machines/${esc(machine.id)}/screen/stop"><button type="submit" class="primary">Stop showing my screen</button></form>--<form method="post" action="/machines/${esc(machine.id)}/screen"><button type="submit" class="button quiet">${error ? 'Try again' : 'Start again (10 more minutes)'}</button></form>++<form method="post" action="/machines/${esc(machine.id)}/screen/stop"><button type="submit" class="primary">Stop showing it</button></form>++<form method="post" action="/machines/${esc(machine.id)}/screen">${desk !== 'main' ? `<input type="hidden" name="desk" value="${esc(desk)}">` : ''}<button type="submit" class="button quiet">${error ? 'Try again' : 'Start again (10 more minutes)'}</button></form></div><p class="muted small">The machine says on its desktop that its screen is live. <code>openvibe-node stop</code> on the machine ends it at once, and it ends on its own after its time.</p>${sharesBlock(machine, shares, madeLink, madeShare)}</main>`;-- return page(`${name}: screen`, body, { scripts: ['/panel/panel.js'], description: 'Your machine\'s screen, live, for you only.', canonical: '/machines' });++ return page(`${name}: screen`, body, { scripts: ['/panel/panel.js', '/panel/screen-share.js'], description: 'Your machine\'s screen, live, for you only.', canonical: '/machines' });}/**@@@ -596,8 +596,10 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limiif (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site form posts are refused');const mine = await myMachine(req, me);try {-- const out = await machineScreen.start(mine, me.subject, asUser(me), { desk: req.body && req.body.desk === 'private' ? 'private' : 'main' });-- res.redirect(303, `/machines/${encodeURIComponent(mine.id)}/screen?until=${encodeURIComponent(out.until)}`);++ // Any desk: the person's own screen (main), the private desk, or an agent task's own desk (desk-<name>).++ const desk = req.body && machineScreen.DESK.test(String(req.body.desk || '')) ? String(req.body.desk) : 'main';++ const out = await machineScreen.start(mine, me.subject, asUser(me), { desk });++ res.redirect(303, `/machines/${encodeURIComponent(mine.id)}/screen?until=${encodeURIComponent(out.until)}${desk !== 'main' ? `&desk=${encodeURIComponent(desk)}` : ''}`);} catch (e) {if (!(e instanceof BotError)) throw e;res.setHeader('Content-Security-Policy', PANEL_CSP);@@@ -608,11 +610,35 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limiconst me = requireUser(req);const mine = await myMachine(req, me);const until = typeof req.query.until === 'string' && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d{1,3})?Z$/.test(req.query.until) ? req.query.until : null;++ // A link from an agent's task names its desk (Desks plan D4: Watch, Take over): offer that one first.++ const desk = typeof req.query.desk === 'string' && machineScreen.DESK.test(req.query.desk) ? req.query.desk : 'main';// Opened from a link with nothing shown: offer to start it rather than wait for a stream nobody started.-- const idle = !until && (await machineScreen.state(mine, me.subject)).state !== 'live';++ const st = await machineScreen.state(mine, me.subject);++ const idle = !until && st.state !== 'live';res.setHeader('Vary', 'Cookie');res.setHeader('Content-Security-Policy', PANEL_CSP);-- res.type('html').send(renderMachineScreenPage({ machine: mine, until, idle, shares: await shares.list(mine, me.subject).catch(() => []), hasDesk: hasDesk(mine) }));++ res.type('html').send(renderMachineScreenPage({ machine: mine, until, idle, shares: await shares.list(mine, me.subject).catch(() => []), hasDesk: hasDesk(mine), desk: idle ? desk : st.desk }));++ }));++ // The owner's page drives what it shows (public/screen-share.js, owner mode): its state, start, stop and the mouse++ // and keyboard ("Take over"), as JSON, same-site only.++ r.get('/machines/:id/screen/state.json', page(async (req, res) => {++ const me = requireUser(req);++ const mine = await myMachine(req, me);++ res.setHeader('Cache-Control', 'private, no-store');++ res.json(await machineScreen.state(mine, me.subject));++ }));++ r.post('/machines/:id/screen/start', ...manage, express.json({ limit: '1kb' }), page(async (req, res) => {++ const me = requireUser(req);++ if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site requests are refused');++ const mine = await myMachine(req, me);++ const desk = req.body && machineScreen.DESK.test(String(req.body.desk || '')) ? String(req.body.desk) : 'main';++ res.json(await machineScreen.start(mine, me.subject, asUser(me), { desk }));++ }));++ r.post('/machines/:id/screen/input', ...manage, express.json({ limit: '2kb' }), page(async (req, res) => {++ const me = requireUser(req);++ if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site requests are refused');++ const mine = await myMachine(req, me);++ res.json(await machineScreen.input(mine, me.subject, req.body || {}));}));r.get('/machines/:id/screen.json', page(async (req, res) => {const me = requireUser(req);@@@ -620,11 +646,12 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limires.setHeader('Cache-Control', 'private, no-store');res.json(await machineScreen.state(mine, me.subject));}));-- r.post('/machines/:id/screen/stop', ...manage, express.urlencoded({ extended: false, limit: '1kb' }), page(async (req, res) => {++ r.post('/machines/:id/screen/stop', ...manage, express.urlencoded({ extended: false, limit: '1kb' }), express.json({ limit: '1kb' }), page(async (req, res) => {const me = requireUser(req);if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site form posts are refused');const mine = await myMachine(req, me);-- await machineScreen.stop(mine, me.subject, asUser(me));++ const out = await machineScreen.stop(mine, me.subject, asUser(me));++ if (req.is('application/json')) return res.json(out); // the page's own Stop (owner mode)res.redirect(303, '/machines');}));// ── Screen share links (server/screen-shares.js): the owner shares the live screen with a group, to watch or use ──@@@ -33,6 +33,7 @@ const { boot, check, done } = require('./helpers/app');if (m.kind === 'screen_stream' && m.value.action === 'start') {c.send(frame('ack', { id: m.id, data: { streaming: true, sid: 'abc', seconds: m.value.seconds, fps: m.value.fps, until_ms: Date.now() + m.value.seconds * 1000 } }));} else if (m.kind === 'screen_stream') c.send(frame('ack', { id: m.id, data: { streaming: false, stopped: true } }));++ else if (m.kind === 'input') c.send(frame('ack', { id: m.id, data: { done: true, action: m.value.action } }));else c.send(frame('nack', { id: m.id, fault_code: 'unsupported' }));});// Heartbeats keep it online, as a real Node's do.@@@ -173,6 +174,38 @@ const { boot, check, done } = require('./helpers/app');single.c.close();});++ await check('watch and take over an agent\'s desk (Desks plan D4): a task\'s link opens its desk; the owner\'s mouse and keyboard go to the desk shown, as the owner; same site only', async () => {++ const desk = { level: 'trusted', window_manager: 'openbox', input: true, screen: true, named: true, max_desks: 4 };++ const m = await machine(alex, { ...SCREEN, input: { level: 'trusted', tool: 'xdotool' }, desk });++ const idle = await (await fetch(`${t.base}/machines/${m.deviceId}/screen?desk=desk-task-a`, { headers: cookie(alex) })).text();++ assert.ok(idle.includes('Watch your agent's desk') || idle.includes("Watch your agent's desk"), 'a task\'s link offers its desk first');++ assert.ok(idle.includes('name="desk" value="desk-task-a"'));++ const bad = await (await fetch(`${t.base}/machines/${m.deviceId}/screen?desk=desk-../x`, { headers: cookie(alex) })).text();++ assert.ok(!bad.includes('name="desk" value="desk-../x"'), 'a bad name is never offered');++ const started = await fetch(`${t.base}/machines/${m.deviceId}/screen`, { method: 'POST', redirect: 'manual', body: 'desk=desk-task-a', headers: { 'Content-Type': 'application/x-www-form-urlencoded', ...cookie(alex) } });++ assert.strictEqual(started.status, 303);++ assert.match(started.headers.get('location'), /&desk=desk-task-a$/);++ assert.strictEqual(seen.at(-1).value.desk, 'desk-task-a', 'the machine streams that desk');++ const html = await (await fetch(t.base + started.headers.get('location'), { headers: cookie(alex) })).text();++ assert.ok(html.includes('data-owner="1"') && html.includes('/panel/screen-share.js') && html.includes('Take over: use the mouse and keyboard'), 'the page offers Take over');++ const st = await (await fetch(`${t.base}/machines/${m.deviceId}/screen/state.json`, { headers: cookie(alex) })).json();++ assert.strictEqual(st.desk, 'desk-task-a');++ const json = (body, headers = {}) => ({ method: 'POST', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json', ...cookie(alex), ...headers } });++ const click = await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, json({ action: 'click', x: 10, y: 20, width: 100, height: 100 }));++ assert.strictEqual(click.status, 200, await click.clone().text());++ assert.deepStrictEqual(await click.json(), { done: true, desk: 'desk-task-a' });++ const sent = seen.at(-1);++ assert.deepStrictEqual([sent.kind, sent.value.desk, sent.value.action, sent.value.x], ['input', 'desk-task-a', 'click', 10], 'on the desk shown, never the owner\'s own screen');++ assert.strictEqual(sent.operator && sent.operator.subject, alex.subject, 'as the owner');++ assert.strictEqual((await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, json({ action: 'click', x: 1, y: 1, width: 2, height: 2 }, { Origin: 'https://evil.test' }))).status, 403, 'no cross-site input');++ assert.strictEqual((await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, json({ action: 'reboot' }))).status, 422);++ const bobInput = await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, { method: 'POST', body: '{"action":"click"}', headers: { 'Content-Type': 'application/json', ...cookie(bob) } });++ assert.strictEqual(bobInput.status, 404, 'only the owner');++ const stop = await fetch(`${t.base}/machines/${m.deviceId}/screen/stop`, json({}));++ assert.deepStrictEqual([stop.status, (await stop.json()).stopped !== undefined], [200, true], 'the page\'s own Stop answers JSON');++ m.c.close();++ });++await check('removing the machine archives its screen stream', async () => {const r = await form(`/machines/${mine.deviceId}/remove`, alex);assert.strictEqual(r.status, 303);
@@ -7,6 +7,9 @@
@@ -7,6 +7,9 @@
const main = document.querySelector('main[data-share]');
const main = document.querySelector('main[data-share]');
if (!main || main.dataset.control !== '1') return;
if (!main || main.dataset.control !== '1') return;
const base = main.dataset.share;
const base = main.dataset.share;
+ // The owner's own screen page (data-owner): signed in by cookie, same site; a share link: the token only.
+ const credentials = main.dataset.owner === '1' ? 'same-origin' : 'omit';
+ const desk = new URLSearchParams(location.search).get('desk');
const fig = main.querySelector('[data-screen-state]');
const fig = main.querySelector('[data-screen-state]');
const use = main.querySelector('[data-share-use]');
const use = main.querySelector('[data-share-use]');
const startBtn = main.querySelector('[data-share-start]');
const startBtn = main.querySelector('[data-share-start]');
@@ -15,7 +18,7 @@
@@ -15,7 +18,7 @@
const say = (t) => { if (note) note.textContent = t || ''; };
const say = (t) => { if (note) note.textContent = t || ''; };
async function post(path, body) {
async function post(path, body) {
- const r = await fetch(base + path, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify(body || {}), credentials: 'omit' });
+ const r = await fetch(base + path, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify(body || {}), credentials });
const j = await r.json().catch(() => ({}));
const j = await r.json().catch(() => ({}));
if (!r.ok) throw new Error(j.detail || ('the computer answered ' + r.status));
if (!r.ok) throw new Error(j.detail || ('the computer answered ' + r.status));
return j;
return j;
@@ -24,7 +27,7 @@
@@ -24,7 +27,7 @@
// Start when the screen is off; the state comes from the same JSON panel.js reads.
// Start when the screen is off; the state comes from the same JSON panel.js reads.
async function refresh() {
async function refresh() {
try {
try {
- const r = await fetch(base + '/state.json', { headers: { Accept: 'application/json' }, cache: 'no-store', credentials: 'omit' });
+ const r = await fetch(base + '/state.json', { headers: { Accept: 'application/json' }, cache: 'no-store', credentials });
const s = r.ok ? await r.json() : null;
const s = r.ok ? await r.json() : null;
if (startBtn) startBtn.hidden = !s || s.live || !s.online;
if (startBtn) startBtn.hidden = !s || s.live || !s.online;
if (s && !s.online) say('The computer is offline.');
if (s && !s.online) say('The computer is offline.');
@@ -34,7 +37,7 @@
@@ -34,7 +37,7 @@
setInterval(refresh, 5000);
setInterval(refresh, 5000);
if (startBtn) startBtn.addEventListener('click', async () => {
if (startBtn) startBtn.addEventListener('click', async () => {
startBtn.disabled = true; say('Starting the screen…');
startBtn.disabled = true; say('Starting the screen…');
- try { await post('/start'); say('Starting: the picture appears in a few seconds.'); } catch (e) { say(e.message); } finally { startBtn.disabled = false; refresh(); }
+ try { await post('/start', desk ? { desk } : {}); say('Starting: the picture appears in a few seconds.'); } catch (e) { say(e.message); } finally { startBtn.disabled = false; refresh(); }
});
});
if (stopBtn) stopBtn.addEventListener('click', async () => {
if (stopBtn) stopBtn.addEventListener('click', async () => {
try { await post('/stop'); say('The screen is stopped.'); } catch (e) { say(e.message); } finally { refresh(); }
try { await post('/stop'); say('The screen is stopped.'); } catch (e) { say(e.message); } finally { refresh(); }
@@ -20,6 +20,26 @@
@@ -20,6 +20,26 @@
const { BotError } = require('./util');
const { BotError } = require('./util');
const SECONDS = { min: 30, max: 1800, default: 600 };
const SECONDS = { min: 30, max: 1800, default: 600 };
+const DESK = /^(main|private|desk-[a-z0-9][a-z0-9-]{0,23})$/;
+const ACTIONS = new Set(['click', 'double_click', 'right_click', 'move', 'scroll', 'type', 'key']);
+
+/** A person's mouse or keyboard action on a shown screen, as bot.command@1 input takes it (share links and the owner). */
+function inputValue(value) {
+ const v = value && typeof value === 'object' ? value : {};
+ if (!ACTIONS.has(v.action)) throw new BotError(422, 'bot.invalid_input', `action is one of ${[...ACTIONS].join(', ')}`);
+ const out = { action: v.action };
+ if (['click', 'double_click', 'right_click', 'move', 'scroll'].includes(v.action) && v.x != null) {
+ for (const k of ['x', 'y', 'width', 'height']) {
+ const n = Math.round(Number(v[k]));
+ if (!Number.isFinite(n) || n < 0 || n > 10000) throw new BotError(422, 'bot.invalid_input', `${k} is a position in the picture`);
+ out[k] = n;
+ }
+ }
+ if (v.action === 'scroll') out.dy = Math.max(-50, Math.min(50, Math.round(Number(v.dy) || 0)));
+ if (v.action === 'type') out.text = String(v.text || '').slice(0, 500);
+ if (v.action === 'key') out.keys = String(v.keys || '').slice(0, 40);
+ return out;
+}
const FPS = { min: 1, max: 5, default: 2 };
const FPS = { min: 1, max: 5, default: 2 };
function whole(v, { min, max, default: dflt }, name) {
function whole(v, { min, max, default: dflt }, name) {
@@ -31,14 +51,19 @@ function whole(v, { min, max, default: dflt }, name) {
@@ -31,14 +51,19 @@ function whole(v, { min, max, default: dflt }, name) {
function createMachineScreen({ domain, hub, now = () => Date.now() }) {
function createMachineScreen({ domain, hub, now = () => Date.now() }) {
const unavailable = () => new BotError(503, 'bot.unavailable', 'machines are not connected here');
const unavailable = () => new BotError(503, 'bot.unavailable', 'machines are not connected here');
+ // The desk each machine's live screen shows (this process): the owner's input goes there, never to another screen.
+ const shown = new Map();
+ const owner = (subject) => ({ kind: 'user', sub: subject });
+ const rate = new Map();
return {
return {
+ DESK,
limits: { SECONDS, FPS },
limits: { SECONDS, FPS },
async start(device, subject, principal, opts = {}) {
async start(device, subject, principal, opts = {}) {
if (!hub) throw unavailable();
if (!hub) throw unavailable();
const seconds = whole(opts.seconds, SECONDS, 'seconds');
const seconds = whole(opts.seconds, SECONDS, 'seconds');
const fps = whole(opts.fps, FPS, 'fps');
const fps = whole(opts.fps, FPS, 'fps');
const desk = opts.desk == null || opts.desk === '' ? 'main' : String(opts.desk);
const desk = opts.desk == null || opts.desk === '' ? 'main' : String(opts.desk);
- if (desk !== 'main' && !/^(private|desk-[a-z0-9][a-z0-9-]{0,23})$/.test(desk)) throw new BotError(422, 'bot.invalid_input', 'desk is main, private or desk-<name>');
+ if (!DESK.test(desk)) throw new BotError(422, 'bot.invalid_input', 'desk is main, private or desk-<name>');
if (!hub.isOnline(device.id)) throw new BotError(409, 'bot.device_offline', 'the machine is offline');
if (!hub.isOnline(device.id)) throw new BotError(409, 'bot.device_offline', 'the machine is offline');
// Before any key is made: the machine says whether it can (Node v0.12.0's computer plugin, a capture tool).
// Before any key is made: the machine says whether it can (Node v0.12.0's computer plugin, a capture tool).
const offered = hub.computerOf(device.id);
const offered = hub.computerOf(device.id);
@@ -60,13 +85,32 @@ function createMachineScreen({ domain, hub, now = () => Date.now() }) {
@@ -60,13 +85,32 @@ function createMachineScreen({ domain, hub, now = () => Date.now() }) {
}
}
const data = r.data && typeof r.data === 'object' ? r.data : {};
const data = r.data && typeof r.data === 'object' ? r.data : {};
const until = Number.isFinite(data.until_ms) ? new Date(data.until_ms).toISOString() : new Date(now() + seconds * 1000).toISOString();
const until = Number.isFinite(data.until_ms) ? new Date(data.until_ms).toISOString() : new Date(now() + seconds * 1000).toISOString();
+ shown.set(device.id, desk);
return { streaming: true, seconds, fps, until, desk };
return { streaming: true, seconds, fps, until, desk };
},
},
async state(device, subject) {
async state(device, subject) {
const sensors = hub ? ((hub.deviceState(device.id).telemetry || {}).sensors || {}) : {};
const sensors = hub ? ((hub.deviceState(device.id).telemetry || {}).sensors || {}) : {};
const state = sensors.computer_screen_stream === 'live' ? 'live' : 'off';
const state = sensors.computer_screen_stream === 'live' ? 'live' : 'off';
const url = state === 'live' ? await domain.machines.screen.live(device, subject) : null;
const url = state === 'live' ? await domain.machines.screen.live(device, subject) : null;
- return { live: !!url, signaling_url: url, state, online: !!(hub && hub.isOnline(device.id)) };
+ return { live: !!url, signaling_url: url, state, online: !!(hub && hub.isOnline(device.id)), desk: shown.get(device.id) || 'main' };
+ },
+ /**
+ * The owner's own mouse or keyboard on the screen shown now (Desks plan D4, "Take over"): sent as the owner, so
+ * Bot's person-first rule holds an agent off that screen while they use it, and it goes on when they stop.
+ */
+ async input(device, subject, value) {
+ if (!hub) throw unavailable();
+ const out = inputValue(value);
+ const desk = shown.get(device.id) || 'main';
+ if (desk !== 'main') out.desk = desk;
+ const recent = (rate.get(device.id) || []).filter((x) => now() - x < 1000);
+ if (recent.length >= 8) throw new BotError(429, 'bot.rate_limited', 'slow down a little');
+ recent.push(now());
+ rate.set(device.id, recent);
+ const r = await hub.machineCommand(device.id, owner(subject), subject, { kind: 'input', value: out, ms: 15000 });
+ if (r && r.result === 'ack') return { done: true, desk };
+ const code = (r && r.code) || 'bot.device_refused';
+ throw new BotError(r && r.result === 'expired' ? 504 : 409, code.startsWith('bot.') ? code : 'bot.device_refused', `the computer did not do it: ${(r && (r.reason || r.result)) || 'no answer'}`.slice(0, 300));
},
},
async stop(device, subject, principal) {
async stop(device, subject, principal) {
let told = false;
let told = false;
@@ -75,9 +119,10 @@ function createMachineScreen({ domain, hub, now = () => Date.now() }) {
@@ -75,9 +119,10 @@ function createMachineScreen({ domain, hub, now = () => Date.now() }) {
told = r.result === 'ack';
told = r.result === 'ack';
}
}
const ended = await domain.machines.screen.end(device, subject);
const ended = await domain.machines.screen.end(device, subject);
+ shown.delete(device.id);
return { stopped: told || ended };
return { stopped: told || ended };
},
},
};
};
}
}
-module.exports = { createMachineScreen, SECONDS, FPS };
+module.exports = { createMachineScreen, inputValue, ACTIONS, SECONDS, FPS, DESK };
@@ -23,9 +23,9 @@
@@ -23,9 +23,9 @@
* move, scroll, type, key) → the machine's answer
* move, scroll, type, key) → the machine's answer
*/
*/
const { BotError, prefixedId, sha256, token: newToken } = require('./util');
const { BotError, prefixedId, sha256, token: newToken } = require('./util');
+const { inputValue, ACTIONS } = require('./machine-screen');
const DAYS = { min: 1, max: 365, default: 30 };
const DAYS = { min: 1, max: 365, default: 30 };
-const ACTIONS = new Set(['click', 'double_click', 'right_click', 'move', 'scroll', 'type', 'key']);
const PER_SECOND = 8; // a person clicks and types a few times a second, not more
const PER_SECOND = 8; // a person clicks and types a few times a second, not more
function createScreenShares({ db, domain, hub, machineScreen, now = () => Date.now() }) {
function createScreenShares({ db, domain, hub, machineScreen, now = () => Date.now() }) {
@@ -106,19 +106,7 @@ function createScreenShares({ db, domain, hub, machineScreen, now = () => Date.n
@@ -106,19 +106,7 @@ function createScreenShares({ db, domain, hub, machineScreen, now = () => Date.n
async function input(t, value = {}) {
async function input(t, value = {}) {
const s = await must(t, { control: true });
const s = await must(t, { control: true });
if (!hub) throw new BotError(503, 'bot.unavailable', 'machines are not connected here');
if (!hub) throw new BotError(503, 'bot.unavailable', 'machines are not connected here');
- const v = value && typeof value === 'object' ? value : {};
- if (!ACTIONS.has(v.action)) throw new BotError(422, 'bot.invalid_input', `action is one of ${[...ACTIONS].join(', ')}`);
- const out = { action: v.action };
- if (['click', 'double_click', 'right_click', 'move', 'scroll'].includes(v.action) && v.x != null) {
- for (const k of ['x', 'y', 'width', 'height']) {
- const n = Math.round(Number(v[k]));
- if (!Number.isFinite(n) || n < 0 || n > 10000) throw new BotError(422, 'bot.invalid_input', `${k} is a position in the picture`);
- out[k] = n;
- }
- }
- if (v.action === 'scroll') out.dy = Math.max(-50, Math.min(50, Math.round(Number(v.dy) || 0)));
- if (v.action === 'type') out.text = String(v.text || '').slice(0, 500);
- if (v.action === 'key') out.keys = String(v.keys || '').slice(0, 40);
+ const out = inputValue(value);
if (s.share.desk === 'private') out.desk = 'private'; // a desk link drives the desk, never the owner's screen
if (s.share.desk === 'private') out.desk = 'private'; // a desk link drives the desk, never the owner's screen
const recent = (rate.get(s.share.id) || []).filter((x) => now() - x < 1000);
const recent = (rate.get(s.share.id) || []).filter((x) => now() - x < 1000);
if (recent.length >= PER_SECOND) throw new BotError(429, 'bot.rate_limited', 'slow down a little');
if (recent.length >= PER_SECOND) throw new BotError(429, 'bot.rate_limited', 'slow down a little');
@@ -822,16 +822,22 @@ ${control ? `<div class="share-controls">
@@ -822,16 +822,22 @@ ${control ? `<div class="share-controls">
return page(`${name}: shared screen`, body, { scripts: ['/panel/panel.js', ...(control ? ['/panel/screen-share.js'] : [])], bodyClass: 'share-body', description: 'A shared computer screen on OpenVibe.Bot.' });
return page(`${name}: shared screen`, body, { scripts: ['/panel/panel.js', ...(control ? ['/panel/screen-share.js'] : [])], bodyClass: 'share-body', description: 'A shared computer screen on OpenVibe.Bot.' });
}
}
-function renderMachineScreenPage({ machine, until = null, error = null, idle = false, shares = [], madeLink = null, madeShare = null, hasDesk = false }) {
+function renderMachineScreenPage({ machine, until = null, error = null, idle = false, shares = [], madeLink = null, madeShare = null, hasDesk = false, desk = 'main' }) {
const name = machine.name || machine.id;
const name = machine.name || machine.id;
const stateUrl = `/machines/${encodeURIComponent(machine.id)}/screen.json`;
const stateUrl = `/machines/${encodeURIComponent(machine.id)}/screen.json`;
+ const base = `/machines/${encodeURIComponent(machine.id)}/screen`;
+ // An agent task's own desk (desk-<name>, Desks plan D1) is what a task's Watch link opens.
+ const agentDesk = /^desk-/.test(desk);
+ const deskWords = desk === 'main' ? 'your own screen' : desk === 'private' ? 'the private desk' : 'your agent\'s desk';
// Idle (nothing is shown, as from a link): a start button instead of waiting for a stream nobody started.
// Idle (nothing is shown, as from a link): a start button instead of waiting for a stream nobody started.
if (idle && !error) {
if (idle && !error) {
const body = `${topbar(`<a class="crumb" href="/machines">Machines</a><span class="crumb">${esc(name)}</span>`)}
const body = `${topbar(`<a class="crumb" href="/machines">Machines</a><span class="crumb">${esc(name)}</span>`)}
<main class="page">
<main class="page">
<h1>${esc(name)}: screen</h1>
<h1>${esc(name)}: screen</h1>
+${agentDesk ? `<p>Your agent works on a desk of its own on this computer. Watch it live, and take it over whenever you like: while you use the mouse and keyboard, the agent waits; when you stop, it goes on.</p>
+<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><input type="hidden" name="desk" value="${esc(desk)}"><button type="submit" class="primary">Watch your agent's desk</button></form>` : ''}
<p class="muted">Show this computer's screen here, live, to you only, for 10 minutes. Nothing is recorded, and the computer says on its desktop that its screen is live.</p>
<p class="muted">Show this computer's screen here, live, to you only, for 10 minutes. Nothing is recorded, and the computer says on its desktop that its screen is live.</p>
-<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><button type="submit" class="primary">Show my screen</button></form>
+<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><button type="submit" class="${agentDesk ? 'button' : 'primary'}">Show my screen</button></form>
${hasDesk ? `<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><input type="hidden" name="desk" value="private"><button type="submit" class="button">Show the private desk</button></form>
${hasDesk ? `<form method="post" action="/machines/${esc(machine.id)}/screen" class="inline-form"><input type="hidden" name="desk" value="private"><button type="submit" class="button">Show the private desk</button></form>
<p class="muted">The private desk is a screen of its own on this computer, where your agent works without touching yours.</p>` : ''}
<p class="muted">The private desk is a screen of its own on this computer, where your agent works without touching yours.</p>` : ''}
${sharesBlock(machine, shares, madeLink, madeShare, hasDesk)}
${sharesBlock(machine, shares, madeLink, madeShare, hasDesk)}
@@ -839,22 +845,30 @@ ${sharesBlock(machine, shares, madeLink, madeShare, hasDesk)}
@@ -839,22 +845,30 @@ ${sharesBlock(machine, shares, madeLink, madeShare, hasDesk)}
return page(`${name}: screen`, body, { description: 'Your machine\'s screen, live, for you only.', canonical: '/machines' });
return page(`${name}: screen`, body, { description: 'Your machine\'s screen, live, for you only.', canonical: '/machines' });
}
}
const ends = until ? `<p class="muted">Shown until <time datetime="${esc(until)}" data-local-time>${esc(until)}</time>, to you only. Nothing is recorded.</p>` : '';
const ends = until ? `<p class="muted">Shown until <time datetime="${esc(until)}" data-local-time>${esc(until)}</time>, to you only. Nothing is recorded.</p>` : '';
+ // Take over (Desks plan D4): the share page's own controls (public/screen-share.js) in owner mode, on what is shown.
+ const takeOver = error ? '' : `<div class="share-controls">
+<button type="button" class="primary" data-share-start hidden>Start the screen</button>
+<label class="check share-use"><input type="checkbox" data-share-use> ${desk === 'main' ? 'Use the mouse and keyboard' : 'Take over: use the mouse and keyboard'}</label>
+<span class="muted small" data-share-note role="status"></span></div>
+${desk !== 'main' ? '<p class="muted small">While you use it, the agent waits on this desk; uncheck to give it back, and it goes on a moment later.</p>' : ''}`;
const body = `${topbar(`<a class="crumb" href="/machines">Machines</a><span class="crumb">${esc(name)}</span>`)}
const body = `${topbar(`<a class="crumb" href="/machines">Machines</a><span class="crumb">${esc(name)}</span>`)}
-<main class="page">
+<main class="page" data-share="${esc(base)}" data-control="${error ? '0' : '1'}" data-owner="1">
+<p class="muted small">Showing ${esc(deskWords)}${agentDesk ? ` (<code>${esc(desk)}</code>)` : ''}.</p>
<h1>${esc(name)}: screen</h1>
<h1>${esc(name)}: screen</h1>
${error ? `<p class="error" role="alert">${esc(error)}</p>` : ends}
${error ? `<p class="error" role="alert">${esc(error)}</p>` : ends}
-<figure class="camera machine-screen" data-screen-state="${esc(stateUrl)}">
+<figure class="camera machine-screen" data-screen-state="${esc(stateUrl)}" tabindex="0" aria-label="The screen: turn on the mouse and keyboard to click and type on it">
<div class="camera-screen" data-camera-screen>${icon('plug', 'icon camera-icon')}<p class="camera-title" data-screen-status role="status">${error ? 'The screen is not shown.' : 'Waiting for the machine…'}</p>
<div class="camera-screen" data-camera-screen>${icon('plug', 'icon camera-icon')}<p class="camera-title" data-screen-status role="status">${error ? 'The screen is not shown.' : 'Waiting for the machine…'}</p>
<p class="camera-sub">The picture comes from the machine itself, a few frames a second.</p></div>
<p class="camera-sub">The picture comes from the machine itself, a few frames a second.</p></div>
</figure>
</figure>
+${takeOver}
<div class="machine-screen-actions">
<div class="machine-screen-actions">
-<form method="post" action="/machines/${esc(machine.id)}/screen/stop"><button type="submit" class="primary">Stop showing my screen</button></form>
-<form method="post" action="/machines/${esc(machine.id)}/screen"><button type="submit" class="button quiet">${error ? 'Try again' : 'Start again (10 more minutes)'}</button></form>
+<form method="post" action="/machines/${esc(machine.id)}/screen/stop"><button type="submit" class="primary">Stop showing it</button></form>
+<form method="post" action="/machines/${esc(machine.id)}/screen">${desk !== 'main' ? `<input type="hidden" name="desk" value="${esc(desk)}">` : ''}<button type="submit" class="button quiet">${error ? 'Try again' : 'Start again (10 more minutes)'}</button></form>
</div>
</div>
<p class="muted small">The machine says on its desktop that its screen is live. <code>openvibe-node stop</code> on the machine ends it at once, and it ends on its own after its time.</p>
<p class="muted small">The machine says on its desktop that its screen is live. <code>openvibe-node stop</code> on the machine ends it at once, and it ends on its own after its time.</p>
${sharesBlock(machine, shares, madeLink, madeShare)}
${sharesBlock(machine, shares, madeLink, madeShare)}
</main>`;
</main>`;
- return page(`${name}: screen`, body, { scripts: ['/panel/panel.js'], description: 'Your machine\'s screen, live, for you only.', canonical: '/machines' });
+ return page(`${name}: screen`, body, { scripts: ['/panel/panel.js', '/panel/screen-share.js'], description: 'Your machine\'s screen, live, for you only.', canonical: '/machines' });
}
}
/**
/**
@@ -596,8 +596,10 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limi
@@ -596,8 +596,10 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limi
if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site form posts are refused');
if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site form posts are refused');
const mine = await myMachine(req, me);
const mine = await myMachine(req, me);
try {
try {
- const out = await machineScreen.start(mine, me.subject, asUser(me), { desk: req.body && req.body.desk === 'private' ? 'private' : 'main' });
- res.redirect(303, `/machines/${encodeURIComponent(mine.id)}/screen?until=${encodeURIComponent(out.until)}`);
+ // Any desk: the person's own screen (main), the private desk, or an agent task's own desk (desk-<name>).
+ const desk = req.body && machineScreen.DESK.test(String(req.body.desk || '')) ? String(req.body.desk) : 'main';
+ const out = await machineScreen.start(mine, me.subject, asUser(me), { desk });
+ res.redirect(303, `/machines/${encodeURIComponent(mine.id)}/screen?until=${encodeURIComponent(out.until)}${desk !== 'main' ? `&desk=${encodeURIComponent(desk)}` : ''}`);
} catch (e) {
} catch (e) {
if (!(e instanceof BotError)) throw e;
if (!(e instanceof BotError)) throw e;
res.setHeader('Content-Security-Policy', PANEL_CSP);
res.setHeader('Content-Security-Policy', PANEL_CSP);
@@ -608,11 +610,35 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limi
@@ -608,11 +610,35 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limi
const me = requireUser(req);
const me = requireUser(req);
const mine = await myMachine(req, me);
const mine = await myMachine(req, me);
const until = typeof req.query.until === 'string' && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d{1,3})?Z$/.test(req.query.until) ? req.query.until : null;
const until = typeof req.query.until === 'string' && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d{1,3})?Z$/.test(req.query.until) ? req.query.until : null;
+ // A link from an agent's task names its desk (Desks plan D4: Watch, Take over): offer that one first.
+ const desk = typeof req.query.desk === 'string' && machineScreen.DESK.test(req.query.desk) ? req.query.desk : 'main';
// Opened from a link with nothing shown: offer to start it rather than wait for a stream nobody started.
// Opened from a link with nothing shown: offer to start it rather than wait for a stream nobody started.
- const idle = !until && (await machineScreen.state(mine, me.subject)).state !== 'live';
+ const st = await machineScreen.state(mine, me.subject);
+ const idle = !until && st.state !== 'live';
res.setHeader('Vary', 'Cookie');
res.setHeader('Vary', 'Cookie');
res.setHeader('Content-Security-Policy', PANEL_CSP);
res.setHeader('Content-Security-Policy', PANEL_CSP);
- res.type('html').send(renderMachineScreenPage({ machine: mine, until, idle, shares: await shares.list(mine, me.subject).catch(() => []), hasDesk: hasDesk(mine) }));
+ res.type('html').send(renderMachineScreenPage({ machine: mine, until, idle, shares: await shares.list(mine, me.subject).catch(() => []), hasDesk: hasDesk(mine), desk: idle ? desk : st.desk }));
+ }));
+ // The owner's page drives what it shows (public/screen-share.js, owner mode): its state, start, stop and the mouse
+ // and keyboard ("Take over"), as JSON, same-site only.
+ r.get('/machines/:id/screen/state.json', page(async (req, res) => {
+ const me = requireUser(req);
+ const mine = await myMachine(req, me);
+ res.setHeader('Cache-Control', 'private, no-store');
+ res.json(await machineScreen.state(mine, me.subject));
+ }));
+ r.post('/machines/:id/screen/start', ...manage, express.json({ limit: '1kb' }), page(async (req, res) => {
+ const me = requireUser(req);
+ if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site requests are refused');
+ const mine = await myMachine(req, me);
+ const desk = req.body && machineScreen.DESK.test(String(req.body.desk || '')) ? String(req.body.desk) : 'main';
+ res.json(await machineScreen.start(mine, me.subject, asUser(me), { desk }));
+ }));
+ r.post('/machines/:id/screen/input', ...manage, express.json({ limit: '2kb' }), page(async (req, res) => {
+ const me = requireUser(req);
+ if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site requests are refused');
+ const mine = await myMachine(req, me);
+ res.json(await machineScreen.input(mine, me.subject, req.body || {}));
}));
}));
r.get('/machines/:id/screen.json', page(async (req, res) => {
r.get('/machines/:id/screen.json', page(async (req, res) => {
const me = requireUser(req);
const me = requireUser(req);
@@ -620,11 +646,12 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limi
@@ -620,11 +646,12 @@ function createWebRoutes(config, { domain = null, sim = null, onvif = null, limi
res.setHeader('Cache-Control', 'private, no-store');
res.setHeader('Cache-Control', 'private, no-store');
res.json(await machineScreen.state(mine, me.subject));
res.json(await machineScreen.state(mine, me.subject));
}));
}));
- r.post('/machines/:id/screen/stop', ...manage, express.urlencoded({ extended: false, limit: '1kb' }), page(async (req, res) => {
+ r.post('/machines/:id/screen/stop', ...manage, express.urlencoded({ extended: false, limit: '1kb' }), express.json({ limit: '1kb' }), page(async (req, res) => {
const me = requireUser(req);
const me = requireUser(req);
if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site form posts are refused');
if (!sameOrigin(req)) throw new BotError(403, 'bot.forbidden', 'cross-site form posts are refused');
const mine = await myMachine(req, me);
const mine = await myMachine(req, me);
- await machineScreen.stop(mine, me.subject, asUser(me));
+ const out = await machineScreen.stop(mine, me.subject, asUser(me));
+ if (req.is('application/json')) return res.json(out); // the page's own Stop (owner mode)
res.redirect(303, '/machines');
res.redirect(303, '/machines');
}));
}));
// ── Screen share links (server/screen-shares.js): the owner shares the live screen with a group, to watch or use ──
// ── Screen share links (server/screen-shares.js): the owner shares the live screen with a group, to watch or use ──
@@ -33,6 +33,7 @@ const { boot, check, done } = require('./helpers/app');
@@ -33,6 +33,7 @@ const { boot, check, done } = require('./helpers/app');
if (m.kind === 'screen_stream' && m.value.action === 'start') {
if (m.kind === 'screen_stream' && m.value.action === 'start') {
c.send(frame('ack', { id: m.id, data: { streaming: true, sid: 'abc', seconds: m.value.seconds, fps: m.value.fps, until_ms: Date.now() + m.value.seconds * 1000 } }));
c.send(frame('ack', { id: m.id, data: { streaming: true, sid: 'abc', seconds: m.value.seconds, fps: m.value.fps, until_ms: Date.now() + m.value.seconds * 1000 } }));
} else if (m.kind === 'screen_stream') c.send(frame('ack', { id: m.id, data: { streaming: false, stopped: true } }));
} else if (m.kind === 'screen_stream') c.send(frame('ack', { id: m.id, data: { streaming: false, stopped: true } }));
+ else if (m.kind === 'input') c.send(frame('ack', { id: m.id, data: { done: true, action: m.value.action } }));
else c.send(frame('nack', { id: m.id, fault_code: 'unsupported' }));
else c.send(frame('nack', { id: m.id, fault_code: 'unsupported' }));
});
});
// Heartbeats keep it online, as a real Node's do.
// Heartbeats keep it online, as a real Node's do.
@@ -173,6 +174,38 @@ const { boot, check, done } = require('./helpers/app');
@@ -173,6 +174,38 @@ const { boot, check, done } = require('./helpers/app');
single.c.close();
single.c.close();
});
});
+ await check('watch and take over an agent\'s desk (Desks plan D4): a task\'s link opens its desk; the owner\'s mouse and keyboard go to the desk shown, as the owner; same site only', async () => {
+ const desk = { level: 'trusted', window_manager: 'openbox', input: true, screen: true, named: true, max_desks: 4 };
+ const m = await machine(alex, { ...SCREEN, input: { level: 'trusted', tool: 'xdotool' }, desk });
+ const idle = await (await fetch(`${t.base}/machines/${m.deviceId}/screen?desk=desk-task-a`, { headers: cookie(alex) })).text();
+ assert.ok(idle.includes('Watch your agent's desk') || idle.includes("Watch your agent's desk"), 'a task\'s link offers its desk first');
+ assert.ok(idle.includes('name="desk" value="desk-task-a"'));
+ const bad = await (await fetch(`${t.base}/machines/${m.deviceId}/screen?desk=desk-../x`, { headers: cookie(alex) })).text();
+ assert.ok(!bad.includes('name="desk" value="desk-../x"'), 'a bad name is never offered');
+ const started = await fetch(`${t.base}/machines/${m.deviceId}/screen`, { method: 'POST', redirect: 'manual', body: 'desk=desk-task-a', headers: { 'Content-Type': 'application/x-www-form-urlencoded', ...cookie(alex) } });
+ assert.strictEqual(started.status, 303);
+ assert.match(started.headers.get('location'), /&desk=desk-task-a$/);
+ assert.strictEqual(seen.at(-1).value.desk, 'desk-task-a', 'the machine streams that desk');
+ const html = await (await fetch(t.base + started.headers.get('location'), { headers: cookie(alex) })).text();
+ assert.ok(html.includes('data-owner="1"') && html.includes('/panel/screen-share.js') && html.includes('Take over: use the mouse and keyboard'), 'the page offers Take over');
+ const st = await (await fetch(`${t.base}/machines/${m.deviceId}/screen/state.json`, { headers: cookie(alex) })).json();
+ assert.strictEqual(st.desk, 'desk-task-a');
+ const json = (body, headers = {}) => ({ method: 'POST', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json', ...cookie(alex), ...headers } });
+ const click = await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, json({ action: 'click', x: 10, y: 20, width: 100, height: 100 }));
+ assert.strictEqual(click.status, 200, await click.clone().text());
+ assert.deepStrictEqual(await click.json(), { done: true, desk: 'desk-task-a' });
+ const sent = seen.at(-1);
+ assert.deepStrictEqual([sent.kind, sent.value.desk, sent.value.action, sent.value.x], ['input', 'desk-task-a', 'click', 10], 'on the desk shown, never the owner\'s own screen');
+ assert.strictEqual(sent.operator && sent.operator.subject, alex.subject, 'as the owner');
+ assert.strictEqual((await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, json({ action: 'click', x: 1, y: 1, width: 2, height: 2 }, { Origin: 'https://evil.test' }))).status, 403, 'no cross-site input');
+ assert.strictEqual((await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, json({ action: 'reboot' }))).status, 422);
+ const bobInput = await fetch(`${t.base}/machines/${m.deviceId}/screen/input`, { method: 'POST', body: '{"action":"click"}', headers: { 'Content-Type': 'application/json', ...cookie(bob) } });
+ assert.strictEqual(bobInput.status, 404, 'only the owner');
+ const stop = await fetch(`${t.base}/machines/${m.deviceId}/screen/stop`, json({}));
+ assert.deepStrictEqual([stop.status, (await stop.json()).stopped !== undefined], [200, true], 'the page\'s own Stop answers JSON');
+ m.c.close();
+ });
+
await check('removing the machine archives its screen stream', async () => {
await check('removing the machine archives its screen stream', async () => {
const r = await form(`/machines/${mine.deviceId}/remove`, alex);
const r = await form(`/machines/${mine.deviceId}/remove`, alex);
assert.strictEqual(r.status, 303);
assert.strictEqual(r.status, 303);