openvibers/openvibe.actor Commits
Chat: the agent's desk, live, inside the turn; click into it to take over (#233)
Changes
@@@ -202,6 +202,10 @@ html[data-ov-motion=reduced] .ct-answer.is-streaming .sw { opacity: 1 !important.ct-screen[hidden] { display: none; }.ct-screen-frame { position: relative; overflow: hidden; border-radius: 14px; border: 1px solid var(--line); background: #05070d; aspect-ratio: 16 / 10; display: grid; place-items: center; }.ct-screen-frame img { display: block; width: 100%; height: 100%; object-fit: contain; }++/* The live desk (plan L55 C1b): Bot's page in place of the picture, the same size, so nothing jumps. */++.ct-screen-frame .ct-desk { position: absolute; inset: 0; width: 100%; height: 100%; border: 0; background: #05070d; }++.ct-screen.is-framed .ct-screen-frame img { visibility: hidden; }++.ct-screen.is-framed .ct-screen-frame { aspect-ratio: 16 / 10; }.ct-screen-wait { padding: 16px; font-size: 14px; text-align: center; color: var(--text-muted, #7386a3); }.ct-live { position: absolute; top: 8px; left: 8px; display: inline-flex; align-items: center; gap: 5px; padding: 2px 8px; border-radius: 999px; background: rgba(0, 0, 0, .6); color: #fff; font-size: 11.5px; font-weight: 700; letter-spacing: .04em; text-transform: uppercase; }.ct-live::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: #ef4444; animation: ct-breathe 1.4s ease-in-out infinite; }@@@ -286,9 +286,44 @@}// What Actor sees: the last picture, fetched every two seconds while the turn works (only when it changed).var screens = {};++ // The live desk (plan L55 C1b): when the turn works on one of the person's own desks, the server asks Bot for a++ // short-lived page that shows it moving and takes the mouse, and the turn frames it in place of the picture.++ // If Bot cannot, the picture keeps refreshing as before.++ function live(turn) {++ var fig = turn.querySelector('[data-screen][data-watch]');++ if (!fig || fig.hasAttribute('data-framed') || !turn.hasAttribute('data-open')) return;++ fig.setAttribute('data-framed', 'asking');++ fetch(fig.getAttribute('data-watch'), { method: 'POST', credentials: 'same-origin', headers: { Accept: 'application/json' } })++ .then(function (r) { return r.ok ? r.json() : null; })++ .catch(function () { return null; })++ .then(function (j) {++ if (!j || !/^https:\/\//.test(j.url || '') || !fig.isConnected) { fig.setAttribute('data-framed', 'no'); return; }++ var frame = fig.querySelector('.ct-screen-frame');++ if (!frame) { frame = document.createElement('div'); frame.className = 'ct-screen-frame'; fig.insertBefore(frame, fig.firstChild); }++ var f = document.createElement('iframe');++ f.src = j.url;++ f.title = 'Actor\'s desk, live. Click into it to take over.';++ f.setAttribute('allow', 'fullscreen; clipboard-write');++ f.setAttribute('referrerpolicy', 'no-referrer');++ f.setAttribute('sandbox', 'allow-scripts allow-same-origin allow-forms allow-pointer-lock');++ f.className = 'ct-desk';++ frame.appendChild(f);++ fig.classList.add('is-framed');++ fig.hidden = false;++ fig.setAttribute('data-framed', 'yes');++ stopScreen(turn); // the frame moves by itself: no more pictures to fetch++ var a = fig.querySelector('.ct-watch');++ if (a) { a.href = j.url; a.lastChild.textContent = 'Open it big'; }++ var cap = fig.querySelector('figcaption > span');++ if (cap) cap.textContent = j.control ? 'Actor\'s desk, live: click into it to take over' : 'Actor\'s desk, live';++ });++ }function screen(turn) {var id = turn.getAttribute('data-task');++ live(turn);if (screens[id] || !turn.hasAttribute('data-open')) return;++ var framed = turn.querySelector('[data-screen][data-framed="yes"]');++ if (framed) return;var fig = turn.querySelector('[data-screen]');if (!fig) return;var busyLoad = false;@@@ -372,7 +372,39 @@ function createComputers({ config, fetchImpl = globalThis.fetch, tokens: tokensO/** The person's page to watch a desk live and take it over (Bot's own screen page). */const watchUrl = (machine, desk) => `${pages}/machines/${machine}/screen${desk && desk !== 'main' ? `?desk=${encodeURIComponent(desk)}` : ''}`;-- return { enabled, mine, list, read, run, write, screen, input, browser, desk, deskTurn, watchUrl, openSession, endSession, sessionsOn, isSession: (m) => SESSION_MACHINE.test(String(m || '')) };++ // A watch link (plan L55 C1b, contracts 0.268.0 bot.machine.watch_link): a short-lived page that shows the desk the++ // agent works on, live, and lets the person take it over, framable in the chat (no sign-in: the token is the++ // credential). Its own token client: a Network that has not granted the capability yet refuses only this, never++ // the computer control above.++ const linkTokens = !enabled ? null : tokensOverride || serviceAuth.createTokenClient({++ tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,++ audience: 'openvibe.bot', scope: 'bot.machine.watch_link', fetchImpl,++ });++ async function watchLink(subject, machine, desk, { control = true, minutes = 60, signal } = {}) {++ if (!enabled) return { error: 'computers are off here' };++ const bad = guard(subject, machine);++ if (bad) return bad;++ if (SESSION_MACHINE.test(String(machine))) return { error: 'a desk host session has no watch link' };++ // The agent's own desks only: Bot refuses the owner's screen (main) to an agent anyway.++ if (desk !== 'private' && !DESK_NAME.test(String(desk || ''))) return { error: 'only an agent desk (private or desk-<name>) has a watch link' };++ const sig = signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000);++ let res;++ try {++ res = await fetchImpl(`${base}/api/v1/machines/${encodeURIComponent(machine)}/watch-links`, {++ method: 'POST',++ headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'X-OV-Subject': subject, ...(await linkTokens.authHeaders()) },++ body: JSON.stringify({ desk, control: !!control, minutes, label: 'the chat on openvibe.actor' }), signal: sig,++ });++ } catch {++ return { error: 'Bot did not answer for the watch link' };++ }++ const data = await res.json().catch(() => null);++ if (!res.ok) return problem({ status: res.status, data }, 'the watch link');++ if (!data || !/^https:\/\/[^\s"<>]{1,300}$/.test(String(data.url || ''))) return { error: 'Bot answered without a link' };++ return { url: data.url, expires_at: data.expires_at || null, control: !!data.control, desk: data.desk || desk };++ }++++ return { enabled, mine, list, read, run, write, screen, input, browser, desk, deskTurn, watchUrl, watchLink, openSession, endSession, sessionsOn, isSession: (m) => SESSION_MACHINE.test(String(m || '')) };}module.exports = { createComputers };@@@ -177,7 +177,8 @@ async function createApp(opts = {}) {fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com', 'https://cdnjs.cloudflare.com'],imgSrc: ["'self'", 'data:', 'https://openvibe.network', 'https://openvibe.media'],connectSrc: ["'self'", 'https://openvibe.network', 'https://cloudflareinsights.com', 'https://openvibe.events'],-- frameSrc: ["'self'", 'https://openvibe.network'],++ // Bot's watch links: a chat turn frames the desk the agent works on, live (plan L55 C1b).++ frameSrc: ["'self'", 'https://openvibe.network', ...(() => { try { return [new URL(config.services.botPublicUrl).origin]; } catch { return []; } })()],frameAncestors: ["'none'"],objectSrc: ["'none'"],baseUri: ["'self'"],@@@ -242,7 +243,7 @@ async function createApp(opts = {}) {app.use(createMemoryPages(ctx));app.use(createConnectionsPages(ctx));app.use(createConsolePages(ctx, opts.bot || {}));-- app.use(createChatPages(ctx));++ app.use(createChatPages(ctx, opts.bot || {}));const pageRoutes = createPageRoutes(ctx);// The agent layer (openvibe-shared/agent-layer): /.well-known/agent.json and the Markdown twins of the public// pages. The pages themselves keep their own routes — a signed-in visitor's account bar and task box are@@@ -79,9 +79,15 @@ function tookOf(row) {return s < 60 ? `${s} s` : `${Math.floor(s / 60)} min${s % 60 ? ` ${s % 60} s` : ''}`;}--function createChatPages(ctx) {++function createChatPages(ctx, { computers = null } = {}) {const { config, s, engine } = ctx;const r = asyncRouter();++ // Bot's computers, for the live desk in a turn (plan L55 C1b): made when first asked for.++ let pcsMade = computers;++ const pcs = () => (pcsMade = pcsMade || require('../agents/computers').createComputers({ config }));++ // Watch links are credentials (whoever holds one watches, and may take over): kept in memory for the task's own++ // requester, never written to the task's events or the database, and reused until a few minutes before they end.++ const links = new Map();// Helpers from the task page, read when first used (pages.js is the bigger module; no load-order coupling).const pageKit = () => require('./pages');const signedIn = (req) => req.viewer && req.viewer.kind === 'user' && req.viewer.subject;@@@ -145,7 +151,9 @@ function createChatPages(ctx) {const desk = open ? store.deskFrom(events) : null;const look = looks.get(row.id);if (!desk && !look) return open ? html`<figure class="ct-screen" data-screen hidden></figure>` : '';-- return html`<figure class="ct-screen${open ? ' is-live' : ''}" data-screen${desk ? html` data-desk="${desk.url}"` : ''}>++ const place = desk && store.deskPlace(desk.url);++ const framable = !!(place && place.desk !== 'main');++ return html`<figure class="ct-screen${open ? ' is-live' : ''}" data-screen${desk ? html` data-desk="${desk.url}"` : ''}${framable ? html` data-watch="/chat/${row.id}/watch"` : ''}><div class="ct-screen-frame">${look ? html`<img data-look src="/tasks/${row.id}/look?at=${look.at}" alt="What Actor ${open ? 'sees now' : 'last saw'}"${look.width && look.height ? html` width="${look.width}" height="${look.height}"` : ''}>` : html`<span class="ct-screen-wait">The screen shows here as soon as Actor looks at it.</span>`}${open ? html`<span class="ct-live">Live</span>` : ''}</div><figcaption><span>${open ? `What Actor sees${desk && desk.kind === 'browser' ? ' in its browser' : ''}` : 'What Actor last saw'}</span>${desk ? html`<a class="ct-watch" href="${desk.url}" target="_blank" rel="noopener">${raw(ICON.open)}Watch live and take over</a>` : ''}</figcaption></figure>`;@@@ -391,6 +399,30 @@ ${composer({ parent: lastTurn ? lastTurn.id : null, mode: lastTurn ? lastTurn.moreturn res.redirect(303, `/chat/${row.id}#t-${row.id}`);});++ // The live desk in a working turn (plan L55 C1b; the owner, 11-01: "show you the display… watch it… and steer it").++ // The page asks once the turn is on a desk; the answer is a short-lived Bot page it frames, where the person can++ // take the mouse. Only the task's requester, only same-origin, only while the turn works.++ r.post('/chat/:id/watch', async (req, res) => {++ res.set('Cache-Control', 'private, no-store');++ if (!signedIn(req) || !TASK_ID.test(req.params.id)) return res.status(401).json({ error: 'Sign in first.' });++ if (!sameOrigin(req, config.baseUrl)) return res.status(403).json({ error: 'Ask from openvibe.actor itself.' });++ const row = await store.getTask(s, req.params.id);++ if (!row || row.requester !== requesterOf(req)) return res.status(404).json({ error: 'That chat is not here.' });++ if (!OPEN.includes(row.state)) return res.status(409).json({ error: 'Actor has finished that one.' });++ const on = await store.taskDesk(s, row);++ const desk = on && store.deskPlace(on.url);++ if (!desk) return res.status(409).json({ error: 'Actor is not on a desk just now.' });++ const key = `${row.id}|${desk.machine}|${desk.desk}`;++ const kept = links.get(key);++ if (kept && kept.until > now()) return res.json({ url: kept.url, expires_at: kept.expires_at, control: kept.control });++ const out = await pcs().watchLink(req.viewer.subject, desk.machine, desk.desk, { control: true, minutes: 60 });++ if (out.error) return res.status(502).json({ error: 'The live desk is not available here; the picture still updates.', reason: out.code || null });++ const ends = Date.parse(out.expires_at || '') || now() + 60 * 60000;++ links.set(key, { url: out.url, expires_at: out.expires_at, control: out.control, until: ends - 3 * 60000 });++ if (links.size > 500) links.delete(links.keys().next().value);++ return res.json({ url: out.url, expires_at: out.expires_at, control: out.control });++ });++const PROBLEM = {'actor.allowance.exhausted': 'You have used today\'s free tasks. They come back at midnight UTC.','actor.budget.day_spent': 'Today\'s budget is spent. It resets at midnight UTC.',@@@ -136,6 +136,15 @@ function deskFrom(events) {return desk;}++/**++ * Where a desk is, from Bot's screen page address (agents/computers.js watchUrl): { machine, desk }, the desk 'main'++ * when the address names none; null for any other address. What a watch link needs (plan L55 C1b).++ */++function deskPlace(url) {++ const at = /\/machines\/(dev_[A-Za-z0-9_-]{1,64})\/screen(?:\?desk=([a-z0-9-]{1,40}))?$/.exec(String(url || ''));++ return at ? { machine: at[1], desk: at[2] || 'main' } : null;++}++async function taskDesk(s, row) {if (!row || !OPEN.includes(row.state)) return null;const rows = await s.db.many('SELECT body FROM task_events WHERE task_id = $1 ORDER BY seq DESC LIMIT 200', [row.id]);@@@ -250,4 +259,4 @@ function toWire(row, { baseUrl }) {return t;}--module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, deskFrom, taskDesk, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, openTasks, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS, followContext, followUps, forks, recentSucceeded };++module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, deskFrom, deskPlace, taskDesk, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, openTasks, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS, followContext, followUps, forks, recentSucceeded };nnew file mode 100644@@@ -0,0 +1,89 @@++'use strict';++// The live desk in a chat turn (plan L55 C1b; the owner, 11-01: "show you the display… watch it… and steer it"): a++// turn working on one of the person's own agent desks asks Bot for a short-lived watch link and frames it. The link is++// a credential: only the task's requester gets it, only same-origin, only while the turn works, never stored.++// node test/chat-live-desk.test.js++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const { boot, check, done } = require('./helpers/boot');++const store = require('../server/tasks/store');++++const SAME = { 'sec-fetch-site': 'same-origin', accept: 'application/json' };++const MACHINE = 'dev_01JZ0000000000000000000000';++++(async () => {++ const asked = [];++ let answer = (desk) => ({ url: 'https://openvibe.bot/screen/tok_live_1', expires_at: new Date(Date.now() + 3600e3).toISOString(), control: true, desk });++ const computers = {++ async watchLink(subject, machine, desk, opts) { asked.push({ subject, machine, desk, opts }); return answer(desk); },++ };++ const t = await boot({ bot: { computers } });++ const kim = t.network.addUser('kim');++ const bo = t.network.addUser('bo');++ const s = t.ctx.s;++ try {++ const id = (await t.get('/api/v1/tasks', { as: kim, json: { task: 'Book the cheapest train to Porto' }, headers: SAME })).json().id;++ await t.waitFor(id);++ const deskEvent = (watch, running = true) => store.appendEvent(s, id, { step: 'tool_result', tool: 'computer_desk', is_error: false, text: JSON.stringify({ watch, running, kind: 'browser' }) });++++ await check('Bot\'s screen page address names the machine and the desk', async () => {++ assert.deepStrictEqual(store.deskPlace(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`), { machine: MACHINE, desk: 'desk-trains' });++ assert.deepStrictEqual(store.deskPlace(`https://openvibe.bot/machines/${MACHINE}/screen`), { machine: MACHINE, desk: 'main' }, 'no desk named is the machine\'s own screen');++ assert.strictEqual(store.deskPlace('https://example.com/elsewhere'), null);++ });++++ await check('a working turn on an agent desk offers the live desk; the owner\'s screen does not', async () => {++ await s.db.query("UPDATE tasks SET state = 'running' WHERE id = $1", [id]);++ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`);++ let page = await t.get(`/chat/${id}`, { as: kim });++ assert.match(page.text, new RegExp(`data-watch="/chat/${id}/watch"`), 'the figure says where to ask');++ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen`);++ page = await t.get(`/chat/${id}`, { as: kim });++ assert.ok(!page.text.includes('data-watch='), 'the owner\'s own screen is never framed');++ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`);++ });++++ await check('only the requester, from openvibe.actor, while it works, gets the link; it is reused, not remade', async () => {++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: bo, method: 'POST', headers: SAME })).status, 404, 'another person: not there');++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: { 'sec-fetch-site': 'cross-site', accept: 'application/json' } })).status, 403, 'another site');++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { method: 'POST', headers: SAME })).status, 401, 'signed out');++ const r = await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME });++ assert.strictEqual(r.status, 200, r.text);++ assert.deepStrictEqual(r.json().url, 'https://openvibe.bot/screen/tok_live_1');++ assert.match(r.headers.get('cache-control'), /no-store/);++ assert.deepStrictEqual(asked.map((a) => [a.subject, a.machine, a.desk, a.opts.control, a.opts.minutes]), [[kim.subject, MACHINE, 'desk-trains', true, 60]]);++ await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME });++ assert.strictEqual(asked.length, 1, 'the same link again until it nears its end');++ const events = await s.db.many('SELECT body FROM task_events WHERE task_id = $1', [id]);++ assert.ok(!JSON.stringify(events).includes('tok_live_1'), 'the link is never written to the task');++ });++++ await check('Bot saying no leaves the picture; a finished turn has no live desk', async () => {++ const other = (await t.get('/api/v1/tasks', { as: kim, json: { task: 'And back on Sunday' }, headers: SAME })).json().id;++ await t.waitFor(other);++ await s.db.query("UPDATE tasks SET state = 'running' WHERE id = $1", [other]);++ await store.appendEvent(s, other, { step: 'tool_result', tool: 'computer_desk', is_error: false, text: JSON.stringify({ watch: `https://openvibe.bot/machines/${MACHINE}/screen?desk=private`, running: true }) });++ answer = () => ({ error: 'capability.denied', code: 'capability.denied' });++ const no = await t.get(`/chat/${other}/watch`, { as: kim, method: 'POST', headers: SAME });++ assert.strictEqual(no.status, 502);++ assert.match(no.json().error, /the picture still updates/);++ await s.db.query("UPDATE tasks SET state = 'succeeded' WHERE id = $1", [id]);++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME })).status, 409);++ const page = await t.get(`/chat/${id}`, { as: kim });++ assert.ok(!page.text.includes('data-watch='), 'a finished turn keeps its last picture');++ });++++ await check('the page frames Bot\'s page (CSP), and the script swaps the picture for it', async () => {++ const page = await t.get('/chat', { as: kim });++ assert.match(page.headers.get('content-security-policy'), /frame-src [^;]*https:\/\/openvibe\.bot/);++ const js = fs.readFileSync(path.join(__dirname, '..', 'public', 'js', 'chat.js'), 'utf8');++ assert.match(js, /function live\(turn\)/);++ assert.match(js, /fetch\(fig\.getAttribute\('data-watch'\), \{ method: 'POST', credentials: 'same-origin'/);++ assert.match(js, /f\.setAttribute\('sandbox', 'allow-scripts allow-same-origin allow-forms allow-pointer-lock'\)/, 'the frame cannot move the page');++ assert.match(js, /stopScreen\(turn\);\s+\/\/ the frame moves by itself/);++ });++ } finally {++ await done(t);++ }++})();
@@ -202,6 +202,10 @@ html[data-ov-motion=reduced] .ct-answer.is-streaming .sw { opacity: 1 !important
@@ -202,6 +202,10 @@ html[data-ov-motion=reduced] .ct-answer.is-streaming .sw { opacity: 1 !important
.ct-screen[hidden] { display: none; }
.ct-screen[hidden] { display: none; }
.ct-screen-frame { position: relative; overflow: hidden; border-radius: 14px; border: 1px solid var(--line); background: #05070d; aspect-ratio: 16 / 10; display: grid; place-items: center; }
.ct-screen-frame { position: relative; overflow: hidden; border-radius: 14px; border: 1px solid var(--line); background: #05070d; aspect-ratio: 16 / 10; display: grid; place-items: center; }
.ct-screen-frame img { display: block; width: 100%; height: 100%; object-fit: contain; }
.ct-screen-frame img { display: block; width: 100%; height: 100%; object-fit: contain; }
+/* The live desk (plan L55 C1b): Bot's page in place of the picture, the same size, so nothing jumps. */
+.ct-screen-frame .ct-desk { position: absolute; inset: 0; width: 100%; height: 100%; border: 0; background: #05070d; }
+.ct-screen.is-framed .ct-screen-frame img { visibility: hidden; }
+.ct-screen.is-framed .ct-screen-frame { aspect-ratio: 16 / 10; }
.ct-screen-wait { padding: 16px; font-size: 14px; text-align: center; color: var(--text-muted, #7386a3); }
.ct-screen-wait { padding: 16px; font-size: 14px; text-align: center; color: var(--text-muted, #7386a3); }
.ct-live { position: absolute; top: 8px; left: 8px; display: inline-flex; align-items: center; gap: 5px; padding: 2px 8px; border-radius: 999px; background: rgba(0, 0, 0, .6); color: #fff; font-size: 11.5px; font-weight: 700; letter-spacing: .04em; text-transform: uppercase; }
.ct-live { position: absolute; top: 8px; left: 8px; display: inline-flex; align-items: center; gap: 5px; padding: 2px 8px; border-radius: 999px; background: rgba(0, 0, 0, .6); color: #fff; font-size: 11.5px; font-weight: 700; letter-spacing: .04em; text-transform: uppercase; }
.ct-live::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: #ef4444; animation: ct-breathe 1.4s ease-in-out infinite; }
.ct-live::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: #ef4444; animation: ct-breathe 1.4s ease-in-out infinite; }
@@ -286,9 +286,44 @@
@@ -286,9 +286,44 @@
}
}
// What Actor sees: the last picture, fetched every two seconds while the turn works (only when it changed).
// What Actor sees: the last picture, fetched every two seconds while the turn works (only when it changed).
var screens = {};
var screens = {};
+ // The live desk (plan L55 C1b): when the turn works on one of the person's own desks, the server asks Bot for a
+ // short-lived page that shows it moving and takes the mouse, and the turn frames it in place of the picture.
+ // If Bot cannot, the picture keeps refreshing as before.
+ function live(turn) {
+ var fig = turn.querySelector('[data-screen][data-watch]');
+ if (!fig || fig.hasAttribute('data-framed') || !turn.hasAttribute('data-open')) return;
+ fig.setAttribute('data-framed', 'asking');
+ fetch(fig.getAttribute('data-watch'), { method: 'POST', credentials: 'same-origin', headers: { Accept: 'application/json' } })
+ .then(function (r) { return r.ok ? r.json() : null; })
+ .catch(function () { return null; })
+ .then(function (j) {
+ if (!j || !/^https:\/\//.test(j.url || '') || !fig.isConnected) { fig.setAttribute('data-framed', 'no'); return; }
+ var frame = fig.querySelector('.ct-screen-frame');
+ if (!frame) { frame = document.createElement('div'); frame.className = 'ct-screen-frame'; fig.insertBefore(frame, fig.firstChild); }
+ var f = document.createElement('iframe');
+ f.src = j.url;
+ f.title = 'Actor\'s desk, live. Click into it to take over.';
+ f.setAttribute('allow', 'fullscreen; clipboard-write');
+ f.setAttribute('referrerpolicy', 'no-referrer');
+ f.setAttribute('sandbox', 'allow-scripts allow-same-origin allow-forms allow-pointer-lock');
+ f.className = 'ct-desk';
+ frame.appendChild(f);
+ fig.classList.add('is-framed');
+ fig.hidden = false;
+ fig.setAttribute('data-framed', 'yes');
+ stopScreen(turn); // the frame moves by itself: no more pictures to fetch
+ var a = fig.querySelector('.ct-watch');
+ if (a) { a.href = j.url; a.lastChild.textContent = 'Open it big'; }
+ var cap = fig.querySelector('figcaption > span');
+ if (cap) cap.textContent = j.control ? 'Actor\'s desk, live: click into it to take over' : 'Actor\'s desk, live';
+ });
+ }
function screen(turn) {
function screen(turn) {
var id = turn.getAttribute('data-task');
var id = turn.getAttribute('data-task');
+ live(turn);
if (screens[id] || !turn.hasAttribute('data-open')) return;
if (screens[id] || !turn.hasAttribute('data-open')) return;
+ var framed = turn.querySelector('[data-screen][data-framed="yes"]');
+ if (framed) return;
var fig = turn.querySelector('[data-screen]');
var fig = turn.querySelector('[data-screen]');
if (!fig) return;
if (!fig) return;
var busyLoad = false;
var busyLoad = false;
@@ -372,7 +372,39 @@ function createComputers({ config, fetchImpl = globalThis.fetch, tokens: tokensO
@@ -372,7 +372,39 @@ function createComputers({ config, fetchImpl = globalThis.fetch, tokens: tokensO
/** The person's page to watch a desk live and take it over (Bot's own screen page). */
/** The person's page to watch a desk live and take it over (Bot's own screen page). */
const watchUrl = (machine, desk) => `${pages}/machines/${machine}/screen${desk && desk !== 'main' ? `?desk=${encodeURIComponent(desk)}` : ''}`;
const watchUrl = (machine, desk) => `${pages}/machines/${machine}/screen${desk && desk !== 'main' ? `?desk=${encodeURIComponent(desk)}` : ''}`;
- return { enabled, mine, list, read, run, write, screen, input, browser, desk, deskTurn, watchUrl, openSession, endSession, sessionsOn, isSession: (m) => SESSION_MACHINE.test(String(m || '')) };
+ // A watch link (plan L55 C1b, contracts 0.268.0 bot.machine.watch_link): a short-lived page that shows the desk the
+ // agent works on, live, and lets the person take it over, framable in the chat (no sign-in: the token is the
+ // credential). Its own token client: a Network that has not granted the capability yet refuses only this, never
+ // the computer control above.
+ const linkTokens = !enabled ? null : tokensOverride || serviceAuth.createTokenClient({
+ tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
+ audience: 'openvibe.bot', scope: 'bot.machine.watch_link', fetchImpl,
+ });
+ async function watchLink(subject, machine, desk, { control = true, minutes = 60, signal } = {}) {
+ if (!enabled) return { error: 'computers are off here' };
+ const bad = guard(subject, machine);
+ if (bad) return bad;
+ if (SESSION_MACHINE.test(String(machine))) return { error: 'a desk host session has no watch link' };
+ // The agent's own desks only: Bot refuses the owner's screen (main) to an agent anyway.
+ if (desk !== 'private' && !DESK_NAME.test(String(desk || ''))) return { error: 'only an agent desk (private or desk-<name>) has a watch link' };
+ const sig = signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000);
+ let res;
+ try {
+ res = await fetchImpl(`${base}/api/v1/machines/${encodeURIComponent(machine)}/watch-links`, {
+ method: 'POST',
+ headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'X-OV-Subject': subject, ...(await linkTokens.authHeaders()) },
+ body: JSON.stringify({ desk, control: !!control, minutes, label: 'the chat on openvibe.actor' }), signal: sig,
+ });
+ } catch {
+ return { error: 'Bot did not answer for the watch link' };
+ }
+ const data = await res.json().catch(() => null);
+ if (!res.ok) return problem({ status: res.status, data }, 'the watch link');
+ if (!data || !/^https:\/\/[^\s"<>]{1,300}$/.test(String(data.url || ''))) return { error: 'Bot answered without a link' };
+ return { url: data.url, expires_at: data.expires_at || null, control: !!data.control, desk: data.desk || desk };
+ }
+
+ return { enabled, mine, list, read, run, write, screen, input, browser, desk, deskTurn, watchUrl, watchLink, openSession, endSession, sessionsOn, isSession: (m) => SESSION_MACHINE.test(String(m || '')) };
}
}
module.exports = { createComputers };
module.exports = { createComputers };
@@ -177,7 +177,8 @@ async function createApp(opts = {}) {
@@ -177,7 +177,8 @@ async function createApp(opts = {}) {
fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com', 'https://cdnjs.cloudflare.com'],
fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com', 'https://cdnjs.cloudflare.com'],
imgSrc: ["'self'", 'data:', 'https://openvibe.network', 'https://openvibe.media'],
imgSrc: ["'self'", 'data:', 'https://openvibe.network', 'https://openvibe.media'],
connectSrc: ["'self'", 'https://openvibe.network', 'https://cloudflareinsights.com', 'https://openvibe.events'],
connectSrc: ["'self'", 'https://openvibe.network', 'https://cloudflareinsights.com', 'https://openvibe.events'],
- frameSrc: ["'self'", 'https://openvibe.network'],
+ // Bot's watch links: a chat turn frames the desk the agent works on, live (plan L55 C1b).
+ frameSrc: ["'self'", 'https://openvibe.network', ...(() => { try { return [new URL(config.services.botPublicUrl).origin]; } catch { return []; } })()],
frameAncestors: ["'none'"],
frameAncestors: ["'none'"],
objectSrc: ["'none'"],
objectSrc: ["'none'"],
baseUri: ["'self'"],
baseUri: ["'self'"],
@@ -242,7 +243,7 @@ async function createApp(opts = {}) {
@@ -242,7 +243,7 @@ async function createApp(opts = {}) {
app.use(createMemoryPages(ctx));
app.use(createMemoryPages(ctx));
app.use(createConnectionsPages(ctx));
app.use(createConnectionsPages(ctx));
app.use(createConsolePages(ctx, opts.bot || {}));
app.use(createConsolePages(ctx, opts.bot || {}));
- app.use(createChatPages(ctx));
+ app.use(createChatPages(ctx, opts.bot || {}));
const pageRoutes = createPageRoutes(ctx);
const pageRoutes = createPageRoutes(ctx);
// The agent layer (openvibe-shared/agent-layer): /.well-known/agent.json and the Markdown twins of the public
// The agent layer (openvibe-shared/agent-layer): /.well-known/agent.json and the Markdown twins of the public
// pages. The pages themselves keep their own routes — a signed-in visitor's account bar and task box are
// pages. The pages themselves keep their own routes — a signed-in visitor's account bar and task box are
@@ -79,9 +79,15 @@ function tookOf(row) {
@@ -79,9 +79,15 @@ function tookOf(row) {
return s < 60 ? `${s} s` : `${Math.floor(s / 60)} min${s % 60 ? ` ${s % 60} s` : ''}`;
return s < 60 ? `${s} s` : `${Math.floor(s / 60)} min${s % 60 ? ` ${s % 60} s` : ''}`;
}
}
-function createChatPages(ctx) {
+function createChatPages(ctx, { computers = null } = {}) {
const { config, s, engine } = ctx;
const { config, s, engine } = ctx;
const r = asyncRouter();
const r = asyncRouter();
+ // Bot's computers, for the live desk in a turn (plan L55 C1b): made when first asked for.
+ let pcsMade = computers;
+ const pcs = () => (pcsMade = pcsMade || require('../agents/computers').createComputers({ config }));
+ // Watch links are credentials (whoever holds one watches, and may take over): kept in memory for the task's own
+ // requester, never written to the task's events or the database, and reused until a few minutes before they end.
+ const links = new Map();
// Helpers from the task page, read when first used (pages.js is the bigger module; no load-order coupling).
// Helpers from the task page, read when first used (pages.js is the bigger module; no load-order coupling).
const pageKit = () => require('./pages');
const pageKit = () => require('./pages');
const signedIn = (req) => req.viewer && req.viewer.kind === 'user' && req.viewer.subject;
const signedIn = (req) => req.viewer && req.viewer.kind === 'user' && req.viewer.subject;
@@ -145,7 +151,9 @@ function createChatPages(ctx) {
@@ -145,7 +151,9 @@ function createChatPages(ctx) {
const desk = open ? store.deskFrom(events) : null;
const desk = open ? store.deskFrom(events) : null;
const look = looks.get(row.id);
const look = looks.get(row.id);
if (!desk && !look) return open ? html`<figure class="ct-screen" data-screen hidden></figure>` : '';
if (!desk && !look) return open ? html`<figure class="ct-screen" data-screen hidden></figure>` : '';
- return html`<figure class="ct-screen${open ? ' is-live' : ''}" data-screen${desk ? html` data-desk="${desk.url}"` : ''}>
+ const place = desk && store.deskPlace(desk.url);
+ const framable = !!(place && place.desk !== 'main');
+ return html`<figure class="ct-screen${open ? ' is-live' : ''}" data-screen${desk ? html` data-desk="${desk.url}"` : ''}${framable ? html` data-watch="/chat/${row.id}/watch"` : ''}>
<div class="ct-screen-frame">${look ? html`<img data-look src="/tasks/${row.id}/look?at=${look.at}" alt="What Actor ${open ? 'sees now' : 'last saw'}"${look.width && look.height ? html` width="${look.width}" height="${look.height}"` : ''}>` : html`<span class="ct-screen-wait">The screen shows here as soon as Actor looks at it.</span>`}${open ? html`<span class="ct-live">Live</span>` : ''}</div>
<div class="ct-screen-frame">${look ? html`<img data-look src="/tasks/${row.id}/look?at=${look.at}" alt="What Actor ${open ? 'sees now' : 'last saw'}"${look.width && look.height ? html` width="${look.width}" height="${look.height}"` : ''}>` : html`<span class="ct-screen-wait">The screen shows here as soon as Actor looks at it.</span>`}${open ? html`<span class="ct-live">Live</span>` : ''}</div>
<figcaption><span>${open ? `What Actor sees${desk && desk.kind === 'browser' ? ' in its browser' : ''}` : 'What Actor last saw'}</span>${desk ? html`<a class="ct-watch" href="${desk.url}" target="_blank" rel="noopener">${raw(ICON.open)}Watch live and take over</a>` : ''}</figcaption>
<figcaption><span>${open ? `What Actor sees${desk && desk.kind === 'browser' ? ' in its browser' : ''}` : 'What Actor last saw'}</span>${desk ? html`<a class="ct-watch" href="${desk.url}" target="_blank" rel="noopener">${raw(ICON.open)}Watch live and take over</a>` : ''}</figcaption>
</figure>`;
</figure>`;
@@ -391,6 +399,30 @@ ${composer({ parent: lastTurn ? lastTurn.id : null, mode: lastTurn ? lastTurn.mo
@@ -391,6 +399,30 @@ ${composer({ parent: lastTurn ? lastTurn.id : null, mode: lastTurn ? lastTurn.mo
return res.redirect(303, `/chat/${row.id}#t-${row.id}`);
return res.redirect(303, `/chat/${row.id}#t-${row.id}`);
});
});
+ // The live desk in a working turn (plan L55 C1b; the owner, 11-01: "show you the display… watch it… and steer it").
+ // The page asks once the turn is on a desk; the answer is a short-lived Bot page it frames, where the person can
+ // take the mouse. Only the task's requester, only same-origin, only while the turn works.
+ r.post('/chat/:id/watch', async (req, res) => {
+ res.set('Cache-Control', 'private, no-store');
+ if (!signedIn(req) || !TASK_ID.test(req.params.id)) return res.status(401).json({ error: 'Sign in first.' });
+ if (!sameOrigin(req, config.baseUrl)) return res.status(403).json({ error: 'Ask from openvibe.actor itself.' });
+ const row = await store.getTask(s, req.params.id);
+ if (!row || row.requester !== requesterOf(req)) return res.status(404).json({ error: 'That chat is not here.' });
+ if (!OPEN.includes(row.state)) return res.status(409).json({ error: 'Actor has finished that one.' });
+ const on = await store.taskDesk(s, row);
+ const desk = on && store.deskPlace(on.url);
+ if (!desk) return res.status(409).json({ error: 'Actor is not on a desk just now.' });
+ const key = `${row.id}|${desk.machine}|${desk.desk}`;
+ const kept = links.get(key);
+ if (kept && kept.until > now()) return res.json({ url: kept.url, expires_at: kept.expires_at, control: kept.control });
+ const out = await pcs().watchLink(req.viewer.subject, desk.machine, desk.desk, { control: true, minutes: 60 });
+ if (out.error) return res.status(502).json({ error: 'The live desk is not available here; the picture still updates.', reason: out.code || null });
+ const ends = Date.parse(out.expires_at || '') || now() + 60 * 60000;
+ links.set(key, { url: out.url, expires_at: out.expires_at, control: out.control, until: ends - 3 * 60000 });
+ if (links.size > 500) links.delete(links.keys().next().value);
+ return res.json({ url: out.url, expires_at: out.expires_at, control: out.control });
+ });
+
const PROBLEM = {
const PROBLEM = {
'actor.allowance.exhausted': 'You have used today\'s free tasks. They come back at midnight UTC.',
'actor.allowance.exhausted': 'You have used today\'s free tasks. They come back at midnight UTC.',
'actor.budget.day_spent': 'Today\'s budget is spent. It resets at midnight UTC.',
'actor.budget.day_spent': 'Today\'s budget is spent. It resets at midnight UTC.',
@@ -136,6 +136,15 @@ function deskFrom(events) {
@@ -136,6 +136,15 @@ function deskFrom(events) {
return desk;
return desk;
}
}
+/**
+ * Where a desk is, from Bot's screen page address (agents/computers.js watchUrl): { machine, desk }, the desk 'main'
+ * when the address names none; null for any other address. What a watch link needs (plan L55 C1b).
+ */
+function deskPlace(url) {
+ const at = /\/machines\/(dev_[A-Za-z0-9_-]{1,64})\/screen(?:\?desk=([a-z0-9-]{1,40}))?$/.exec(String(url || ''));
+ return at ? { machine: at[1], desk: at[2] || 'main' } : null;
+}
+
async function taskDesk(s, row) {
async function taskDesk(s, row) {
if (!row || !OPEN.includes(row.state)) return null;
if (!row || !OPEN.includes(row.state)) return null;
const rows = await s.db.many('SELECT body FROM task_events WHERE task_id = $1 ORDER BY seq DESC LIMIT 200', [row.id]);
const rows = await s.db.many('SELECT body FROM task_events WHERE task_id = $1 ORDER BY seq DESC LIMIT 200', [row.id]);
@@ -250,4 +259,4 @@ function toWire(row, { baseUrl }) {
@@ -250,4 +259,4 @@ function toWire(row, { baseUrl }) {
return t;
return t;
}
}
-module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, deskFrom, taskDesk, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, openTasks, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS, followContext, followUps, forks, recentSucceeded };
+module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, deskFrom, deskPlace, taskDesk, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, openTasks, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS, followContext, followUps, forks, recentSucceeded };
new file mode 100644
new file mode 100644
@@ -0,0 +1,89 @@
@@ -0,0 +1,89 @@
+'use strict';
+// The live desk in a chat turn (plan L55 C1b; the owner, 11-01: "show you the display… watch it… and steer it"): a
+// turn working on one of the person's own agent desks asks Bot for a short-lived watch link and frames it. The link is
+// a credential: only the task's requester gets it, only same-origin, only while the turn works, never stored.
+// node test/chat-live-desk.test.js
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { boot, check, done } = require('./helpers/boot');
+const store = require('../server/tasks/store');
+
+const SAME = { 'sec-fetch-site': 'same-origin', accept: 'application/json' };
+const MACHINE = 'dev_01JZ0000000000000000000000';
+
+(async () => {
+ const asked = [];
+ let answer = (desk) => ({ url: 'https://openvibe.bot/screen/tok_live_1', expires_at: new Date(Date.now() + 3600e3).toISOString(), control: true, desk });
+ const computers = {
+ async watchLink(subject, machine, desk, opts) { asked.push({ subject, machine, desk, opts }); return answer(desk); },
+ };
+ const t = await boot({ bot: { computers } });
+ const kim = t.network.addUser('kim');
+ const bo = t.network.addUser('bo');
+ const s = t.ctx.s;
+ try {
+ const id = (await t.get('/api/v1/tasks', { as: kim, json: { task: 'Book the cheapest train to Porto' }, headers: SAME })).json().id;
+ await t.waitFor(id);
+ const deskEvent = (watch, running = true) => store.appendEvent(s, id, { step: 'tool_result', tool: 'computer_desk', is_error: false, text: JSON.stringify({ watch, running, kind: 'browser' }) });
+
+ await check('Bot\'s screen page address names the machine and the desk', async () => {
+ assert.deepStrictEqual(store.deskPlace(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`), { machine: MACHINE, desk: 'desk-trains' });
+ assert.deepStrictEqual(store.deskPlace(`https://openvibe.bot/machines/${MACHINE}/screen`), { machine: MACHINE, desk: 'main' }, 'no desk named is the machine\'s own screen');
+ assert.strictEqual(store.deskPlace('https://example.com/elsewhere'), null);
+ });
+
+ await check('a working turn on an agent desk offers the live desk; the owner\'s screen does not', async () => {
+ await s.db.query("UPDATE tasks SET state = 'running' WHERE id = $1", [id]);
+ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`);
+ let page = await t.get(`/chat/${id}`, { as: kim });
+ assert.match(page.text, new RegExp(`data-watch="/chat/${id}/watch"`), 'the figure says where to ask');
+ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen`);
+ page = await t.get(`/chat/${id}`, { as: kim });
+ assert.ok(!page.text.includes('data-watch='), 'the owner\'s own screen is never framed');
+ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`);
+ });
+
+ await check('only the requester, from openvibe.actor, while it works, gets the link; it is reused, not remade', async () => {
+ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: bo, method: 'POST', headers: SAME })).status, 404, 'another person: not there');
+ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: { 'sec-fetch-site': 'cross-site', accept: 'application/json' } })).status, 403, 'another site');
+ assert.strictEqual((await t.get(`/chat/${id}/watch`, { method: 'POST', headers: SAME })).status, 401, 'signed out');
+ const r = await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME });
+ assert.strictEqual(r.status, 200, r.text);
+ assert.deepStrictEqual(r.json().url, 'https://openvibe.bot/screen/tok_live_1');
+ assert.match(r.headers.get('cache-control'), /no-store/);
+ assert.deepStrictEqual(asked.map((a) => [a.subject, a.machine, a.desk, a.opts.control, a.opts.minutes]), [[kim.subject, MACHINE, 'desk-trains', true, 60]]);
+ await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME });
+ assert.strictEqual(asked.length, 1, 'the same link again until it nears its end');
+ const events = await s.db.many('SELECT body FROM task_events WHERE task_id = $1', [id]);
+ assert.ok(!JSON.stringify(events).includes('tok_live_1'), 'the link is never written to the task');
+ });
+
+ await check('Bot saying no leaves the picture; a finished turn has no live desk', async () => {
+ const other = (await t.get('/api/v1/tasks', { as: kim, json: { task: 'And back on Sunday' }, headers: SAME })).json().id;
+ await t.waitFor(other);
+ await s.db.query("UPDATE tasks SET state = 'running' WHERE id = $1", [other]);
+ await store.appendEvent(s, other, { step: 'tool_result', tool: 'computer_desk', is_error: false, text: JSON.stringify({ watch: `https://openvibe.bot/machines/${MACHINE}/screen?desk=private`, running: true }) });
+ answer = () => ({ error: 'capability.denied', code: 'capability.denied' });
+ const no = await t.get(`/chat/${other}/watch`, { as: kim, method: 'POST', headers: SAME });
+ assert.strictEqual(no.status, 502);
+ assert.match(no.json().error, /the picture still updates/);
+ await s.db.query("UPDATE tasks SET state = 'succeeded' WHERE id = $1", [id]);
+ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME })).status, 409);
+ const page = await t.get(`/chat/${id}`, { as: kim });
+ assert.ok(!page.text.includes('data-watch='), 'a finished turn keeps its last picture');
+ });
+
+ await check('the page frames Bot\'s page (CSP), and the script swaps the picture for it', async () => {
+ const page = await t.get('/chat', { as: kim });
+ assert.match(page.headers.get('content-security-policy'), /frame-src [^;]*https:\/\/openvibe\.bot/);
+ const js = fs.readFileSync(path.join(__dirname, '..', 'public', 'js', 'chat.js'), 'utf8');
+ assert.match(js, /function live\(turn\)/);
+ assert.match(js, /fetch\(fig\.getAttribute\('data-watch'\), \{ method: 'POST', credentials: 'same-origin'/);
+ assert.match(js, /f\.setAttribute\('sandbox', 'allow-scripts allow-same-origin allow-forms allow-pointer-lock'\)/, 'the frame cannot move the page');
+ assert.match(js, /stopScreen\(turn\);\s+\/\/ the frame moves by itself/);
+ });
+ } finally {
+ await done(t);
+ }
+})();