openvibers/openvibe.actor Commits

Chat: the agent's desk, live, inside the turn; click into it to take over (#233)

OP OpenVibers committed 12 days ago · 487f68ee94 · 7 files, +208 −6

Plan L55 C1b (the owner, 11-01: "show you the display… watch it… just chill while you do shit, and steer it").
A turn working on one of the person's own agent desks (private or desk-<name>, never their own screen) asks Bot for a
watch link (bot.machine.watch_link, contracts 0.268.0) and frames it in place of the refreshed picture: the desk
moves live and the person can take the mouse. The link is a credential: only the task's requester gets it, only from
openvibe.actor, only while the turn works; it lives in memory (reused until near its end), never in the task's
events. Its own token client, so a missing grant refuses only this. Bot saying no keeps the picture. The frame is
sandboxed (no top navigation); CSP frame-src allows Bot's public origin.

Changes

ddiff --git a/public/css/chat.css b/public/css/chat.css
iindex fa20429..1b2ae6f 100644
---- a/public/css/chat.css
++++ b/public/css/chat.css
@@@ -202,6 +202,10 @@ html[data-ov-motion=reduced] .ct-answer.is-streaming .sw { opacity: 1 !important
.ct-screen[hidden] { display: none; }
.ct-screen-frame { position: relative; overflow: hidden; border-radius: 14px; border: 1px solid var(--line); background: #05070d; aspect-ratio: 16 / 10; display: grid; place-items: center; }
.ct-screen-frame img { display: block; width: 100%; height: 100%; object-fit: contain; }
++/* The live desk (plan L55 C1b): Bot's page in place of the picture, the same size, so nothing jumps. */
++.ct-screen-frame .ct-desk { position: absolute; inset: 0; width: 100%; height: 100%; border: 0; background: #05070d; }
++.ct-screen.is-framed .ct-screen-frame img { visibility: hidden; }
++.ct-screen.is-framed .ct-screen-frame { aspect-ratio: 16 / 10; }
.ct-screen-wait { padding: 16px; font-size: 14px; text-align: center; color: var(--text-muted, #7386a3); }
.ct-live { position: absolute; top: 8px; left: 8px; display: inline-flex; align-items: center; gap: 5px; padding: 2px 8px; border-radius: 999px; background: rgba(0, 0, 0, .6); color: #fff; font-size: 11.5px; font-weight: 700; letter-spacing: .04em; text-transform: uppercase; }
.ct-live::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: #ef4444; animation: ct-breathe 1.4s ease-in-out infinite; }
ddiff --git a/public/js/chat.js b/public/js/chat.js
iindex 4d1b39f..df22cf5 100644
---- a/public/js/chat.js
++++ b/public/js/chat.js
@@@ -286,9 +286,44 @@
}
// What Actor sees: the last picture, fetched every two seconds while the turn works (only when it changed).
var screens = {};
++ // The live desk (plan L55 C1b): when the turn works on one of the person's own desks, the server asks Bot for a
++ // short-lived page that shows it moving and takes the mouse, and the turn frames it in place of the picture.
++ // If Bot cannot, the picture keeps refreshing as before.
++ function live(turn) {
++ var fig = turn.querySelector('[data-screen][data-watch]');
++ if (!fig || fig.hasAttribute('data-framed') || !turn.hasAttribute('data-open')) return;
++ fig.setAttribute('data-framed', 'asking');
++ fetch(fig.getAttribute('data-watch'), { method: 'POST', credentials: 'same-origin', headers: { Accept: 'application/json' } })
++ .then(function (r) { return r.ok ? r.json() : null; })
++ .catch(function () { return null; })
++ .then(function (j) {
++ if (!j || !/^https:\/\//.test(j.url || '') || !fig.isConnected) { fig.setAttribute('data-framed', 'no'); return; }
++ var frame = fig.querySelector('.ct-screen-frame');
++ if (!frame) { frame = document.createElement('div'); frame.className = 'ct-screen-frame'; fig.insertBefore(frame, fig.firstChild); }
++ var f = document.createElement('iframe');
++ f.src = j.url;
++ f.title = 'Actor\'s desk, live. Click into it to take over.';
++ f.setAttribute('allow', 'fullscreen; clipboard-write');
++ f.setAttribute('referrerpolicy', 'no-referrer');
++ f.setAttribute('sandbox', 'allow-scripts allow-same-origin allow-forms allow-pointer-lock');
++ f.className = 'ct-desk';
++ frame.appendChild(f);
++ fig.classList.add('is-framed');
++ fig.hidden = false;
++ fig.setAttribute('data-framed', 'yes');
++ stopScreen(turn); // the frame moves by itself: no more pictures to fetch
++ var a = fig.querySelector('.ct-watch');
++ if (a) { a.href = j.url; a.lastChild.textContent = 'Open it big'; }
++ var cap = fig.querySelector('figcaption > span');
++ if (cap) cap.textContent = j.control ? 'Actor\'s desk, live: click into it to take over' : 'Actor\'s desk, live';
++ });
++ }
function screen(turn) {
var id = turn.getAttribute('data-task');
++ live(turn);
if (screens[id] || !turn.hasAttribute('data-open')) return;
++ var framed = turn.querySelector('[data-screen][data-framed="yes"]');
++ if (framed) return;
var fig = turn.querySelector('[data-screen]');
if (!fig) return;
var busyLoad = false;
ddiff --git a/server/agents/computers.js b/server/agents/computers.js
iindex b01f011..85b9beb 100644
---- a/server/agents/computers.js
++++ b/server/agents/computers.js
@@@ -372,7 +372,39 @@ function createComputers({ config, fetchImpl = globalThis.fetch, tokens: tokensO
/** The person's page to watch a desk live and take it over (Bot's own screen page). */
const watchUrl = (machine, desk) => `${pages}/machines/${machine}/screen${desk && desk !== 'main' ? `?desk=${encodeURIComponent(desk)}` : ''}`;
-- return { enabled, mine, list, read, run, write, screen, input, browser, desk, deskTurn, watchUrl, openSession, endSession, sessionsOn, isSession: (m) => SESSION_MACHINE.test(String(m || '')) };
++ // A watch link (plan L55 C1b, contracts 0.268.0 bot.machine.watch_link): a short-lived page that shows the desk the
++ // agent works on, live, and lets the person take it over, framable in the chat (no sign-in: the token is the
++ // credential). Its own token client: a Network that has not granted the capability yet refuses only this, never
++ // the computer control above.
++ const linkTokens = !enabled ? null : tokensOverride || serviceAuth.createTokenClient({
++ tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
++ audience: 'openvibe.bot', scope: 'bot.machine.watch_link', fetchImpl,
++ });
++ async function watchLink(subject, machine, desk, { control = true, minutes = 60, signal } = {}) {
++ if (!enabled) return { error: 'computers are off here' };
++ const bad = guard(subject, machine);
++ if (bad) return bad;
++ if (SESSION_MACHINE.test(String(machine))) return { error: 'a desk host session has no watch link' };
++ // The agent's own desks only: Bot refuses the owner's screen (main) to an agent anyway.
++ if (desk !== 'private' && !DESK_NAME.test(String(desk || ''))) return { error: 'only an agent desk (private or desk-<name>) has a watch link' };
++ const sig = signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000);
++ let res;
++ try {
++ res = await fetchImpl(`${base}/api/v1/machines/${encodeURIComponent(machine)}/watch-links`, {
++ method: 'POST',
++ headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'X-OV-Subject': subject, ...(await linkTokens.authHeaders()) },
++ body: JSON.stringify({ desk, control: !!control, minutes, label: 'the chat on openvibe.actor' }), signal: sig,
++ });
++ } catch {
++ return { error: 'Bot did not answer for the watch link' };
++ }
++ const data = await res.json().catch(() => null);
++ if (!res.ok) return problem({ status: res.status, data }, 'the watch link');
++ if (!data || !/^https:\/\/[^\s"<>]{1,300}$/.test(String(data.url || ''))) return { error: 'Bot answered without a link' };
++ return { url: data.url, expires_at: data.expires_at || null, control: !!data.control, desk: data.desk || desk };
++ }
++
++ return { enabled, mine, list, read, run, write, screen, input, browser, desk, deskTurn, watchUrl, watchLink, openSession, endSession, sessionsOn, isSession: (m) => SESSION_MACHINE.test(String(m || '')) };
}
module.exports = { createComputers };
ddiff --git a/server/app.js b/server/app.js
iindex cd110b1..477aae7 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -177,7 +177,8 @@ async function createApp(opts = {}) {
fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com', 'https://cdnjs.cloudflare.com'],
imgSrc: ["'self'", 'data:', 'https://openvibe.network', 'https://openvibe.media'],
connectSrc: ["'self'", 'https://openvibe.network', 'https://cloudflareinsights.com', 'https://openvibe.events'],
-- frameSrc: ["'self'", 'https://openvibe.network'],
++ // Bot's watch links: a chat turn frames the desk the agent works on, live (plan L55 C1b).
++ frameSrc: ["'self'", 'https://openvibe.network', ...(() => { try { return [new URL(config.services.botPublicUrl).origin]; } catch { return []; } })()],
frameAncestors: ["'none'"],
objectSrc: ["'none'"],
baseUri: ["'self'"],
@@@ -242,7 +243,7 @@ async function createApp(opts = {}) {
app.use(createMemoryPages(ctx));
app.use(createConnectionsPages(ctx));
app.use(createConsolePages(ctx, opts.bot || {}));
-- app.use(createChatPages(ctx));
++ app.use(createChatPages(ctx, opts.bot || {}));
const pageRoutes = createPageRoutes(ctx);
// The agent layer (openvibe-shared/agent-layer): /.well-known/agent.json and the Markdown twins of the public
// pages. The pages themselves keep their own routes — a signed-in visitor's account bar and task box are
ddiff --git a/server/http/chat-pages.js b/server/http/chat-pages.js
iindex 662cc54..518b58a 100644
---- a/server/http/chat-pages.js
++++ b/server/http/chat-pages.js
@@@ -79,9 +79,15 @@ function tookOf(row) {
return s < 60 ? `${s} s` : `${Math.floor(s / 60)} min${s % 60 ? ` ${s % 60} s` : ''}`;
}
--function createChatPages(ctx) {
++function createChatPages(ctx, { computers = null } = {}) {
const { config, s, engine } = ctx;
const r = asyncRouter();
++ // Bot's computers, for the live desk in a turn (plan L55 C1b): made when first asked for.
++ let pcsMade = computers;
++ const pcs = () => (pcsMade = pcsMade || require('../agents/computers').createComputers({ config }));
++ // Watch links are credentials (whoever holds one watches, and may take over): kept in memory for the task's own
++ // requester, never written to the task's events or the database, and reused until a few minutes before they end.
++ const links = new Map();
// Helpers from the task page, read when first used (pages.js is the bigger module; no load-order coupling).
const pageKit = () => require('./pages');
const signedIn = (req) => req.viewer && req.viewer.kind === 'user' && req.viewer.subject;
@@@ -145,7 +151,9 @@ function createChatPages(ctx) {
const desk = open ? store.deskFrom(events) : null;
const look = looks.get(row.id);
if (!desk && !look) return open ? html`<figure class="ct-screen" data-screen hidden></figure>` : '';
-- return html`<figure class="ct-screen${open ? ' is-live' : ''}" data-screen${desk ? html` data-desk="${desk.url}"` : ''}>
++ const place = desk && store.deskPlace(desk.url);
++ const framable = !!(place && place.desk !== 'main');
++ return html`<figure class="ct-screen${open ? ' is-live' : ''}" data-screen${desk ? html` data-desk="${desk.url}"` : ''}${framable ? html` data-watch="/chat/${row.id}/watch"` : ''}>
<div class="ct-screen-frame">${look ? html`<img data-look src="/tasks/${row.id}/look?at=${look.at}" alt="What Actor ${open ? 'sees now' : 'last saw'}"${look.width && look.height ? html` width="${look.width}" height="${look.height}"` : ''}>` : html`<span class="ct-screen-wait">The screen shows here as soon as Actor looks at it.</span>`}${open ? html`<span class="ct-live">Live</span>` : ''}</div>
<figcaption><span>${open ? `What Actor sees${desk && desk.kind === 'browser' ? ' in its browser' : ''}` : 'What Actor last saw'}</span>${desk ? html`<a class="ct-watch" href="${desk.url}" target="_blank" rel="noopener">${raw(ICON.open)}Watch live and take over</a>` : ''}</figcaption>
</figure>`;
@@@ -391,6 +399,30 @@ ${composer({ parent: lastTurn ? lastTurn.id : null, mode: lastTurn ? lastTurn.mo
return res.redirect(303, `/chat/${row.id}#t-${row.id}`);
});
++ // The live desk in a working turn (plan L55 C1b; the owner, 11-01: "show you the display… watch it… and steer it").
++ // The page asks once the turn is on a desk; the answer is a short-lived Bot page it frames, where the person can
++ // take the mouse. Only the task's requester, only same-origin, only while the turn works.
++ r.post('/chat/:id/watch', async (req, res) => {
++ res.set('Cache-Control', 'private, no-store');
++ if (!signedIn(req) || !TASK_ID.test(req.params.id)) return res.status(401).json({ error: 'Sign in first.' });
++ if (!sameOrigin(req, config.baseUrl)) return res.status(403).json({ error: 'Ask from openvibe.actor itself.' });
++ const row = await store.getTask(s, req.params.id);
++ if (!row || row.requester !== requesterOf(req)) return res.status(404).json({ error: 'That chat is not here.' });
++ if (!OPEN.includes(row.state)) return res.status(409).json({ error: 'Actor has finished that one.' });
++ const on = await store.taskDesk(s, row);
++ const desk = on && store.deskPlace(on.url);
++ if (!desk) return res.status(409).json({ error: 'Actor is not on a desk just now.' });
++ const key = `${row.id}|${desk.machine}|${desk.desk}`;
++ const kept = links.get(key);
++ if (kept && kept.until > now()) return res.json({ url: kept.url, expires_at: kept.expires_at, control: kept.control });
++ const out = await pcs().watchLink(req.viewer.subject, desk.machine, desk.desk, { control: true, minutes: 60 });
++ if (out.error) return res.status(502).json({ error: 'The live desk is not available here; the picture still updates.', reason: out.code || null });
++ const ends = Date.parse(out.expires_at || '') || now() + 60 * 60000;
++ links.set(key, { url: out.url, expires_at: out.expires_at, control: out.control, until: ends - 3 * 60000 });
++ if (links.size > 500) links.delete(links.keys().next().value);
++ return res.json({ url: out.url, expires_at: out.expires_at, control: out.control });
++ });
++
const PROBLEM = {
'actor.allowance.exhausted': 'You have used today\'s free tasks. They come back at midnight UTC.',
'actor.budget.day_spent': 'Today\'s budget is spent. It resets at midnight UTC.',
ddiff --git a/server/tasks/store.js b/server/tasks/store.js
iindex bf7c700..bdce477 100644
---- a/server/tasks/store.js
++++ b/server/tasks/store.js
@@@ -136,6 +136,15 @@ function deskFrom(events) {
return desk;
}
++/**
++ * Where a desk is, from Bot's screen page address (agents/computers.js watchUrl): { machine, desk }, the desk 'main'
++ * when the address names none; null for any other address. What a watch link needs (plan L55 C1b).
++ */
++function deskPlace(url) {
++ const at = /\/machines\/(dev_[A-Za-z0-9_-]{1,64})\/screen(?:\?desk=([a-z0-9-]{1,40}))?$/.exec(String(url || ''));
++ return at ? { machine: at[1], desk: at[2] || 'main' } : null;
++}
++
async function taskDesk(s, row) {
if (!row || !OPEN.includes(row.state)) return null;
const rows = await s.db.many('SELECT body FROM task_events WHERE task_id = $1 ORDER BY seq DESC LIMIT 200', [row.id]);
@@@ -250,4 +259,4 @@ function toWire(row, { baseUrl }) {
return t;
}
--module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, deskFrom, taskDesk, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, openTasks, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS, followContext, followUps, forks, recentSucceeded };
++module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, deskFrom, deskPlace, taskDesk, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, openTasks, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS, followContext, followUps, forks, recentSucceeded };
ddiff --git a/test/chat-live-desk.test.js b/test/chat-live-desk.test.js
nnew file mode 100644
iindex 0000000..042f1b4
---- /dev/null
++++ b/test/chat-live-desk.test.js
@@@ -0,0 +1,89 @@
++'use strict';
++// The live desk in a chat turn (plan L55 C1b; the owner, 11-01: "show you the display… watch it… and steer it"): a
++// turn working on one of the person's own agent desks asks Bot for a short-lived watch link and frames it. The link is
++// a credential: only the task's requester gets it, only same-origin, only while the turn works, never stored.
++// node test/chat-live-desk.test.js
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const { boot, check, done } = require('./helpers/boot');
++const store = require('../server/tasks/store');
++
++const SAME = { 'sec-fetch-site': 'same-origin', accept: 'application/json' };
++const MACHINE = 'dev_01JZ0000000000000000000000';
++
++(async () => {
++ const asked = [];
++ let answer = (desk) => ({ url: 'https://openvibe.bot/screen/tok_live_1', expires_at: new Date(Date.now() + 3600e3).toISOString(), control: true, desk });
++ const computers = {
++ async watchLink(subject, machine, desk, opts) { asked.push({ subject, machine, desk, opts }); return answer(desk); },
++ };
++ const t = await boot({ bot: { computers } });
++ const kim = t.network.addUser('kim');
++ const bo = t.network.addUser('bo');
++ const s = t.ctx.s;
++ try {
++ const id = (await t.get('/api/v1/tasks', { as: kim, json: { task: 'Book the cheapest train to Porto' }, headers: SAME })).json().id;
++ await t.waitFor(id);
++ const deskEvent = (watch, running = true) => store.appendEvent(s, id, { step: 'tool_result', tool: 'computer_desk', is_error: false, text: JSON.stringify({ watch, running, kind: 'browser' }) });
++
++ await check('Bot\'s screen page address names the machine and the desk', async () => {
++ assert.deepStrictEqual(store.deskPlace(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`), { machine: MACHINE, desk: 'desk-trains' });
++ assert.deepStrictEqual(store.deskPlace(`https://openvibe.bot/machines/${MACHINE}/screen`), { machine: MACHINE, desk: 'main' }, 'no desk named is the machine\'s own screen');
++ assert.strictEqual(store.deskPlace('https://example.com/elsewhere'), null);
++ });
++
++ await check('a working turn on an agent desk offers the live desk; the owner\'s screen does not', async () => {
++ await s.db.query("UPDATE tasks SET state = 'running' WHERE id = $1", [id]);
++ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`);
++ let page = await t.get(`/chat/${id}`, { as: kim });
++ assert.match(page.text, new RegExp(`data-watch="/chat/${id}/watch"`), 'the figure says where to ask');
++ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen`);
++ page = await t.get(`/chat/${id}`, { as: kim });
++ assert.ok(!page.text.includes('data-watch='), 'the owner\'s own screen is never framed');
++ await deskEvent(`https://openvibe.bot/machines/${MACHINE}/screen?desk=desk-trains`);
++ });
++
++ await check('only the requester, from openvibe.actor, while it works, gets the link; it is reused, not remade', async () => {
++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: bo, method: 'POST', headers: SAME })).status, 404, 'another person: not there');
++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: { 'sec-fetch-site': 'cross-site', accept: 'application/json' } })).status, 403, 'another site');
++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { method: 'POST', headers: SAME })).status, 401, 'signed out');
++ const r = await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME });
++ assert.strictEqual(r.status, 200, r.text);
++ assert.deepStrictEqual(r.json().url, 'https://openvibe.bot/screen/tok_live_1');
++ assert.match(r.headers.get('cache-control'), /no-store/);
++ assert.deepStrictEqual(asked.map((a) => [a.subject, a.machine, a.desk, a.opts.control, a.opts.minutes]), [[kim.subject, MACHINE, 'desk-trains', true, 60]]);
++ await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME });
++ assert.strictEqual(asked.length, 1, 'the same link again until it nears its end');
++ const events = await s.db.many('SELECT body FROM task_events WHERE task_id = $1', [id]);
++ assert.ok(!JSON.stringify(events).includes('tok_live_1'), 'the link is never written to the task');
++ });
++
++ await check('Bot saying no leaves the picture; a finished turn has no live desk', async () => {
++ const other = (await t.get('/api/v1/tasks', { as: kim, json: { task: 'And back on Sunday' }, headers: SAME })).json().id;
++ await t.waitFor(other);
++ await s.db.query("UPDATE tasks SET state = 'running' WHERE id = $1", [other]);
++ await store.appendEvent(s, other, { step: 'tool_result', tool: 'computer_desk', is_error: false, text: JSON.stringify({ watch: `https://openvibe.bot/machines/${MACHINE}/screen?desk=private`, running: true }) });
++ answer = () => ({ error: 'capability.denied', code: 'capability.denied' });
++ const no = await t.get(`/chat/${other}/watch`, { as: kim, method: 'POST', headers: SAME });
++ assert.strictEqual(no.status, 502);
++ assert.match(no.json().error, /the picture still updates/);
++ await s.db.query("UPDATE tasks SET state = 'succeeded' WHERE id = $1", [id]);
++ assert.strictEqual((await t.get(`/chat/${id}/watch`, { as: kim, method: 'POST', headers: SAME })).status, 409);
++ const page = await t.get(`/chat/${id}`, { as: kim });
++ assert.ok(!page.text.includes('data-watch='), 'a finished turn keeps its last picture');
++ });
++
++ await check('the page frames Bot\'s page (CSP), and the script swaps the picture for it', async () => {
++ const page = await t.get('/chat', { as: kim });
++ assert.match(page.headers.get('content-security-policy'), /frame-src [^;]*https:\/\/openvibe\.bot/);
++ const js = fs.readFileSync(path.join(__dirname, '..', 'public', 'js', 'chat.js'), 'utf8');
++ assert.match(js, /function live\(turn\)/);
++ assert.match(js, /fetch\(fig\.getAttribute\('data-watch'\), \{ method: 'POST', credentials: 'same-origin'/);
++ assert.match(js, /f\.setAttribute\('sandbox', 'allow-scripts allow-same-origin allow-forms allow-pointer-lock'\)/, 'the frame cannot move the page');
++ assert.match(js, /stopScreen\(turn\);\s+\/\/ the frame moves by itself/);
++ });
++ } finally {
++ await done(t);
++ }
++})();