openvibers/openvibe.actor Commits

Watch and take over: a running task's desk is on its record (desk: url, kind) and its page (a card with the link, shown and hidden live as the desk starts and stops) (#137)

OP OpenVibers committed 17 days ago · 374637724f · 6 files, +78 −1

Changes

ddiff --git a/public/css/actor.css b/public/css/actor.css
iindex 6c6cf39..6776ec0 100644
---- a/public/css/actor.css
++++ b/public/css/actor.css
@@@ -124,6 +124,9 @@ button.sc-btn.sc-primary { border: 0; }
.tl-err::before { border-color: var(--warning, #f59e0b); }
.tl-err .tl-k { color: var(--warning-text, var(--warning, #f59e0b)); }
/* Desks plan D4: the agent handed its person a step (a sign-in, a CAPTCHA): the one item that asks for them. */
++.desk-card { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin: 0 0 18px; padding: 12px 14px; border-radius: 12px; background: color-mix(in srgb, var(--accent, #3b82f6) 10%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 40%, transparent); }
++.desk-card[hidden] { display: none; }
++.desk-card p { margin: 0; flex: 1 1 260px; }
.tl-needs { padding: 8px 10px; border-radius: 10px; background: color-mix(in srgb, var(--accent, #3b82f6) 12%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 45%, transparent); }
.tl-needs::before { border-color: var(--accent, #3b82f6); background: var(--accent, #3b82f6); left: -19px; }
.tl-needs .tl-k { color: var(--accent-light, #60a5fa); }
ddiff --git a/public/js/task.js b/public/js/task.js
iindex 38ddc86..2e7277a 100644
---- a/public/js/task.js
++++ b/public/js/task.js
@@@ -60,7 +60,21 @@
li.appendChild(v);
current.appendChild(li);
}
++ // The desk it works on (Desks plan): a computer_desk start shows "Watch and take over", its stop hides it.
++ var deskCard = document.getElementById('desk-card');
++ var deskWatch = new RegExp('^\\{"watch":"(' + bot + '/machines/dev_[0-9A-HJKMNP-TV-Z]{26}/screen(?:\\?desk=[a-z0-9-]{1,40})?)"');
++ function onDesk(e) {
++ if (!deskCard || e.step !== 'tool_result' || e.tool !== 'computer_desk' || e.is_error) return;
++ var t = String(e.text || '');
++ var m = deskWatch.exec(t);
++ if (m && /"running":true/.test(t)) {
++ document.getElementById('desk-link').setAttribute('href', m[1]);
++ document.getElementById('desk-kind').textContent = /"kind":"browser"/.test(t) ? ' (a browser)' : '';
++ deskCard.hidden = false;
++ } else if (/"running":false/.test(t)) deskCard.hidden = true;
++ }
function onOutput(e) {
++ onDesk(e);
if ((e.step === 'plan' || e.step === 'handoff') && e.agent) return attempt((e.text || e.agent).replace(/ takes it.*$/, ''));
if (e.step === 'plan') return undefined;
if (e.step === 'check') {
ddiff --git a/server/http/api.js b/server/http/api.js
iindex 9f533fe..6e2e2df 100644
---- a/server/http/api.js
++++ b/server/http/api.js
@@@ -177,6 +177,9 @@ function createApi(ctx) {
// Waiting for its person on its desk (Desks plan D4): what, and the link to the desk.
const needs = await store.needsYou(s, row);
if (needs) out.needs_you = needs;
++ // The desk it works on now, to watch live and take over (Desks plan).
++ const desk = await store.taskDesk(s, row);
++ if (desk) out.desk = desk;
res.json(out);
});
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex 5806989..bcfd068 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -315,6 +315,7 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
const open = store.OPEN.includes(row.state);
const cls = (events.find((e) => e.step === 'plan' && !e.agent && /"(\w+)" task/.test(e.text || '')) || { text: '' }).text.match(/"(\w+)"/);
const kind = cls ? CLASS_TEXT[cls[1]] || cls[1] : null;
++ const desk = open ? store.deskFrom(events) : null;
page(req, res, {
title: row.task.length > 60 ? `${row.task.slice(0, 59)}…` : row.task,
crumbs: [{ label: 'Your tasks', href: '/tasks' }, { label: 'Task' }],
@@@ -328,6 +329,10 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
<h1 class="task-text">${row.task}</h1>
</header>
${outcome(t, row)}
++${open ? html`<section class="desk-card" id="desk-card" aria-live="polite"${desk ? '' : raw(' hidden')}>
++<p><strong>Your agent is working on a desk of its own</strong><span id="desk-kind">${desk && desk.kind === 'browser' ? ' (a browser)' : ''}</span>. Watch it live, and take it over whenever you like: while you use it, the agent waits; when you stop, it goes on.</p>
++<a class="sc-btn sc-primary" id="desk-link" href="${desk ? desk.url : botPages}" rel="noopener" target="_blank">Watch and take over</a>
++</section>` : ''}
<section class="how" aria-labelledby="h-how"><h2 id="h-how">How it went</h2>
<div class="attempts" id="attempts">${attemptsOf(events).map(attemptBlock)}</div>
${open ? html`<p class="muted small live-note" id="live-note"><span class="dot-pulse" aria-hidden="true"></span> Updating as the agent works.</p>` : ''}
ddiff --git a/server/tasks/store.js b/server/tasks/store.js
iindex 596485d..cbc3ed7 100644
---- a/server/tasks/store.js
++++ b/server/tasks/store.js
@@@ -87,6 +87,30 @@ async function needsYou(s, row) {
return { seq: Number(e.seq), what: text.replace(/\s*Open your agent's desk.*$/s, '').trim().slice(0, 300), url };
}
++/**
++ * The desk a running task works on now (Desks plan: Watch and Take over), from its own computer_desk results: a start
++ * answers {"watch": <the person's page for that desk on OpenVibe.Bot>, "running": true, "kind"?}, a stop
++ * {"running": false}. Tool results come from Actor's own tools, never from the model. → { url, kind } or null.
++ */
++const DESK_WATCH = /^\{"watch":"(https:\/\/[^"\\\s]{1,300})"/;
++function deskFrom(events) {
++ let desk = null;
++ for (const e of events) {
++ if (!e || e.step !== 'tool_result' || e.tool !== 'computer_desk' || e.is_error) continue;
++ const t = String(e.text || '');
++ const m = DESK_WATCH.exec(t);
++ if (m && /"running":true/.test(t)) desk = { url: m[1], kind: (/"kind":"(browser|desktop)"/.exec(t) || [])[1] || null };
++ else if (/"running":false/.test(t)) desk = null;
++ }
++ return desk;
++}
++
++async function taskDesk(s, row) {
++ if (!row || !OPEN.includes(row.state)) return null;
++ const rows = await s.db.many('SELECT body FROM task_events WHERE task_id = $1 ORDER BY seq DESC LIMIT 200', [row.id]);
++ return deskFrom(rows.map((r) => parse(r.body)).reverse());
++}
++
async function eventsAfter(s, taskId, after = 0) {
const rows = await s.db.many('SELECT body FROM task_events WHERE task_id = $1 AND seq > $2 ORDER BY seq', [taskId, after]);
return rows.map((r) => parse(r.body));
@@@ -181,4 +205,4 @@ function toWire(row, { baseUrl }) {
return t;
}
--module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS };
++module.exports = { insertTask, getTask, byIdempotency, listTasks, updateTask, appendEvent, eventsAfter, lastEvent, needsYou, deskFrom, taskDesk, addSpend, spendToday, spendDays, lockSpendToday, failInterrupted, prune, toWire, subjectRef, dayOf, parseJson: parse, OPEN, RETENTION_DAYS };
ddiff --git a/test/desk-handoff.test.js b/test/desk-handoff.test.js
iindex b83d7ea..b612572 100644
---- a/test/desk-handoff.test.js
++++ b/test/desk-handoff.test.js
@@@ -123,6 +123,34 @@ const toolsFor = (bot) => createOpenVibeTools({ config, fetchImpl: bot.fetchImpl
await store.appendEvent(t.ctx.s, id, { kind: 'output', step: 'tool_result', tool: 'hand_to_person', text: '{"done":true}' });
assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: bea })).json().needs_you, undefined, 'the person let go: nothing waits');
});
++ await check('a running task shows the desk it works on, to watch and take over, on its record and its page; not after the desk stops', async () => {
++ const cy = t.network.addUser('desk-cy');
++ const id = t.ctx.s.newId('tsk');
++ await store.insertTask(t.ctx.s, { id, requester: `user:${cy.subject}`, task: 'Find a flight', mode: 'balanced', budget_task: 0.05, budget_day: 1, created_at: t.ctx.s.iso() });
++ const watch = `https://openvibe.bot/machines/${PC}/screen?desk=desk-0123456789`;
++ await store.appendEvent(t.ctx.s, id, { kind: 'output', step: 'plan', agent: 'openvibe-runtime', text: 'OpenVibe runtime takes it.' });
++ const before = await t.get(`/tasks/${id}`, { as: cy });
++ assert.ok(/<section class="desk-card" id="desk-card" aria-live="polite" hidden>/.test(before.text), 'no desk yet: the card waits, hidden');
++ assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, undefined);
++ await store.appendEvent(t.ctx.s, id, { kind: 'output', step: 'tool_call', tool: 'computer_desk', input: `${PC}: ` });
++ await store.appendEvent(t.ctx.s, id, { kind: 'output', step: 'tool_result', tool: 'computer_desk', text: JSON.stringify({ watch, running: true, kind: 'browser', width: 1280, height: 800 }) });
++ assert.deepStrictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, { url: watch, kind: 'browser' });
++ const during = await t.get(`/tasks/${id}`, { as: cy });
++ assert.ok(during.text.includes('<section class="desk-card" id="desk-card" aria-live="polite">'), 'shown');
++ assert.ok(during.text.includes('Watch and take over') && during.text.includes(' (a browser)'));
++ assert.ok(during.text.includes(`id="desk-link" href="${watch}"`) || during.text.includes(`id="desk-link" href="${watch.replace(/&/g, '&amp;')}"`));
++ // A failed start or a model's text never makes a desk; a stop ends it.
++ await store.appendEvent(t.ctx.s, id, { kind: 'output', step: 'tool_result', tool: 'computer_browser', text: JSON.stringify({ watch: 'https://evil.example/x', running: true }) });
++ assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk.url, watch, 'only computer_desk results count');
++ await store.appendEvent(t.ctx.s, id, { kind: 'output', step: 'tool_result', tool: 'computer_desk', text: JSON.stringify({ running: false }) });
++ assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, undefined, 'stopped');
++ assert.deepStrictEqual(store.deskFrom([{ step: 'tool_result', tool: 'computer_desk', is_error: true, text: JSON.stringify({ watch, running: true }) }]), null, 'an error is no desk');
++ // A finished task shows none.
++ await store.appendEvent(t.ctx.s, id, { kind: 'output', step: 'tool_result', tool: 'computer_desk', text: JSON.stringify({ watch, running: true }) });
++ await store.updateTask(t.ctx.s, id, { state: 'succeeded', finished_at: t.ctx.s.iso() });
++ assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, undefined);
++ assert.ok(!(await t.get(`/tasks/${id}`, { as: cy })).text.includes('desk-card'));
++ });
await t.close();
done();
})();