openvibers/openvibe.actor Commits

Glance at the desk: the task page shows what the agent last saw on the screen (its last computer_screen picture), to its owner only, in memory only (#139)

OP OpenVibers committed 17 days ago · e11407f1d6 · 7 files, +94 −6

Changes

ddiff --git a/public/css/actor.css b/public/css/actor.css
iindex 6776ec0..76517c1 100644
---- a/public/css/actor.css
++++ b/public/css/actor.css
@@@ -126,6 +126,10 @@ button.sc-btn.sc-primary { border: 0; }
/* Desks plan D4: the agent handed its person a step (a sign-in, a CAPTCHA): the one item that asks for them. */
.desk-card { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin: 0 0 18px; padding: 12px 14px; border-radius: 12px; background: color-mix(in srgb, var(--accent, #3b82f6) 10%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 40%, transparent); }
.desk-card[hidden] { display: none; }
++.last-look { margin: 0 0 18px; }
++.last-look[hidden] { display: none; }
++.last-look img { display: block; max-width: 100%; height: auto; border-radius: 10px; border: 1px solid var(--line, rgba(127, 127, 127, 0.3)); }
++.last-look figcaption { margin-top: 6px; }
.desk-card p { margin: 0; flex: 1 1 260px; }
.tl-needs { padding: 8px 10px; border-radius: 10px; background: color-mix(in srgb, var(--accent, #3b82f6) 12%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 45%, transparent); }
.tl-needs::before { border-color: var(--accent, #3b82f6); background: var(--accent, #3b82f6); left: -19px; }
ddiff --git a/public/js/task.js b/public/js/task.js
iindex 2e7277a..68dd78e 100644
---- a/public/js/task.js
++++ b/public/js/task.js
@@@ -73,8 +73,19 @@
deskCard.hidden = false;
} else if (/"running":false/.test(t)) deskCard.hidden = true;
}
++ // What the agent last saw: a new look at its screen refreshes the picture.
++ var lastLook = document.getElementById('last-look');
++ function onLook(e) {
++ if (!lastLook || e.step !== 'tool_result' || e.tool !== 'computer_screen' || e.is_error) return;
++ var img = document.getElementById('last-look-img');
++ img.removeAttribute('width');
++ img.removeAttribute('height');
++ img.setAttribute('src', '/tasks/' + encodeURIComponent(id) + '/look?at=' + Date.now());
++ lastLook.hidden = false;
++ }
function onOutput(e) {
onDesk(e);
++ onLook(e);
if ((e.step === 'plan' || e.step === 'handoff') && e.agent) return attempt((e.text || e.agent).replace(/ takes it.*$/, ''));
if (e.step === 'plan') return undefined;
if (e.step === 'check') {
ddiff --git a/server/agents/adapters.js b/server/agents/adapters.js
iindex 81786a3..6a3e2bb 100644
---- a/server/agents/adapters.js
++++ b/server/agents/adapters.js
@@@ -56,7 +56,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
const runtimeModel = own ? chatClient({ baseUrl: own.url, apiKey: own.apiKey, model: own.model, rates: ownRates, fetchImpl, timeoutMs: 180_000 }) : deepseek;
const runtimeModelName = own ? own.model : p.deepseek.model;
-- async function runtime({ task, requester = null, signal, emit, meter, budgetLeft, waitForPerson = null }) {
++ async function runtime({ task, requester = null, taskId = null, signal, emit, meter, budgetLeft, waitForPerson = null }) {
if (!runtimeModel) throw new ProviderError('the runtime has no model configured', { code: 'actor.agent.unavailable' });
const tools = await ovTools.definitions({ signal, requester });
const robots = tools.some((d) => d.function && d.function.name === 'robot_steps');
@@@ -65,7 +65,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
let last = '';
const evidence = []; // what each tool answered, for the check (agents/check.js)
// This run's own desk on any machine that runs several (Desks plan D1), stopped when the run ends, however it ends.
-- const desks = ovTools.deskRun ? ovTools.deskRun() : null;
++ const desks = ovTools.deskRun ? ovTools.deskRun(taskId) : null;
try {
for (let turn = 1; turn <= MAX_TURNS; turn++) {
if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });
ddiff --git a/server/agents/looks.js b/server/agents/looks.js
nnew file mode 100644
iindex 0000000..e863dec
---- /dev/null
++++ b/server/agents/looks.js
@@@ -0,0 +1,31 @@
++'use strict';
++/**
++ * The last picture an agent looked at, per task (Desks plan: glance at any desk). When the agent reads a screen
++ * (computer_screen), the picture it sent to the vision model is kept here, so the person's own task page can show what
++ * the agent last saw. In memory only: at most MAX tasks, each for KEEP_MS, gone when the server restarts. Never stored,
++ * never sent anywhere else.
++ */
++const MAX = 200;
++const KEEP_MS = 60 * 60 * 1000;
++const MAX_BYTES = 2 * 1024 * 1024;
++
++const looks = new Map(); // task id -> { format, data, width, height, at }
++
++function put(taskId, shot, now = Date.now()) {
++ if (!taskId || !shot || typeof shot.content_b64 !== 'string' || !shot.content_b64) return false;
++ const data = Buffer.from(shot.content_b64, 'base64');
++ if (!data.length || data.length > MAX_BYTES) return false;
++ looks.delete(taskId);
++ looks.set(taskId, { format: shot.format === 'png' ? 'png' : 'jpeg', data, width: Number(shot.width) || null, height: Number(shot.height) || null, at: now });
++ while (looks.size > MAX) looks.delete(looks.keys().next().value);
++ return true;
++}
++
++function get(taskId, now = Date.now()) {
++ const l = looks.get(taskId);
++ if (!l) return null;
++ if (now - l.at > KEEP_MS) { looks.delete(taskId); return null; }
++ return l;
++}
++
++module.exports = { put, get, MAX, KEEP_MS };
ddiff --git a/server/agents/tools.js b/server/agents/tools.js
iindex 02c0c59..8a9c937 100644
---- a/server/agents/tools.js
++++ b/server/agents/tools.js
@@@ -1,6 +1,7 @@
'use strict';
const crypto = require('crypto');
++const looks = require('./looks');
/**
* What the OpenVibe runtime can do, as model tools — every one of them an OpenVibe service's public API, called the
@@@ -143,11 +144,13 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
}
/** One look at a computer's screen: the picture to the vision model with the question, its answer back. */
-- async function lookAtScreen(person, a, { signal, meter, budgetLeft }) {
++ async function lookAtScreen(person, a, { signal, meter, budgetLeft, desks = null }) {
if (!vision) return { error: 'no vision model is configured on this server, so the screen cannot be read' };
if (budgetLeft && budgetLeft() <= 0) return { error: 'the task reached its budget' };
const shot = await pcs.screen(person, a.machine, { signal, desk: a.desk && a.desk !== 'main' ? a.desk : 'main' });
if (shot.error) return shot;
++ // What the agent last saw, for the person's own task page (in memory only).
++ if (desks && desks.taskId) looks.put(desks.taskId, shot);
const question = String(a.question || '').trim().slice(0, 500) || 'Describe what is on the screen: the windows, what each shows, and any message or error in full.';
const r = await vision({
messages: [
@@@ -166,8 +169,8 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
* other's screen, browser or files; elsewhere it is the machine's one private desk, as before. close() stops every
* desk the run used. A model's own desk name is refused, so a run can never reach another task's desk.
*/
-- function deskRun() {
-- return { name: `desk-${crypto.randomBytes(5).toString('hex')}`, named: null, kinds: new Map(), keeps: new Set(), used: new Set() };
++ function deskRun(taskId = null) {
++ return { name: `desk-${crypto.randomBytes(5).toString('hex')}`, taskId, named: null, kinds: new Map(), keeps: new Set(), used: new Set() };
}
async function deskFor(run, person, machine, { signal } = {}) {
@@@ -267,7 +270,7 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
if (name === 'computer_list') return json(await pcs.list(person, a.machine, a.path, { signal }));
if (name === 'computer_read') return json(await pcs.read(person, a.machine, a.path, { offset: a.offset, signal }));
if (name === 'computer_run') return json(await pcs.run(person, a.machine, a.argv, { cwd: a.cwd, timeoutS: a.timeout_s, signal }));
-- if (name === 'computer_screen') return json(await lookAtScreen(person, a, { signal, meter, budgetLeft }));
++ if (name === 'computer_screen') return json(await lookAtScreen(person, a, { signal, meter, budgetLeft, desks }));
if (name === 'computer_input') return json(await pcs.input(person, a.machine, a, { signal }));
if (name === 'computer_browser') return json(await pcs.browser(person, a.machine, a, { signal }));
if (name === 'computer_desk') {
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex bcfd068..cf85a9b 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -22,6 +22,7 @@ const { markdown } = require('../render/markdown');
const store = require('../tasks/store');
const { createTask } = require('../tasks/create');
const catalog = require('../agents/catalog');
++const looks = require('../agents/looks');
const { MODES } = require('../agents/router');
const seo = require('openvibe-shared/seo');
@@@ -304,6 +305,15 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
const STATE_TEXT = { queued: 'Waiting to start', running: 'Working', verifying: 'Checking the answer', succeeded: 'Done', failed: 'Didn\'t finish', cancelled: 'Cancelled' };
const CLASS_TEXT = { answer: 'Answer', lookup: 'Lookup', research: 'Research', web: 'Web search', code: 'Code', robot: 'Your robot', computer: 'Your computer' };
++ // What the agent last saw (agents/looks.js): its task's owner only, never cached.
++ r.get('/tasks/:id/look', async (req, res) => {
++ const row = TASK_ID.test(req.params.id) && signedIn(req) ? await store.getTask(s, req.params.id) : null;
++ const look = row && row.requester === requesterOf(req) ? looks.get(row.id) : null;
++ if (!look) return res.status(404).type('text').send('No picture');
++ res.setHeader('Cache-Control', 'private, no-store');
++ res.type(look.format === 'png' ? 'image/png' : 'image/jpeg').send(look.data);
++ });
++
r.get('/tasks/:id', async (req, res) => {
const row = TASK_ID.test(req.params.id) && signedIn(req) ? await store.getTask(s, req.params.id) : null;
if (!row || row.requester !== requesterOf(req)) {
@@@ -316,6 +326,7 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
const cls = (events.find((e) => e.step === 'plan' && !e.agent && /"(\w+)" task/.test(e.text || '')) || { text: '' }).text.match(/"(\w+)"/);
const kind = cls ? CLASS_TEXT[cls[1]] || cls[1] : null;
const desk = open ? store.deskFrom(events) : null;
++ const look = looks.get(row.id);
page(req, res, {
title: row.task.length > 60 ? `${row.task.slice(0, 59)}…` : row.task,
crumbs: [{ label: 'Your tasks', href: '/tasks' }, { label: 'Task' }],
@@@ -333,6 +344,10 @@ ${open ? html`<section class="desk-card" id="desk-card" aria-live="polite"${desk
<p><strong>Your agent is working on a desk of its own</strong><span id="desk-kind">${desk && desk.kind === 'browser' ? ' (a browser)' : ''}</span>. Watch it live, and take it over whenever you like: while you use it, the agent waits; when you stop, it goes on.</p>
<a class="sc-btn sc-primary" id="desk-link" href="${desk ? desk.url : botPages}" rel="noopener" target="_blank">Watch and take over</a>
</section>` : ''}
++${open || look ? html`<figure class="last-look" id="last-look"${look ? '' : raw(' hidden')}>
++<img id="last-look-img" src="${look ? `/tasks/${row.id}/look?at=${look.at}` : ''}" alt="The screen your agent last looked at"${look && look.width ? raw(` width="${Number(look.width)}" height="${Number(look.height)}"`) : ''} loading="lazy">
++<figcaption class="muted small">What your agent last saw on the screen${look ? html` · ${time(new Date(look.at).toISOString())}` : ''}</figcaption>
++</figure>` : ''}
<section class="how" aria-labelledby="h-how"><h2 id="h-how">How it went</h2>
<div class="attempts" id="attempts">${attemptsOf(events).map(attemptBlock)}</div>
${open ? html`<p class="muted small live-note" id="live-note"><span class="dot-pulse" aria-hidden="true"></span> Updating as the agent works.</p>` : ''}
ddiff --git a/test/desk-handoff.test.js b/test/desk-handoff.test.js
iindex b612572..5491860 100644
---- a/test/desk-handoff.test.js
++++ b/test/desk-handoff.test.js
@@@ -151,6 +151,30 @@ const toolsFor = (bot) => createOpenVibeTools({ config, fetchImpl: bot.fetchImpl
assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, undefined);
assert.ok(!(await t.get(`/tasks/${id}`, { as: cy })).text.includes('desk-card'));
});
++ await check('what the agent last saw: its last screen picture on the owner\'s task page, nowhere else, in memory only', async () => {
++ const looks = require('../server/agents/looks');
++ const dee = t.network.addUser('desk-dee');
++ const eve = t.network.addUser('desk-eve');
++ const id = t.ctx.s.newId('tsk');
++ await store.insertTask(t.ctx.s, { id, requester: `user:${dee.subject}`, task: 'Check the form', mode: 'balanced', budget_task: 0.05, budget_day: 1, created_at: t.ctx.s.iso() });
++ assert.strictEqual((await t.get(`/tasks/${id}/look`, { as: dee })).status, 404, 'no look yet');
++ const before = await t.get(`/tasks/${id}`, { as: dee });
++ assert.ok(/<figure class="last-look" id="last-look" hidden>/.test(before.text), 'the picture waits, hidden, while the task runs');
++ const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3]);
++ assert.ok(looks.put(id, { format: 'jpeg', width: 1280, height: 800, content_b64: jpeg.toString('base64') }));
++ const img = await t.get(`/tasks/${id}/look`, { as: dee });
++ assert.strictEqual(img.status, 200);
++ assert.match(img.headers['content-type'] || (img.headers.get && img.headers.get('content-type')) || '', /^image\/jpeg/);
++ assert.match(img.headers['cache-control'] || (img.headers.get && img.headers.get('cache-control')) || '', /no-store/);
++ assert.strictEqual((await t.get(`/tasks/${id}/look`, { as: eve })).status, 404, 'someone else never sees it');
++ assert.strictEqual((await t.get(`/tasks/${id}/look`)).status, 404, 'nor someone signed out');
++ const page = await t.get(`/tasks/${id}`, { as: dee });
++ assert.ok(page.text.includes('<figure class="last-look" id="last-look">') && page.text.includes('width="1280" height="800"'));
++ assert.ok(page.text.includes('What your agent last saw on the screen'));
++ // Kept in memory for a while, then gone; never more than MAX tasks.
++ assert.strictEqual(looks.get(id, Date.now() + looks.KEEP_MS + 1), null);
++ assert.strictEqual(looks.put('tsk_x', { content_b64: '' }), false, 'an empty picture is not kept');
++ });
await t.close();
done();
})();