openvibers/openvibe.actor Commits
Glance at the desk: the task page shows what the agent last saw on the screen (its last computer_screen picture), to its owner only, in memory only (#139)
Changes
@@@ -126,6 +126,10 @@ button.sc-btn.sc-primary { border: 0; }/* Desks plan D4: the agent handed its person a step (a sign-in, a CAPTCHA): the one item that asks for them. */.desk-card { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin: 0 0 18px; padding: 12px 14px; border-radius: 12px; background: color-mix(in srgb, var(--accent, #3b82f6) 10%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 40%, transparent); }.desk-card[hidden] { display: none; }++.last-look { margin: 0 0 18px; }++.last-look[hidden] { display: none; }++.last-look img { display: block; max-width: 100%; height: auto; border-radius: 10px; border: 1px solid var(--line, rgba(127, 127, 127, 0.3)); }++.last-look figcaption { margin-top: 6px; }.desk-card p { margin: 0; flex: 1 1 260px; }.tl-needs { padding: 8px 10px; border-radius: 10px; background: color-mix(in srgb, var(--accent, #3b82f6) 12%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 45%, transparent); }.tl-needs::before { border-color: var(--accent, #3b82f6); background: var(--accent, #3b82f6); left: -19px; }@@@ -73,8 +73,19 @@deskCard.hidden = false;} else if (/"running":false/.test(t)) deskCard.hidden = true;}++ // What the agent last saw: a new look at its screen refreshes the picture.++ var lastLook = document.getElementById('last-look');++ function onLook(e) {++ if (!lastLook || e.step !== 'tool_result' || e.tool !== 'computer_screen' || e.is_error) return;++ var img = document.getElementById('last-look-img');++ img.removeAttribute('width');++ img.removeAttribute('height');++ img.setAttribute('src', '/tasks/' + encodeURIComponent(id) + '/look?at=' + Date.now());++ lastLook.hidden = false;++ }function onOutput(e) {onDesk(e);++ onLook(e);if ((e.step === 'plan' || e.step === 'handoff') && e.agent) return attempt((e.text || e.agent).replace(/ takes it.*$/, ''));if (e.step === 'plan') return undefined;if (e.step === 'check') {@@@ -56,7 +56,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = conconst runtimeModel = own ? chatClient({ baseUrl: own.url, apiKey: own.apiKey, model: own.model, rates: ownRates, fetchImpl, timeoutMs: 180_000 }) : deepseek;const runtimeModelName = own ? own.model : p.deepseek.model;-- async function runtime({ task, requester = null, signal, emit, meter, budgetLeft, waitForPerson = null }) {++ async function runtime({ task, requester = null, taskId = null, signal, emit, meter, budgetLeft, waitForPerson = null }) {if (!runtimeModel) throw new ProviderError('the runtime has no model configured', { code: 'actor.agent.unavailable' });const tools = await ovTools.definitions({ signal, requester });const robots = tools.some((d) => d.function && d.function.name === 'robot_steps');@@@ -65,7 +65,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = conlet last = '';const evidence = []; // what each tool answered, for the check (agents/check.js)// This run's own desk on any machine that runs several (Desks plan D1), stopped when the run ends, however it ends.-- const desks = ovTools.deskRun ? ovTools.deskRun() : null;++ const desks = ovTools.deskRun ? ovTools.deskRun(taskId) : null;try {for (let turn = 1; turn <= MAX_TURNS; turn++) {if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });nnew file mode 100644@@@ -0,0 +1,31 @@++'use strict';++/**++ * The last picture an agent looked at, per task (Desks plan: glance at any desk). When the agent reads a screen++ * (computer_screen), the picture it sent to the vision model is kept here, so the person's own task page can show what++ * the agent last saw. In memory only: at most MAX tasks, each for KEEP_MS, gone when the server restarts. Never stored,++ * never sent anywhere else.++ */++const MAX = 200;++const KEEP_MS = 60 * 60 * 1000;++const MAX_BYTES = 2 * 1024 * 1024;++++const looks = new Map(); // task id -> { format, data, width, height, at }++++function put(taskId, shot, now = Date.now()) {++ if (!taskId || !shot || typeof shot.content_b64 !== 'string' || !shot.content_b64) return false;++ const data = Buffer.from(shot.content_b64, 'base64');++ if (!data.length || data.length > MAX_BYTES) return false;++ looks.delete(taskId);++ looks.set(taskId, { format: shot.format === 'png' ? 'png' : 'jpeg', data, width: Number(shot.width) || null, height: Number(shot.height) || null, at: now });++ while (looks.size > MAX) looks.delete(looks.keys().next().value);++ return true;++}++++function get(taskId, now = Date.now()) {++ const l = looks.get(taskId);++ if (!l) return null;++ if (now - l.at > KEEP_MS) { looks.delete(taskId); return null; }++ return l;++}++++module.exports = { put, get, MAX, KEEP_MS };@@@ -1,6 +1,7 @@'use strict';const crypto = require('crypto');++const looks = require('./looks');/*** What the OpenVibe runtime can do, as model tools — every one of them an OpenVibe service's public API, called the@@@ -143,11 +144,13 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>}/** One look at a computer's screen: the picture to the vision model with the question, its answer back. */-- async function lookAtScreen(person, a, { signal, meter, budgetLeft }) {++ async function lookAtScreen(person, a, { signal, meter, budgetLeft, desks = null }) {if (!vision) return { error: 'no vision model is configured on this server, so the screen cannot be read' };if (budgetLeft && budgetLeft() <= 0) return { error: 'the task reached its budget' };const shot = await pcs.screen(person, a.machine, { signal, desk: a.desk && a.desk !== 'main' ? a.desk : 'main' });if (shot.error) return shot;++ // What the agent last saw, for the person's own task page (in memory only).++ if (desks && desks.taskId) looks.put(desks.taskId, shot);const question = String(a.question || '').trim().slice(0, 500) || 'Describe what is on the screen: the windows, what each shows, and any message or error in full.';const r = await vision({messages: [@@@ -166,8 +169,8 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>* other's screen, browser or files; elsewhere it is the machine's one private desk, as before. close() stops every* desk the run used. A model's own desk name is refused, so a run can never reach another task's desk.*/-- function deskRun() {-- return { name: `desk-${crypto.randomBytes(5).toString('hex')}`, named: null, kinds: new Map(), keeps: new Set(), used: new Set() };++ function deskRun(taskId = null) {++ return { name: `desk-${crypto.randomBytes(5).toString('hex')}`, taskId, named: null, kinds: new Map(), keeps: new Set(), used: new Set() };}async function deskFor(run, person, machine, { signal } = {}) {@@@ -267,7 +270,7 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>if (name === 'computer_list') return json(await pcs.list(person, a.machine, a.path, { signal }));if (name === 'computer_read') return json(await pcs.read(person, a.machine, a.path, { offset: a.offset, signal }));if (name === 'computer_run') return json(await pcs.run(person, a.machine, a.argv, { cwd: a.cwd, timeoutS: a.timeout_s, signal }));-- if (name === 'computer_screen') return json(await lookAtScreen(person, a, { signal, meter, budgetLeft }));++ if (name === 'computer_screen') return json(await lookAtScreen(person, a, { signal, meter, budgetLeft, desks }));if (name === 'computer_input') return json(await pcs.input(person, a.machine, a, { signal }));if (name === 'computer_browser') return json(await pcs.browser(person, a.machine, a, { signal }));if (name === 'computer_desk') {@@@ -22,6 +22,7 @@ const { markdown } = require('../render/markdown');const store = require('../tasks/store');const { createTask } = require('../tasks/create');const catalog = require('../agents/catalog');++const looks = require('../agents/looks');const { MODES } = require('../agents/router');const seo = require('openvibe-shared/seo');@@@ -304,6 +305,15 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</const STATE_TEXT = { queued: 'Waiting to start', running: 'Working', verifying: 'Checking the answer', succeeded: 'Done', failed: 'Didn\'t finish', cancelled: 'Cancelled' };const CLASS_TEXT = { answer: 'Answer', lookup: 'Lookup', research: 'Research', web: 'Web search', code: 'Code', robot: 'Your robot', computer: 'Your computer' };++ // What the agent last saw (agents/looks.js): its task's owner only, never cached.++ r.get('/tasks/:id/look', async (req, res) => {++ const row = TASK_ID.test(req.params.id) && signedIn(req) ? await store.getTask(s, req.params.id) : null;++ const look = row && row.requester === requesterOf(req) ? looks.get(row.id) : null;++ if (!look) return res.status(404).type('text').send('No picture');++ res.setHeader('Cache-Control', 'private, no-store');++ res.type(look.format === 'png' ? 'image/png' : 'image/jpeg').send(look.data);++ });++r.get('/tasks/:id', async (req, res) => {const row = TASK_ID.test(req.params.id) && signedIn(req) ? await store.getTask(s, req.params.id) : null;if (!row || row.requester !== requesterOf(req)) {@@@ -316,6 +326,7 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</const cls = (events.find((e) => e.step === 'plan' && !e.agent && /"(\w+)" task/.test(e.text || '')) || { text: '' }).text.match(/"(\w+)"/);const kind = cls ? CLASS_TEXT[cls[1]] || cls[1] : null;const desk = open ? store.deskFrom(events) : null;++ const look = looks.get(row.id);page(req, res, {title: row.task.length > 60 ? `${row.task.slice(0, 59)}…` : row.task,crumbs: [{ label: 'Your tasks', href: '/tasks' }, { label: 'Task' }],@@@ -333,6 +344,10 @@ ${open ? html`<section class="desk-card" id="desk-card" aria-live="polite"${desk<p><strong>Your agent is working on a desk of its own</strong><span id="desk-kind">${desk && desk.kind === 'browser' ? ' (a browser)' : ''}</span>. Watch it live, and take it over whenever you like: while you use it, the agent waits; when you stop, it goes on.</p><a class="sc-btn sc-primary" id="desk-link" href="${desk ? desk.url : botPages}" rel="noopener" target="_blank">Watch and take over</a></section>` : ''}++${open || look ? html`<figure class="last-look" id="last-look"${look ? '' : raw(' hidden')}>++<img id="last-look-img" src="${look ? `/tasks/${row.id}/look?at=${look.at}` : ''}" alt="The screen your agent last looked at"${look && look.width ? raw(` width="${Number(look.width)}" height="${Number(look.height)}"`) : ''} loading="lazy">++<figcaption class="muted small">What your agent last saw on the screen${look ? html` · ${time(new Date(look.at).toISOString())}` : ''}</figcaption>++</figure>` : ''}<section class="how" aria-labelledby="h-how"><h2 id="h-how">How it went</h2><div class="attempts" id="attempts">${attemptsOf(events).map(attemptBlock)}</div>${open ? html`<p class="muted small live-note" id="live-note"><span class="dot-pulse" aria-hidden="true"></span> Updating as the agent works.</p>` : ''}@@@ -151,6 +151,30 @@ const toolsFor = (bot) => createOpenVibeTools({ config, fetchImpl: bot.fetchImplassert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, undefined);assert.ok(!(await t.get(`/tasks/${id}`, { as: cy })).text.includes('desk-card'));});++ await check('what the agent last saw: its last screen picture on the owner\'s task page, nowhere else, in memory only', async () => {++ const looks = require('../server/agents/looks');++ const dee = t.network.addUser('desk-dee');++ const eve = t.network.addUser('desk-eve');++ const id = t.ctx.s.newId('tsk');++ await store.insertTask(t.ctx.s, { id, requester: `user:${dee.subject}`, task: 'Check the form', mode: 'balanced', budget_task: 0.05, budget_day: 1, created_at: t.ctx.s.iso() });++ assert.strictEqual((await t.get(`/tasks/${id}/look`, { as: dee })).status, 404, 'no look yet');++ const before = await t.get(`/tasks/${id}`, { as: dee });++ assert.ok(/<figure class="last-look" id="last-look" hidden>/.test(before.text), 'the picture waits, hidden, while the task runs');++ const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3]);++ assert.ok(looks.put(id, { format: 'jpeg', width: 1280, height: 800, content_b64: jpeg.toString('base64') }));++ const img = await t.get(`/tasks/${id}/look`, { as: dee });++ assert.strictEqual(img.status, 200);++ assert.match(img.headers['content-type'] || (img.headers.get && img.headers.get('content-type')) || '', /^image\/jpeg/);++ assert.match(img.headers['cache-control'] || (img.headers.get && img.headers.get('cache-control')) || '', /no-store/);++ assert.strictEqual((await t.get(`/tasks/${id}/look`, { as: eve })).status, 404, 'someone else never sees it');++ assert.strictEqual((await t.get(`/tasks/${id}/look`)).status, 404, 'nor someone signed out');++ const page = await t.get(`/tasks/${id}`, { as: dee });++ assert.ok(page.text.includes('<figure class="last-look" id="last-look">') && page.text.includes('width="1280" height="800"'));++ assert.ok(page.text.includes('What your agent last saw on the screen'));++ // Kept in memory for a while, then gone; never more than MAX tasks.++ assert.strictEqual(looks.get(id, Date.now() + looks.KEEP_MS + 1), null);++ assert.strictEqual(looks.put('tsk_x', { content_b64: '' }), false, 'an empty picture is not kept');++ });await t.close();done();})();
@@ -126,6 +126,10 @@ button.sc-btn.sc-primary { border: 0; }
@@ -126,6 +126,10 @@ button.sc-btn.sc-primary { border: 0; }
/* Desks plan D4: the agent handed its person a step (a sign-in, a CAPTCHA): the one item that asks for them. */
/* Desks plan D4: the agent handed its person a step (a sign-in, a CAPTCHA): the one item that asks for them. */
.desk-card { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin: 0 0 18px; padding: 12px 14px; border-radius: 12px; background: color-mix(in srgb, var(--accent, #3b82f6) 10%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 40%, transparent); }
.desk-card { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin: 0 0 18px; padding: 12px 14px; border-radius: 12px; background: color-mix(in srgb, var(--accent, #3b82f6) 10%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 40%, transparent); }
.desk-card[hidden] { display: none; }
.desk-card[hidden] { display: none; }
+.last-look { margin: 0 0 18px; }
+.last-look[hidden] { display: none; }
+.last-look img { display: block; max-width: 100%; height: auto; border-radius: 10px; border: 1px solid var(--line, rgba(127, 127, 127, 0.3)); }
+.last-look figcaption { margin-top: 6px; }
.desk-card p { margin: 0; flex: 1 1 260px; }
.desk-card p { margin: 0; flex: 1 1 260px; }
.tl-needs { padding: 8px 10px; border-radius: 10px; background: color-mix(in srgb, var(--accent, #3b82f6) 12%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 45%, transparent); }
.tl-needs { padding: 8px 10px; border-radius: 10px; background: color-mix(in srgb, var(--accent, #3b82f6) 12%, transparent); border: 1px solid color-mix(in srgb, var(--accent, #3b82f6) 45%, transparent); }
.tl-needs::before { border-color: var(--accent, #3b82f6); background: var(--accent, #3b82f6); left: -19px; }
.tl-needs::before { border-color: var(--accent, #3b82f6); background: var(--accent, #3b82f6); left: -19px; }
@@ -73,8 +73,19 @@
@@ -73,8 +73,19 @@
deskCard.hidden = false;
deskCard.hidden = false;
} else if (/"running":false/.test(t)) deskCard.hidden = true;
} else if (/"running":false/.test(t)) deskCard.hidden = true;
}
}
+ // What the agent last saw: a new look at its screen refreshes the picture.
+ var lastLook = document.getElementById('last-look');
+ function onLook(e) {
+ if (!lastLook || e.step !== 'tool_result' || e.tool !== 'computer_screen' || e.is_error) return;
+ var img = document.getElementById('last-look-img');
+ img.removeAttribute('width');
+ img.removeAttribute('height');
+ img.setAttribute('src', '/tasks/' + encodeURIComponent(id) + '/look?at=' + Date.now());
+ lastLook.hidden = false;
+ }
function onOutput(e) {
function onOutput(e) {
onDesk(e);
onDesk(e);
+ onLook(e);
if ((e.step === 'plan' || e.step === 'handoff') && e.agent) return attempt((e.text || e.agent).replace(/ takes it.*$/, ''));
if ((e.step === 'plan' || e.step === 'handoff') && e.agent) return attempt((e.text || e.agent).replace(/ takes it.*$/, ''));
if (e.step === 'plan') return undefined;
if (e.step === 'plan') return undefined;
if (e.step === 'check') {
if (e.step === 'check') {
@@ -56,7 +56,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
@@ -56,7 +56,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
const runtimeModel = own ? chatClient({ baseUrl: own.url, apiKey: own.apiKey, model: own.model, rates: ownRates, fetchImpl, timeoutMs: 180_000 }) : deepseek;
const runtimeModel = own ? chatClient({ baseUrl: own.url, apiKey: own.apiKey, model: own.model, rates: ownRates, fetchImpl, timeoutMs: 180_000 }) : deepseek;
const runtimeModelName = own ? own.model : p.deepseek.model;
const runtimeModelName = own ? own.model : p.deepseek.model;
- async function runtime({ task, requester = null, signal, emit, meter, budgetLeft, waitForPerson = null }) {
+ async function runtime({ task, requester = null, taskId = null, signal, emit, meter, budgetLeft, waitForPerson = null }) {
if (!runtimeModel) throw new ProviderError('the runtime has no model configured', { code: 'actor.agent.unavailable' });
if (!runtimeModel) throw new ProviderError('the runtime has no model configured', { code: 'actor.agent.unavailable' });
const tools = await ovTools.definitions({ signal, requester });
const tools = await ovTools.definitions({ signal, requester });
const robots = tools.some((d) => d.function && d.function.name === 'robot_steps');
const robots = tools.some((d) => d.function && d.function.name === 'robot_steps');
@@ -65,7 +65,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
@@ -65,7 +65,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
let last = '';
let last = '';
const evidence = []; // what each tool answered, for the check (agents/check.js)
const evidence = []; // what each tool answered, for the check (agents/check.js)
// This run's own desk on any machine that runs several (Desks plan D1), stopped when the run ends, however it ends.
// This run's own desk on any machine that runs several (Desks plan D1), stopped when the run ends, however it ends.
- const desks = ovTools.deskRun ? ovTools.deskRun() : null;
+ const desks = ovTools.deskRun ? ovTools.deskRun(taskId) : null;
try {
try {
for (let turn = 1; turn <= MAX_TURNS; turn++) {
for (let turn = 1; turn <= MAX_TURNS; turn++) {
if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });
if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });
new file mode 100644
new file mode 100644
@@ -0,0 +1,31 @@
@@ -0,0 +1,31 @@
+'use strict';
+/**
+ * The last picture an agent looked at, per task (Desks plan: glance at any desk). When the agent reads a screen
+ * (computer_screen), the picture it sent to the vision model is kept here, so the person's own task page can show what
+ * the agent last saw. In memory only: at most MAX tasks, each for KEEP_MS, gone when the server restarts. Never stored,
+ * never sent anywhere else.
+ */
+const MAX = 200;
+const KEEP_MS = 60 * 60 * 1000;
+const MAX_BYTES = 2 * 1024 * 1024;
+
+const looks = new Map(); // task id -> { format, data, width, height, at }
+
+function put(taskId, shot, now = Date.now()) {
+ if (!taskId || !shot || typeof shot.content_b64 !== 'string' || !shot.content_b64) return false;
+ const data = Buffer.from(shot.content_b64, 'base64');
+ if (!data.length || data.length > MAX_BYTES) return false;
+ looks.delete(taskId);
+ looks.set(taskId, { format: shot.format === 'png' ? 'png' : 'jpeg', data, width: Number(shot.width) || null, height: Number(shot.height) || null, at: now });
+ while (looks.size > MAX) looks.delete(looks.keys().next().value);
+ return true;
+}
+
+function get(taskId, now = Date.now()) {
+ const l = looks.get(taskId);
+ if (!l) return null;
+ if (now - l.at > KEEP_MS) { looks.delete(taskId); return null; }
+ return l;
+}
+
+module.exports = { put, get, MAX, KEEP_MS };
@@ -1,6 +1,7 @@
@@ -1,6 +1,7 @@
'use strict';
'use strict';
const crypto = require('crypto');
const crypto = require('crypto');
+const looks = require('./looks');
/**
/**
* What the OpenVibe runtime can do, as model tools — every one of them an OpenVibe service's public API, called the
* What the OpenVibe runtime can do, as model tools — every one of them an OpenVibe service's public API, called the
@@ -143,11 +144,13 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -143,11 +144,13 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
}
}
/** One look at a computer's screen: the picture to the vision model with the question, its answer back. */
/** One look at a computer's screen: the picture to the vision model with the question, its answer back. */
- async function lookAtScreen(person, a, { signal, meter, budgetLeft }) {
+ async function lookAtScreen(person, a, { signal, meter, budgetLeft, desks = null }) {
if (!vision) return { error: 'no vision model is configured on this server, so the screen cannot be read' };
if (!vision) return { error: 'no vision model is configured on this server, so the screen cannot be read' };
if (budgetLeft && budgetLeft() <= 0) return { error: 'the task reached its budget' };
if (budgetLeft && budgetLeft() <= 0) return { error: 'the task reached its budget' };
const shot = await pcs.screen(person, a.machine, { signal, desk: a.desk && a.desk !== 'main' ? a.desk : 'main' });
const shot = await pcs.screen(person, a.machine, { signal, desk: a.desk && a.desk !== 'main' ? a.desk : 'main' });
if (shot.error) return shot;
if (shot.error) return shot;
+ // What the agent last saw, for the person's own task page (in memory only).
+ if (desks && desks.taskId) looks.put(desks.taskId, shot);
const question = String(a.question || '').trim().slice(0, 500) || 'Describe what is on the screen: the windows, what each shows, and any message or error in full.';
const question = String(a.question || '').trim().slice(0, 500) || 'Describe what is on the screen: the windows, what each shows, and any message or error in full.';
const r = await vision({
const r = await vision({
messages: [
messages: [
@@ -166,8 +169,8 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -166,8 +169,8 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
* other's screen, browser or files; elsewhere it is the machine's one private desk, as before. close() stops every
* other's screen, browser or files; elsewhere it is the machine's one private desk, as before. close() stops every
* desk the run used. A model's own desk name is refused, so a run can never reach another task's desk.
* desk the run used. A model's own desk name is refused, so a run can never reach another task's desk.
*/
*/
- function deskRun() {
- return { name: `desk-${crypto.randomBytes(5).toString('hex')}`, named: null, kinds: new Map(), keeps: new Set(), used: new Set() };
+ function deskRun(taskId = null) {
+ return { name: `desk-${crypto.randomBytes(5).toString('hex')}`, taskId, named: null, kinds: new Map(), keeps: new Set(), used: new Set() };
}
}
async function deskFor(run, person, machine, { signal } = {}) {
async function deskFor(run, person, machine, { signal } = {}) {
@@ -267,7 +270,7 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -267,7 +270,7 @@ function createOpenVibeTools({ config, fetchImpl = globalThis.fetch, now = () =>
if (name === 'computer_list') return json(await pcs.list(person, a.machine, a.path, { signal }));
if (name === 'computer_list') return json(await pcs.list(person, a.machine, a.path, { signal }));
if (name === 'computer_read') return json(await pcs.read(person, a.machine, a.path, { offset: a.offset, signal }));
if (name === 'computer_read') return json(await pcs.read(person, a.machine, a.path, { offset: a.offset, signal }));
if (name === 'computer_run') return json(await pcs.run(person, a.machine, a.argv, { cwd: a.cwd, timeoutS: a.timeout_s, signal }));
if (name === 'computer_run') return json(await pcs.run(person, a.machine, a.argv, { cwd: a.cwd, timeoutS: a.timeout_s, signal }));
- if (name === 'computer_screen') return json(await lookAtScreen(person, a, { signal, meter, budgetLeft }));
+ if (name === 'computer_screen') return json(await lookAtScreen(person, a, { signal, meter, budgetLeft, desks }));
if (name === 'computer_input') return json(await pcs.input(person, a.machine, a, { signal }));
if (name === 'computer_input') return json(await pcs.input(person, a.machine, a, { signal }));
if (name === 'computer_browser') return json(await pcs.browser(person, a.machine, a, { signal }));
if (name === 'computer_browser') return json(await pcs.browser(person, a.machine, a, { signal }));
if (name === 'computer_desk') {
if (name === 'computer_desk') {
@@ -22,6 +22,7 @@ const { markdown } = require('../render/markdown');
@@ -22,6 +22,7 @@ const { markdown } = require('../render/markdown');
const store = require('../tasks/store');
const store = require('../tasks/store');
const { createTask } = require('../tasks/create');
const { createTask } = require('../tasks/create');
const catalog = require('../agents/catalog');
const catalog = require('../agents/catalog');
+const looks = require('../agents/looks');
const { MODES } = require('../agents/router');
const { MODES } = require('../agents/router');
const seo = require('openvibe-shared/seo');
const seo = require('openvibe-shared/seo');
@@ -304,6 +305,15 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
@@ -304,6 +305,15 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
const STATE_TEXT = { queued: 'Waiting to start', running: 'Working', verifying: 'Checking the answer', succeeded: 'Done', failed: 'Didn\'t finish', cancelled: 'Cancelled' };
const STATE_TEXT = { queued: 'Waiting to start', running: 'Working', verifying: 'Checking the answer', succeeded: 'Done', failed: 'Didn\'t finish', cancelled: 'Cancelled' };
const CLASS_TEXT = { answer: 'Answer', lookup: 'Lookup', research: 'Research', web: 'Web search', code: 'Code', robot: 'Your robot', computer: 'Your computer' };
const CLASS_TEXT = { answer: 'Answer', lookup: 'Lookup', research: 'Research', web: 'Web search', code: 'Code', robot: 'Your robot', computer: 'Your computer' };
+ // What the agent last saw (agents/looks.js): its task's owner only, never cached.
+ r.get('/tasks/:id/look', async (req, res) => {
+ const row = TASK_ID.test(req.params.id) && signedIn(req) ? await store.getTask(s, req.params.id) : null;
+ const look = row && row.requester === requesterOf(req) ? looks.get(row.id) : null;
+ if (!look) return res.status(404).type('text').send('No picture');
+ res.setHeader('Cache-Control', 'private, no-store');
+ res.type(look.format === 'png' ? 'image/png' : 'image/jpeg').send(look.data);
+ });
+
r.get('/tasks/:id', async (req, res) => {
r.get('/tasks/:id', async (req, res) => {
const row = TASK_ID.test(req.params.id) && signedIn(req) ? await store.getTask(s, req.params.id) : null;
const row = TASK_ID.test(req.params.id) && signedIn(req) ? await store.getTask(s, req.params.id) : null;
if (!row || row.requester !== requesterOf(req)) {
if (!row || row.requester !== requesterOf(req)) {
@@ -316,6 +326,7 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
@@ -316,6 +326,7 @@ ${pageRows.next ? html`<p><a href="/tasks?before=${pageRows.next}">Older tasks</
const cls = (events.find((e) => e.step === 'plan' && !e.agent && /"(\w+)" task/.test(e.text || '')) || { text: '' }).text.match(/"(\w+)"/);
const cls = (events.find((e) => e.step === 'plan' && !e.agent && /"(\w+)" task/.test(e.text || '')) || { text: '' }).text.match(/"(\w+)"/);
const kind = cls ? CLASS_TEXT[cls[1]] || cls[1] : null;
const kind = cls ? CLASS_TEXT[cls[1]] || cls[1] : null;
const desk = open ? store.deskFrom(events) : null;
const desk = open ? store.deskFrom(events) : null;
+ const look = looks.get(row.id);
page(req, res, {
page(req, res, {
title: row.task.length > 60 ? `${row.task.slice(0, 59)}…` : row.task,
title: row.task.length > 60 ? `${row.task.slice(0, 59)}…` : row.task,
crumbs: [{ label: 'Your tasks', href: '/tasks' }, { label: 'Task' }],
crumbs: [{ label: 'Your tasks', href: '/tasks' }, { label: 'Task' }],
@@ -333,6 +344,10 @@ ${open ? html`<section class="desk-card" id="desk-card" aria-live="polite"${desk
@@ -333,6 +344,10 @@ ${open ? html`<section class="desk-card" id="desk-card" aria-live="polite"${desk
<p><strong>Your agent is working on a desk of its own</strong><span id="desk-kind">${desk && desk.kind === 'browser' ? ' (a browser)' : ''}</span>. Watch it live, and take it over whenever you like: while you use it, the agent waits; when you stop, it goes on.</p>
<p><strong>Your agent is working on a desk of its own</strong><span id="desk-kind">${desk && desk.kind === 'browser' ? ' (a browser)' : ''}</span>. Watch it live, and take it over whenever you like: while you use it, the agent waits; when you stop, it goes on.</p>
<a class="sc-btn sc-primary" id="desk-link" href="${desk ? desk.url : botPages}" rel="noopener" target="_blank">Watch and take over</a>
<a class="sc-btn sc-primary" id="desk-link" href="${desk ? desk.url : botPages}" rel="noopener" target="_blank">Watch and take over</a>
</section>` : ''}
</section>` : ''}
+${open || look ? html`<figure class="last-look" id="last-look"${look ? '' : raw(' hidden')}>
+<img id="last-look-img" src="${look ? `/tasks/${row.id}/look?at=${look.at}` : ''}" alt="The screen your agent last looked at"${look && look.width ? raw(` width="${Number(look.width)}" height="${Number(look.height)}"`) : ''} loading="lazy">
+<figcaption class="muted small">What your agent last saw on the screen${look ? html` · ${time(new Date(look.at).toISOString())}` : ''}</figcaption>
+</figure>` : ''}
<section class="how" aria-labelledby="h-how"><h2 id="h-how">How it went</h2>
<section class="how" aria-labelledby="h-how"><h2 id="h-how">How it went</h2>
<div class="attempts" id="attempts">${attemptsOf(events).map(attemptBlock)}</div>
<div class="attempts" id="attempts">${attemptsOf(events).map(attemptBlock)}</div>
${open ? html`<p class="muted small live-note" id="live-note"><span class="dot-pulse" aria-hidden="true"></span> Updating as the agent works.</p>` : ''}
${open ? html`<p class="muted small live-note" id="live-note"><span class="dot-pulse" aria-hidden="true"></span> Updating as the agent works.</p>` : ''}
@@ -151,6 +151,30 @@ const toolsFor = (bot) => createOpenVibeTools({ config, fetchImpl: bot.fetchImpl
@@ -151,6 +151,30 @@ const toolsFor = (bot) => createOpenVibeTools({ config, fetchImpl: bot.fetchImpl
assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, undefined);
assert.strictEqual((await t.get(`/api/v1/tasks/${id}`, { as: cy })).json().desk, undefined);
assert.ok(!(await t.get(`/tasks/${id}`, { as: cy })).text.includes('desk-card'));
assert.ok(!(await t.get(`/tasks/${id}`, { as: cy })).text.includes('desk-card'));
});
});
+ await check('what the agent last saw: its last screen picture on the owner\'s task page, nowhere else, in memory only', async () => {
+ const looks = require('../server/agents/looks');
+ const dee = t.network.addUser('desk-dee');
+ const eve = t.network.addUser('desk-eve');
+ const id = t.ctx.s.newId('tsk');
+ await store.insertTask(t.ctx.s, { id, requester: `user:${dee.subject}`, task: 'Check the form', mode: 'balanced', budget_task: 0.05, budget_day: 1, created_at: t.ctx.s.iso() });
+ assert.strictEqual((await t.get(`/tasks/${id}/look`, { as: dee })).status, 404, 'no look yet');
+ const before = await t.get(`/tasks/${id}`, { as: dee });
+ assert.ok(/<figure class="last-look" id="last-look" hidden>/.test(before.text), 'the picture waits, hidden, while the task runs');
+ const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3]);
+ assert.ok(looks.put(id, { format: 'jpeg', width: 1280, height: 800, content_b64: jpeg.toString('base64') }));
+ const img = await t.get(`/tasks/${id}/look`, { as: dee });
+ assert.strictEqual(img.status, 200);
+ assert.match(img.headers['content-type'] || (img.headers.get && img.headers.get('content-type')) || '', /^image\/jpeg/);
+ assert.match(img.headers['cache-control'] || (img.headers.get && img.headers.get('cache-control')) || '', /no-store/);
+ assert.strictEqual((await t.get(`/tasks/${id}/look`, { as: eve })).status, 404, 'someone else never sees it');
+ assert.strictEqual((await t.get(`/tasks/${id}/look`)).status, 404, 'nor someone signed out');
+ const page = await t.get(`/tasks/${id}`, { as: dee });
+ assert.ok(page.text.includes('<figure class="last-look" id="last-look">') && page.text.includes('width="1280" height="800"'));
+ assert.ok(page.text.includes('What your agent last saw on the screen'));
+ // Kept in memory for a while, then gone; never more than MAX tasks.
+ assert.strictEqual(looks.get(id, Date.now() + looks.KEEP_MS + 1), null);
+ assert.strictEqual(looks.put('tsk_x', { content_b64: '' }), false, 'an empty picture is not kept');
+ });
await t.close();
await t.close();
done();
done();
})();
})();