openvibers/openvibe.bot Commits

Bot: input — the owner and their agent use a machine's mouse and keyboard; the audit never keeps typed text (contracts 0.171) (#105)

OP OpenVibers committed 30 days ago · d25e5fb4ca · 8 files, +33 −16

Plan T14/T17 next step: "keyboard and mouse input" (contracts 0.171.0, Node#45).

**Bot accepts `input`.**
- `input` is a computer kind for the machine's owner and the owner's agent (Actor, where "Let my agent use this computer" is on). Watch still sends only `probe`. The Node's level decides what happens.
- It is validated against `bot.command@1`. A key with anything but letters, digits, `+`, `-` and `_` is `422 bot.invalid_input` before it reaches the machine.
- **The audit never keeps typed text:** an `input` row keeps the action, the point, the keys and the number of characters typed.
- `GET /api/v1/machines` reports `input`, and the machines page lists "mouse and keyboard".
- `/docs/computers` documents `input`, and the levels now mention the mouse and keyboard.
- Contracts are pinned to v0.171.0.

**Tests:** `machine-computer.test.js` adds:
- typing and a click reach the machine;
- a shell-like key is refused;
- the audit keeps `{ action: 'type', chars: 22 }` and never the text;
- the page lists "mouse and keyboard".

Changes

ddiff --git a/STATUS.json b/STATUS.json
iindex ef1622b..35afc67 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -5,7 +5,7 @@
    "runtime": "node 22, express 4, ws; PostgreSQL 18 via openvibe-sdk/db (pg, or embedded PGlite in tests); Valkey optional; no build step",
    "port": 4630,
    "domain": "openvibe.bot (www -> apex; /device and /control WebSockets; /install)",
--  "contracts": "openvibe-contracts v0.167.0",
++  "contracts": "openvibe-contracts v0.171.0",
    "sdk": "openvibe-sdk v0.44.0",
    "shared": "openvibe-shared v4.1.3",
    "packages": [
ddiff --git a/package-lock.json b/package-lock.json
iindex 5820ff3..6919324 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -13,7 +13,7 @@
          "dotenv": "^16.4.7",
          "express": "^4.21.2",
          "iovalkey": "^0.4.0",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.167.0",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.171.0",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.1.3",
          "pg": "^8.23.0",
@@@ -845,9 +845,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.167.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.167.0",
--      "integrity": "sha512-CVV3IX+jS6WJM5+7lYS3KK8ZJUNkTyBY55Pal3CG0hZlSDpIqGa/wFw1uSVgoo4vE4vkljE6hIjz6cGvopLrnQ==",
++      "version": "0.171.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.171.0",
++      "integrity": "sha512-GtmDgY/zee9u3qfVOLoh0YA4KEMAqO/HDjEg/wlKtpHSJNmoJnVR669THVVtSE4WK7ro8VJ+b0v/yj+iboch+Q==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex fc52911..ecaf423 100644
---- a/package.json
++++ b/package.json
@@@ -25,7 +25,7 @@
      "dotenv": "^16.4.7",
      "express": "^4.21.2",
      "iovalkey": "^0.4.0",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.167.0",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.171.0",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.1.3",
      "pg": "^8.23.0",
ddiff --git a/server/api/v1.js b/server/api/v1.js
iindex 4786101..a166bbf 100644
---- a/server/api/v1.js
++++ b/server/api/v1.js
@@@ -399,7 +399,7 @@ function v1Router({ domain, apiAuth, limits, hub, jobs: jobService, config, subm
              id: device.id, name: device.name || null, online: !!st.online, agent_computer: !!device.agent_computer, watch_probe: !!device.watch_probe,
              computer: offered ? {
                  level: typeof sensors.computer_level === 'string' ? sensors.computer_level : ((pick('files') || pick('terminal') || pick('screen') || {}).level || null),
--                terminal: pick('terminal'), files: pick('files'), screen: pick('screen'), probe: pick('probe'),
++                terminal: pick('terminal'), files: pick('files'), screen: pick('screen'), probe: pick('probe'), input: pick('input'),
                  waiting: Number.isInteger(sensors.computer_waiting) ? sensors.computer_waiting : null,
              } : null,
          };
ddiff --git a/server/realtime.js b/server/realtime.js
iindex 29e0637..a8b40bf 100644
---- a/server/realtime.js
++++ b/server/realtime.js
@@@ -39,7 +39,7 @@ const WATCH_MAX_PAYLOAD = 4 * 1024;   // a watcher only ever sends join/leave
  const DEVICE_MAX_PAYLOAD = 640 * 1024;
  // The Node's computer plugin (plan T14/T17): the kinds a person's machine takes, the capability each needs in its
  // status, and the reply window (the owner's approval on the machine plus the program's own run).
--const COMPUTER_CAP = { exec: 'terminal', fs: 'files', screen: 'screen', probe: 'probe' };
++const COMPUTER_CAP = { exec: 'terminal', fs: 'files', screen: 'screen', probe: 'probe', input: 'input' };
  const COMPUTER_KINDS = Object.keys(COMPUTER_CAP);
  const COMPUTER_DEFAULT_MS = 60000;
  const COMPUTER_MAX_MS = 600000;
@@@ -48,6 +48,8 @@ function computerSummary(kind, value) {
      if (kind === 'exec') return { program: String(value.argv[0]).slice(0, 120), args: value.argv.length - 1 };
      if (kind === 'fs') return { op: value.op, path: String(value.path).slice(0, 300), ...(typeof value.content_b64 === 'string' ? { bytes: Math.floor((value.content_b64.length * 3) / 4) } : {}) };
      if (kind === 'probe') { try { const u = new URL(value.target); return { scheme: u.protocol.replace(':', ''), host: u.host }; } catch { return { scheme: null }; } }
++    // Typed text can be anything (a password the person asked for): the audit keeps its length, never the text.
++    if (kind === 'input') return { action: value.action, ...(value.x != null ? { x: value.x, y: value.y } : {}), ...(typeof value.text === 'string' ? { chars: value.text.length } : {}), ...(value.keys ? { keys: value.keys } : {}), ...(value.dy ? { dy: value.dy } : {}) };
      return { max_width: value.max_width || null };
  }
  
ddiff --git a/server/web/docs.js b/server/web/docs.js
iindex 5d57322..287a98b 100644
---- a/server/web/docs.js
++++ b/server/web/docs.js
@@@ -362,7 +362,7 @@ const COMPUTER_CALL = `curl -X POST https://openvibe.bot/api/v1/machines/dev_…
  function renderComputersPage({ config }) {
      const content = `
  <h1>Your computer for your agents</h1>
--<p class="muted">The OpenVibe Node's <b>computer</b> plugin lets your agents use your own computer: its terminal, its files and its screen. It is off unless you install it, it never runs as root, and you decide what it may do on the computer itself. Nothing on OpenVibe can raise that level.</p>
++<p class="muted">The OpenVibe Node's <b>computer</b> plugin lets your agents use your own computer: its terminal, its files, its screen and (when you allow it) its mouse and keyboard. It is off unless you install it, it never runs as root, and you decide what it may do on the computer itself. Nothing on OpenVibe can raise that level.</p>
  
  <section class="card" aria-labelledby="install-h"><h2 id="install-h">Install it</h2>
  <p>On ${inLink('/machines', 'Your machines')}, choose <b>Add a machine</b> and run the second command it shows on the computer, from your own account (the installer uses sudo itself):</p>
@@@ -372,8 +372,8 @@ ${code(COMPUTER_INSTALL)}
  <section class="card" aria-labelledby="levels-h"><h2 id="levels-h">Levels, set on the computer</h2>
  <ul>
  <li><b>observe</b> (the default): take a screenshot; list, inspect and read files inside its folders.</li>
--<li><b>ask</b>: also run programs and write files there, each only after you approve it on the computer.</li>
--<li><b>trusted</b>: run programs and write files inside its folders without asking.</li>
++<li><b>ask</b>: also run programs, write files there and use the mouse and keyboard, each only after you approve it on the computer (the approval says exactly what: "click at 640, 360", "type 12 characters", "press ctrl+s").</li>
++<li><b>trusted</b>: run programs, write files inside its folders and use the mouse and keyboard without asking.</li>
  <li><b>full</b>: as trusted, anywhere your account can reach.</li>
  </ul>
  <p>A program is always a program and its arguments, never a shell line. It runs for at most the time you set (30 seconds by default), at most two at once, and its output is cut at 64 KB. A path is checked after links are followed, so a link out of a folder is outside it.</p></section>
@@@ -394,11 +394,12 @@ ${code(COMPUTER_CALL)}
  <li><code>exec</code>: <code>argv</code>, <code>cwd</code>, <code>timeout_s</code>, <code>stdin</code> → <code>exit_code</code>, <code>stdout</code>, <code>stderr</code>, <code>truncated</code>, <code>timed_out</code>.</li>
  <li><code>fs</code>: <code>op</code> list, stat, read (<code>offset</code>, <code>length</code>), write (<code>content_b64</code>, <code>mode</code> create, overwrite or append) or mkdir, and an absolute <code>path</code>.</li>
  <li><code>screen</code>: <code>max_width</code> → a JPEG in <code>content_b64</code>.</li>
++<li><code>input</code>: <code>action</code> (<code>move</code>, <code>click</code>, <code>double_click</code>, <code>right_click</code> at <code>x</code>, <code>y</code>; <code>scroll</code> by <code>dy</code>; <code>type</code> a <code>text</code>; <code>key</code> one key or chord such as <code>ctrl+s</code>), with <code>width</code> and <code>height</code> of the screenshot the coordinates come from; the computer scales them to its real screen. It acts, so your level decides: never at observe, approved one by one at ask. xdotool on Linux (X11), cliclick on macOS. The audit keeps the action and the number of characters typed, never the text.</li>
  <li><code>probe</code>: <code>target</code> (<code>http(s)://…</code> or <code>tcp://host:port</code> on the computer's own network) → the status and body of one GET, or whether the port is open. It is read-only and runs at every level.</li>
  </ul>
  <p class="muted"><code>GET /api/v1/machines</code> lists your machines with their level. A machine whose plugin is off answers <code>409 bot.computer_off</code>; a refusal on the computer answers <code>result: "nack"</code> with the reason. The audit keeps the program name or the file and path, never its content.</p></section>
  `;
--    return docsPage({ config, path: '/docs/computers', title: 'Your computer for your agents', description: 'Let your agents use your own computer through the OpenVibe Node: its terminal, files and screen, at the level you set on the computer, with approvals there and a kill switch.', content });
++    return docsPage({ config, path: '/docs/computers', title: 'Your computer for your agents', description: 'Let your agents use your own computer through the OpenVibe Node: its terminal, files, screen, mouse and keyboard, at the level you set on the computer, with approvals there and a kill switch.', content });
  }
  
  module.exports = { renderDocsIndex, renderDriversPage, renderProfilesPage, renderAutomationPage, renderComputersPage, validationResult };
ddiff --git a/server/web/render.js b/server/web/render.js
iindex 91f3724..0411e8c 100644
---- a/server/web/render.js
++++ b/server/web/render.js
@@@ -594,12 +594,12 @@ function renderMachinesPage({ machines = [], pairing = null }) {
          const c = (m.capabilities && typeof m.capabilities === 'object' ? m.capabilities : {}).computer;
          if (!c || typeof c !== 'object') return null;
          const level = m.computer_level || ((c.files || c.terminal || c.screen || {}).level) || null;
--        return { level: LEVELS[level] ? level : null, terminal: !!c.terminal, files: !!c.files, screen: !!c.screen, probe: !!c.probe };
++        return { level: LEVELS[level] ? level : null, terminal: !!c.terminal, files: !!c.files, screen: !!c.screen, probe: !!c.probe, input: !!c.input };
      };
      const computerRow = (m) => {
          const c = computerOf(m);
          if (!c) return '';
--        const can = [c.terminal && 'terminal', c.files && 'files', c.screen && 'screen'].filter(Boolean).join(', ');
++        const can = [c.terminal && 'terminal', c.files && 'files', c.screen && 'screen', c.input && 'mouse and keyboard'].filter(Boolean).join(', ');
          const waiting = Number.isInteger(m.computer_waiting) && m.computer_waiting > 0
              ? ` <span class="pill warn">${m.computer_waiting} waiting for you on the machine</span>` : '';
          return `<div class="machine-computer"><span class="pill">Computer: ${esc(c.level ? LEVELS[c.level] : 'on')}</span><span class="muted">${esc(can)}</span>${waiting}`
ddiff --git a/test/machine-computer.test.js b/test/machine-computer.test.js
iindex 4015b1e..3a97240 100644
---- a/test/machine-computer.test.js
++++ b/test/machine-computer.test.js
@@@ -14,7 +14,7 @@ const { boot, check, done } = require('./helpers/app');
      let seq = 0;
      const frame = (type, fields) => ({ v: 1, seq: ++seq, ts: Date.now(), type, ...fields });
      const poll = async (fn, ms = 8000) => { const end = Date.now() + ms; while (Date.now() < end) { const v = await fn(); if (v) return v; await t.wait(30); } return null; };
--    const COMPUTER = { terminal: { level: 'trusted', timeout_s: 30 }, files: { level: 'trusted', roots: ['/home/alex'] }, screen: { level: 'trusted', tool: 'grim' }, probe: { lan_only: true } };
++    const COMPUTER = { terminal: { level: 'trusted', timeout_s: 30 }, files: { level: 'trusted', roots: ['/home/alex'] }, screen: { level: 'trusted', tool: 'grim' }, probe: { lan_only: true }, input: { level: 'trusted', tool: 'xdotool', scroll: true } };
      const seen = [];
  
      /** A Network-paired machine of `owner` whose fake Node answers computer commands like the plugin would. */
@@@ -32,6 +32,7 @@ const { boot, check, done } = require('./helpers/app');
              if (m.kind === 'exec') c.send(frame('ack', { id: m.id, latency_ms: 5, data: { exit_code: 0, stdout: m.value.argv.includes('big') ? 'x'.repeat(200 * 1024) : 'hi\n', stderr: '', truncated: false, timed_out: false, duration_ms: 4 } }));
              else if (m.kind === 'fs') c.send(frame('ack', { id: m.id, data: { path: m.value.path, entries: [], total: 0, truncated: false } }));
              else if (m.kind === 'probe') c.send(frame('ack', { id: m.id, data: { status: 200, body: '{"up":true}', ms: 3 } }));
++            else if (m.kind === 'input') c.send(frame('ack', { id: m.id, data: { done: true, action: m.value.action } }));
              else c.send(frame('nack', { id: m.id, fault_code: 'not_allowed', message: 'this machine allows observing only' }));
          });
          await poll(() => t.hub.computerOf(deviceId) !== null || !computer);
@@@ -64,12 +65,25 @@ const { boot, check, done } = require('./helpers/app');
          assert.deepStrictEqual([nack.status, nack.json.result, nack.json.reason], [200, 'nack', 'this machine allows observing only']);
      });
  
++    await check('mouse and keyboard reach the machine as they were asked; a key is a key, never a command line', async () => {
++        const typed = await cmd(mine.deviceId, { kind: 'input', value: { action: 'type', text: 'hunter2 is my password' } }, { user: alex });
++        assert.deepStrictEqual([typed.status, typed.json.result, typed.json.data], [200, 'ack', { done: true, action: 'type' }]);
++        const click = await cmd(mine.deviceId, { kind: 'input', value: { action: 'click', x: 640, y: 360, width: 1280, height: 720 } }, { user: alex });
++        assert.strictEqual(click.json.result, 'ack');
++        assert.deepStrictEqual(seen.filter((m) => m.kind === 'input').map((m) => m.value.action), ['type', 'click']);
++        const shell = await cmd(mine.deviceId, { kind: 'input', value: { action: 'key', keys: 'ctrl+s; rm -rf ~' } }, { user: alex });
++        assert.deepStrictEqual([shell.status, shell.json.code], [422, 'bot.invalid_input']);
++    });
++
      await check('the audit keeps what was asked in short, never stdin, arguments or content', async () => {
          const rows = await t.db.many('SELECT * FROM command_audit WHERE robot_id = $1 ORDER BY id', [mine.deviceId]);
          assert.ok(rows.length >= 3);
          const exec = rows.find((r) => r.kind === 'exec');
          assert.deepStrictEqual(exec.value, { program: 'echo', args: 1 });
          assert.ok(!JSON.stringify(rows).includes('secret input'));
++        const typed = rows.find((r) => r.kind === 'input' && r.value.action === 'type');
++        assert.deepStrictEqual(typed.value, { action: 'type', chars: 22 }, 'typed text: its length, never the text');
++        assert.ok(!JSON.stringify(rows).includes('hunter2'));
      });
  
      await check('nobody else reaches it; a bad value and a missing plugin are refused', async () => {
@@@ -112,7 +126,7 @@ const { boot, check, done } = require('./helpers/app');
          const form = (p, user, body, headers = {}) => fetch(t.base + p, { method: 'POST', redirect: 'manual', body, headers: { 'Content-Type': 'application/x-www-form-urlencoded', ...cookie(user), ...headers } });
          const html = await (await fetch(`${t.base}/machines`, { headers: cookie(alex) })).text();
          assert.match(html, /Computer: Trusted/);
--        assert.match(html, /terminal, files, screen/);
++        assert.match(html, /terminal, files, screen, mouse and keyboard/);
          assert.match(html, /Let my agent use this computer/);
          assert.strictEqual((await form(`/machines/${mine.deviceId}/agent-computer`, alex, 'on=1', { Origin: 'https://evil.test' })).status, 403);
          assert.strictEqual((await form(`/machines/${mine.deviceId}/agent-computer`, bob, 'on=1')).status, 404);