openvibers/openvibe.link Commits
Node: input — the computer plugin uses the mouse and keyboard at the level the owner set (approved one by one at ask) (#45)
Changes
@@@ -25,12 +25,12 @@ By hand, or on another system, add it to `plugins` in the Node's `config.json` a## Control levels--| level | screen, list, stat, read (inside the folders) | run programs, write files (inside the folders) | outside the folders |--|-----------|----------------------------------------------|-----------------------------------------------|---------------------|--| `observe` | yes | no (`nack not_allowed`) | no |--| `ask` | yes | after you approve each one on the machine | no |--| `trusted` | yes | yes | no |--| `full` | yes | yes | yes |++| level | screen, list, stat, read (inside the folders) | run programs, write files (inside the folders), mouse and keyboard | outside the folders |++|-----------|----------------------------------------------|--------------------------------------------------------------------|---------------------|++| `observe` | yes | no (`nack not_allowed`) | no |++| `ask` | yes | after you approve each one on the machine | no |++| `trusted` | yes | yes | no |++| `full` | yes | yes | yes |`observe` is the default. Folders are `roots` (default: the home folder of the account running the Node); a path ischecked after symlinks are resolved, so a link out of a folder is outside it.@@@ -48,7 +48,7 @@ When the Node runs in your graphical session, a waiting request also shows a desthat a request is waiting on the machine.**Kill switch:** `openvibe-node stop` (and an owner's e-stop) drops the plugin to `observe`: what runs is killed, what--waits is denied, and the core also refuses `exec` and file writes until `openvibe-node resume`. `halt` kills what runs.++waits is denied, and the core also refuses `exec`, file writes and `input` until `openvibe-node resume`. `halt` kills what runs.## What it does@@@ -58,12 +58,21 @@ waits is denied, and the core also refuses `exec` and file writes until `openvib| `fs` | `op` `list` / `stat` / `read` (`offset`, `length`) / `write` (`content_b64`, `mode` `create`, `overwrite` or `append`) / `mkdir`, and an absolute `path` | `entries` (≤ 1000), the stat, `content_b64` (≤ `max_read_bytes`), `written`, `created` || `probe` | `target`: `http(s)://…` (one GET, no redirect followed) or `tcp://host:port`; `timeout_s` | `status`, `content_type`, `etag`, `last_modified`, `body` (≤ 64 KB), `ms`; or `open`, `ms` || `screen` | `max_width` (default 1280) | `format` (`jpeg` with Pillow, else `png` ≤ 300 KB), `width`, `height`, `content_b64` |++| `input` | `action`: `move`, `click`, `double_click`, `right_click` (`x`, `y`), `scroll` (`dy` clicks, ±20), `type` (`text`, ≤ 2000), `key` (`keys`: one key or chord, `ctrl+s`, `Return`); `width`, `height`: the picture `x`, `y` are measured in | `done`, `action`, and the real `x`, `y` it used |A program runs in its own process group with the Node account's rights, at most 2 at a time, and is killed at`timeout_s` (the owner's `timeout_s` is the cap). A write with `overwrite` replaces the file atomically. Screens comefrom `screencapture` (macOS), `grim` (Wayland) or `import`, `scrot` or `gnome-screenshot` (X11), so the Node must run inyour graphical session for `screen`; without one the plugin simply does not offer it.++**Mouse and keyboard** (`input`) is an acting command like running a program: refused at `observe`, approved one by one++at `ask` (the approval says exactly what it would do: "click at 640, 360 of a 1280×720 picture of the screen",++"type 12 characters: 'hello there'", "press ctrl+s"). Coordinates are in the space of the screen picture the agent last++saw (`width` × `height`), scaled to your real screen. It uses `xdotool` on an X11 session and `cliclick` on macOS++(`brew install cliclick`; no scrolling there); Wayland has no general input tool yet, so it is not offered there. Typed++text and keys are passed to the tool as arguments, never through a shell, and `keys` takes only letters, digits, `+`,++`-` and `_`.++## Config| key | default | meaning |@@@ -77,8 +86,9 @@ your graphical session for `screen`; without one the plugin simply does not offe| `approval_timeout_s` | 120 (≤ 600) | how long a request waits for you || `notify` | `true` | a desktop notification when a request waits (graphical session only) || `probe` | `true` | `false`: no probes (OpenVibe.Watch's `node` source) |++| `allow_input` | `true` | `false`: never use the mouse or keyboard (no `input` capability) |--`describe` reports `terminal`, `files` and `screen` with the level and folders, so Bot shows what the machine allows;++`describe` reports `terminal`, `files`, `screen` and `input` with the level and folders, so Bot shows what the machine allows;telemetry carries the sensors `computer_level` (observe while a stop is latched), `computer_waiting` and`computer_running`.Every request is logged by the Node (`journalctl -u openvibe-node`), with what it would do, approved or denied.@@@ -218,6 +218,9 @@ func TestComputerLocalStopLeavesObserveOnly(t *testing.T) {if nk := nackOf(t, e.computerCmd("s2", protocol.KindFS, map[string]any{"op": "write", "path": filepath.Join(work, "x"), "content_b64": "eA=="}, 5*time.Second)); nk.FaultCode != protocol.FaultLocalStop {t.Fatalf("write while stopped: %+v", nk)}++ if nk := nackOf(t, e.computerCmd("s2i", protocol.KindInput, map[string]any{"action": "key", "keys": "Return"}, 5*time.Second)); nk.FaultCode != protocol.FaultLocalStop {++ t.Fatalf("input while stopped: %+v", nk)++ }if d := dataOf(t, e.computerCmd("s3", protocol.KindFS, map[string]any{"op": "stat", "path": work}, 5*time.Second)); d["type"] != "dir" {t.Fatalf("observing while stopped: %v", d)}@@@ -21,6 +21,7 @@ var capabilityFor = map[string]string{protocol.KindFS: "files",protocol.KindScreen: "screen",protocol.KindProbe: "probe",++ protocol.KindInput: "input",}// Manager supervises every configured plugin.@@@ -164,13 +164,16 @@ const (// KindProbe is a read-only check of something on the machine's own network (one GET or a TCP connect to a private,// loopback or link-local address): OpenVibe.Watch's node carrier.KindProbe = "probe"++ // KindInput moves the mouse, clicks, scrolls, types or presses keys on the machine's screen (contracts 0.171.0):++ // an acting command, so the owner's level decides and a latched stop refuses it.++ KindInput = "input")// Kinds lists every command kind the Node understands.--var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe}++var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput}// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.--var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true}++var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true}const (// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps@@@ -182,11 +185,12 @@ const (MaxAckData = 512 * 1024)--// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file): those are--// refused while the local stop or an e-stop is latched. Capturing the screen and reading files are not.++// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file, uses the mouse++// or keyboard): those are refused while the local stop or an e-stop is latched. Capturing the screen and reading files++// are not.func ComputerActs(kind string, value json.RawMessage) bool {switch kind {-- case KindExec:++ case KindExec, KindInput:return truecase KindFS:var v struct {@@@ -1,12 +1,13 @@--"""OpenVibe Node computer plugin: the machine's terminal, files and screen for its owner's agents (plan T14/T17).++"""OpenVibe Node computer plugin: the machine's terminal, files, screen and input for its owner's agents (plan T14/T17).Off unless its owner lists it in the Node's config. The owner sets the **control level** on the machine itself; a cloudcommand can never raise it:* ``observe`` (the default): capture the screen; list, stat and read files inside the roots.--* ``ask``: also run programs and write files inside the roots, each only after the owner approves it on the machine++* ``ask``: also run programs, write files inside the roots and use the mouse and keyboard, each only after the owner++ approves it on the machine(``openvibe-node approvals``, ``approve <id>``, ``deny <id>``); unanswered requests expire.--* ``trusted``: run programs and write files inside the roots without asking.++* ``trusted``: run programs, write files inside the roots and use the mouse and keyboard without asking.* ``full``: as trusted, anywhere the account running the Node can reach.The local kill switch (``openvibe-node stop``) and a remote e-stop drop the plugin to observe, kill what is running@@@ -19,6 +20,12 @@ Command kinds and their answers (the ack's ``data``):* ``fs`` ``{"op": "list"|"stat"|"read"|"write"|"mkdir", "path", …}``: ``read`` takes ``offset``/``length`` and answers``content_b64``; ``write`` takes ``content_b64`` and ``mode`` (``create``, ``overwrite`` or ``append``).* ``screen`` ``{"max_width"?}`` → ``{format, width, height, content_b64}`` (JPEG through Pillow when it is installed).++* ``input`` ``{"action", "x"?, "y"?, "width"?, "height"?, "dy"?, "text"?, "keys"?}`` → ``{done, action, x?, y?}``: move,++ click, double_click or right_click at a point, scroll by ``dy`` clicks, type ``text`` into whatever has focus, or press++ one key or chord (``keys``, for example ``ctrl+s``). ``x``/``y`` are in the space of a screen picture ``width`` ×++ ``height`` (the ``screen`` answer) and are scaled to the real screen. An acting command: refused at observe, approved++ one by one at ask. xdotool on X11, cliclick on macOS (no scroll there); none on Wayland yet. Config++ ``allow_input: false`` turns it off.* ``probe`` ``{"target", "timeout_s"?}`` → for ``http(s)://…`` one GET, no redirect followed: ``{status, content_type,etag, last_modified, body, truncated, ms}``; for ``tcp://host:port``: ``{open, ms, error?}``. Read-only, so it runs atevery level, and only towards the machine's own network: every address the name resolves to must be private,@@@ -57,6 +64,17 @@ MAX_WAITING = 8MAX_OUTPUT_CAP = 192 * 1024MAX_READ_CAP = 256 * 1024SCREEN_MAX_BYTES = 300 * 1024++INPUT_ACTIONS = frozenset({"move", "click", "double_click", "right_click", "scroll", "type", "key"})++POINT_ACTIONS = frozenset({"move", "click", "double_click", "right_click"})++MAX_TYPE = 2000++KEYS_OK = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_+-")++MODIFIERS = {"ctrl": "ctrl", "control": "ctrl", "alt": "alt", "option": "alt", "shift": "shift", "cmd": "cmd",++ "command": "cmd", "super": "super", "meta": "super", "win": "super"}++# xdotool key names → cliclick's kp: names (macOS); a single character is typed with t:.++CLICLICK_KEYS = {"return": "return", "enter": "enter", "escape": "esc", "esc": "esc", "tab": "tab", "backspace": "delete",++ "delete": "fwd-delete", "space": "space", "up": "arrow-up", "down": "arrow-down", "left": "arrow-left",++ "right": "arrow-right", "home": "home", "end": "end", "page_up": "page-up", "prior": "page-up",++ "page_down": "page-down", "next": "page-down"}def _int(v, name, lo, hi, default):@@@ -69,13 +87,14 @@ def _int(v, name, lo, hi, default):class Computer(Plugin):driver = "computer"-- version = "0.3.0"++ version = "0.4.0"motion_kinds = frozenset()telemetry_interval_s = 2.0-- def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None):++ def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None, screen_size=None):self.which = whichself.popen = popen++ self.screen_size = screen_size or self._screen_sizeself.clock = clockself.env = os.environ if env is None else envself.platform = platform or sys.platform@@@ -86,6 +105,7 @@ class Computer(Plugin):self.max_read = 256 * 1024self.allow_exec = Trueself.allow_probe = True++ self.allow_input = Trueself.notify = Trueself.approval_timeout_s = 120self.pending = {} # command id -> (run, expires_at, summary)@@@ -111,6 +131,7 @@ class Computer(Plugin):self.approval_timeout_s = _int(config.get("approval_timeout_s"), "approval_timeout_s", 10, 600, 120)self.allow_exec = config.get("allow_exec", True) is not Falseself.allow_probe = config.get("probe", True) is not False++ self.allow_input = config.get("allow_input", True) is not Falseself.notify = config.get("notify", True) is not Falsedef describe(self, config):@@@ -126,6 +147,9 @@ class Computer(Plugin):caps["screen"] = {"level": self.level, "tool": tool[0]}if self.allow_probe:caps["probe"] = {"lan_only": True, "schemes": ["http", "https", "tcp"]}++ itool = self._input_tool() if self.allow_input else None++ if itool:++ caps["input"] = {"level": self.level, "tool": itool, "scroll": itool == "xdotool"}return capsdef setup(self, config):@@@ -221,6 +245,14 @@ class Computer(Plugin):raise Fault("unsupported", "this machine does not allow probes")target, timeout = self._probe_target(v)return self._start(cmd, lambda: self._probe(target, timeout))++ if cmd.kind == "input":++ if not self.allow_input:++ raise Fault("unsupported", "this machine does not allow mouse and keyboard input")++ tool = self._input_tool()++ if not tool:++ raise Fault("unsupported", "no input tool on this machine (xdotool on X11, cliclick on macOS)")++ run, summary = self._input_op(tool, v)++ return self._act(cmd, level, run, summary, threaded=True)if cmd.kind == "exec":if not self.allow_exec:raise Fault("unsupported", "this machine does not allow running programs")@@@ -572,6 +604,138 @@ class Computer(Plugin):verb = {"create": "create", "overwrite": "overwrite", "append": "append to"}[mode]return run, "%s %s (%d bytes)" % (verb, path, len(data))++ # ---- input -----------------------------------------------------------------------------------------------------++ def _input_tool(self):++ """xdotool on an X11 session, cliclick on macOS, else None (Wayland has no general input tool yet)."""++ if self.platform == "darwin" and self.which("cliclick"):++ return "cliclick"++ if self.platform.startswith("linux") and self.env.get("DISPLAY") and not self.env.get("WAYLAND_DISPLAY") and self.which("xdotool"):++ return "xdotool"++ return None++++ def _input_op(self, tool, v):++ """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask)."""++ action = v.get("action")++ if action not in INPUT_ACTIONS:++ raise Fault("bad_value", "action is one of %s" % ", ".join(sorted(INPUT_ACTIONS)))++ point = None++ if action in POINT_ACTIONS or (action == "scroll" and v.get("x") is not None):++ x = _int(v.get("x"), "x", 0, 16384, None)++ y = _int(v.get("y"), "y", 0, 16384, None)++ if x is None or y is None:++ raise Fault("bad_value", "%s needs x and y" % action)++ w = _int(v.get("width"), "width", 1, 16384, None)++ h = _int(v.get("height"), "height", 1, 16384, None)++ if (w is None) != (h is None):++ raise Fault("bad_value", "width and height come together")++ point = (x, y, w, h)++ if action == "scroll":++ dy = _int(v.get("dy"), "dy", -20, 20, None)++ if not dy:++ raise Fault("bad_value", "scroll needs dy (clicks, -20 to 20, not 0)")++ if tool != "xdotool":++ raise Fault("unsupported", "scrolling needs xdotool (Linux X11)")++ if action == "type":++ text = v.get("text")++ if not isinstance(text, str) or not text or len(text) > MAX_TYPE or "\x00" in text:++ raise Fault("bad_value", "text is 1 to %d characters" % MAX_TYPE)++ if action == "key":++ keys = v.get("keys")++ if not isinstance(keys, str) or not keys or len(keys) > 64 or not set(keys) <= KEYS_OK or keys.startswith("-"):++ raise Fault("bad_value", "keys is one key or chord, for example Return or ctrl+s")++++ def run():++ at = self._scale(tool, point) if point else None++ argv = self._input_argv(tool, action, at, v)++ proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True)++ try:++ _, err = proc.communicate(timeout=30)++ except subprocess.TimeoutExpired:++ self._kill(proc)++ raise Fault("hardware", "%s did not finish in 30 s" % tool)++ if proc.returncode != 0:++ raise Fault("hardware", "%s failed: %s" % (tool, (err or b"").decode("utf-8", "replace")[:200]))++ out = {"done": True, "action": action}++ if at:++ out["x"], out["y"] = at++ return out++++ if action == "type":++ text = v["text"]++ shown = text if len(text) <= 120 else text[:120] + "…"++ summary = "type %d characters: %r" % (len(text), shown)++ elif action == "key":++ summary = "press %s" % v["keys"]++ elif action == "scroll":++ summary = "scroll %s %d" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]))++ else:++ x, y, w, h = point++ summary = "%s at %d, %d%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "")++ return run, summary++++ def _screen_size(self, tool):++ """The screen's size in the units the input tool uses: pixels for xdotool, points for cliclick (a Retina++ screen's picture is twice as many pixels)."""++ if tool == "xdotool":++ out = subprocess.run(["xdotool", "getdisplaygeometry"], capture_output=True, text=True, timeout=10, env=self._tool_env())++ parts = out.stdout.split()++ else:++ out = subprocess.run(["osascript", "-e", "tell application \"Finder\" to get bounds of window of desktop"],++ capture_output=True, text=True, timeout=10)++ parts = [p.strip() for p in out.stdout.split(",")][2:4]++ if out.returncode != 0 or len(parts) != 2 or not all(p.isdigit() for p in parts):++ raise Fault("hardware", "could not read the screen size")++ return int(parts[0]), int(parts[1])++++ def _scale(self, tool, point):++ x, y, w, h = point++ if not w:++ return x, y++ sw, sh = self.screen_size(tool)++ return min(sw - 1, round(x * sw / w)), min(sh - 1, round(y * sh / h))++++ def _tool_env(self):++ env = dict(os.environ)++ env.update({k: v for k, v in self.env.items() if k in ("DISPLAY", "XAUTHORITY")})++ return env++++ @staticmethod++ def _input_argv(tool, action, at, v):++ if tool == "xdotool":++ move = ["mousemove", "--sync", str(at[0]), str(at[1])] if at else []++ if action == "move":++ return ["xdotool"] + move++ if action in ("click", "double_click", "right_click"):++ button = "3" if action == "right_click" else "1"++ repeat = ["--repeat", "2"] if action == "double_click" else []++ return ["xdotool"] + move + ["click"] + repeat + [button]++ if action == "scroll":++ return ["xdotool"] + move + ["click", "--repeat", str(abs(v["dy"])), "4" if v["dy"] < 0 else "5"]++ if action == "type":++ return ["xdotool", "type", "--delay", "12", "--", v["text"]]++ return ["xdotool", "key", "--", v["keys"]]++ # cliclick (macOS): c: click, dc: double, rc: right, m: move, t: type, kd/ku: modifier down/up, kp: a named key.++ if action in POINT_ACTIONS:++ verb = {"move": "m", "click": "c", "double_click": "dc", "right_click": "rc"}[action]++ return ["cliclick", "%s:%d,%d" % (verb, at[0], at[1])]++ if action == "type":++ return ["cliclick", "t:" + v["text"]]++ parts = v["keys"].split("+")++ mods = [MODIFIERS.get(p.lower()) for p in parts[:-1]]++ if None in mods:++ raise Fault("bad_value", "unknown modifier in %s" % v["keys"])++ key = parts[-1]++ if len(key) == 1:++ press = ["t:" + key]++ elif key.lower() in CLICLICK_KEYS:++ press = ["kp:" + CLICLICK_KEYS[key.lower()]]++ elif key.lower().startswith("f") and key[1:].isdigit() and 1 <= int(key[1:]) <= 16:++ press = ["kp:" + key.lower()]++ else:++ raise Fault("unsupported", "the key %s is not supported on macOS" % key)++ mods = ["cmd" if m == "super" else m for m in mods]++ return ["cliclick"] + ["kd:" + m for m in mods] + press + ["ku:" + m for m in reversed(mods)]++# ---- screen ----------------------------------------------------------------------------------------------------def _screen_tool(self):"""(name, argv builder) for this machine's screen capture, or None."""@@@ -304,5 +304,92 @@ class ComputerTest(unittest.TestCase):self.assertEqual(base64.b64decode(d["content_b64"])[:2], b"\xff\xd8")++ # ---- input ---------------------------------------------------------------------------------------------------++ def boot_input(self, level, platform="linux", tool="xdotool", size=(2560, 1440)):++ self.argvs = []++++ class Proc:++ returncode = 0++ pid = 1++++ def communicate(self_, timeout=None):++ return b"", b""++++ def popen(argv, **kw):++ self.argvs.append(argv)++ return Proc()++++ self.dir = tempfile.mkdtemp(prefix="ov-computer-")++ self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True))++ self.out = Out()++ env = {"DISPLAY": ":0"} if platform == "linux" else {}++ self.plugin = Computer(which=lambda n: "/usr/bin/" + n if n == tool else None, popen=popen, env=env,++ platform=platform, screen_size=lambda t: size)++ self.rt = Runtime(self.plugin, infile=io.StringIO(""), outfile=self.out)++ self.rt.dispatch({"op": "hello", "config": {"level": level, "roots": [self.dir], "notify": False}})++ self.n = 0++ return self.out.msgs()++++ def test_input_tool_detection(self):++ xdo = lambda n: "/usr/bin/xdotool" if n == "xdotool" else None++ self.assertEqual(Computer(which=xdo, env={"DISPLAY": ":0"}, platform="linux")._input_tool(), "xdotool")++ self.assertIsNone(Computer(which=xdo, env={"DISPLAY": ":0", "WAYLAND_DISPLAY": "wayland-0"}, platform="linux")._input_tool(), "not on Wayland")++ self.assertIsNone(Computer(which=xdo, env={}, platform="linux")._input_tool(), "no session")++ self.assertEqual(Computer(which=lambda n: "/opt/homebrew/bin/cliclick" if n == "cliclick" else None, env={}, platform="darwin")._input_tool(), "cliclick")++ caps = self.boot_input("trusted")[0]["capabilities"]++ self.assertEqual(caps["input"], {"level": "trusted", "tool": "xdotool", "scroll": True})++++ def test_input_is_an_acting_command(self):++ self.boot_input("observe")++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 10, "y": 10}))++ self.assertEqual((r["op"], r["fault_code"]), ("nack", "not_allowed"))++ self.boot_input("ask")++ cid = self.cmd("input", {"action": "click", "x": 640, "y": 360, "width": 1280, "height": 720})++ ev = self.out.wait_for(lambda m: m.get("op") == "event" and m.get("name") == "approval")++ self.assertEqual((ev["kind"], ev["summary"]), ("input", "click at 640, 360 of a 1280\u00d7720 picture of the screen"))++ self.assertEqual(self.argvs, [], "nothing touches the screen before the owner approves")++ self.plugin.on_approval(cid, True)++ self.assertEqual(reply(self.out, cid)["op"], "ack")++++ def test_input_scales_the_picture_to_the_screen(self):++ self.boot_input("trusted")++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 640, "y": 360, "width": 1280, "height": 720}))++ self.assertEqual(r["data"], {"done": True, "action": "click", "x": 1280, "y": 720})++ self.assertEqual(self.argvs[-1], ["xdotool", "mousemove", "--sync", "1280", "720", "click", "1"])++ reply(self.out, self.cmd("input", {"action": "double_click", "x": 5, "y": 5}))++ self.assertEqual(self.argvs[-1], ["xdotool", "mousemove", "--sync", "5", "5", "click", "--repeat", "2", "1"], "no picture: screen pixels")++ reply(self.out, self.cmd("input", {"action": "scroll", "dy": -3}))++ self.assertEqual(self.argvs[-1], ["xdotool", "click", "--repeat", "3", "4"])++ reply(self.out, self.cmd("input", {"action": "type", "text": "-hello"}))++ self.assertEqual(self.argvs[-1], ["xdotool", "type", "--delay", "12", "--", "-hello"], "text is never an option")++ reply(self.out, self.cmd("input", {"action": "key", "keys": "ctrl+s"}))++ self.assertEqual(self.argvs[-1], ["xdotool", "key", "--", "ctrl+s"])++++ def test_input_refuses_what_it_should_not_send(self):++ self.boot_input("trusted")++ before = len(self.argvs)++ for value, code in [({"action": "key", "keys": "ctrl+s; rm -rf ~"}, "bad_value"),++ ({"action": "key", "keys": "--help"}, "bad_value"),++ ({"action": "drag"}, "bad_value"),++ ({"action": "click"}, "bad_value"),++ ({"action": "click", "x": 1, "y": 1, "width": 100}, "bad_value"),++ ({"action": "type", "text": "x" * 2001}, "bad_value"),++ ({"action": "scroll", "dy": 0}, "bad_value")]:++ r = reply(self.out, self.cmd("input", value))++ self.assertEqual((r["op"], r["fault_code"]), ("nack", code), value)++ self.assertEqual(len(self.argvs), before)++++ def test_input_on_macos_uses_cliclick(self):++ self.boot_input("trusted", platform="darwin", tool="cliclick", size=(1440, 900))++ reply(self.out, self.cmd("input", {"action": "right_click", "x": 640, "y": 400, "width": 1280, "height": 800}))++ self.assertEqual(self.argvs[-1], ["cliclick", "rc:720,450"])++ reply(self.out, self.cmd("input", {"action": "key", "keys": "cmd+shift+s"}))++ self.assertEqual(self.argvs[-1], ["cliclick", "kd:cmd", "kd:shift", "t:s", "ku:shift", "ku:cmd"])++ reply(self.out, self.cmd("input", {"action": "key", "keys": "Return"}))++ self.assertEqual(self.argvs[-1], ["cliclick", "kp:return"])++ r = reply(self.out, self.cmd("input", {"action": "scroll", "dy": 2}))++ self.assertEqual((r["op"], r["fault_code"]), ("nack", "unsupported"))++++if __name__ == "__main__":unittest.main()
@@ -25,12 +25,12 @@ By hand, or on another system, add it to `plugins` in the Node's `config.json` a
@@ -25,12 +25,12 @@ By hand, or on another system, add it to `plugins` in the Node's `config.json` a
## Control levels
## Control levels
-| level | screen, list, stat, read (inside the folders) | run programs, write files (inside the folders) | outside the folders |
-|-----------|----------------------------------------------|-----------------------------------------------|---------------------|
-| `observe` | yes | no (`nack not_allowed`) | no |
-| `ask` | yes | after you approve each one on the machine | no |
-| `trusted` | yes | yes | no |
-| `full` | yes | yes | yes |
+| level | screen, list, stat, read (inside the folders) | run programs, write files (inside the folders), mouse and keyboard | outside the folders |
+|-----------|----------------------------------------------|--------------------------------------------------------------------|---------------------|
+| `observe` | yes | no (`nack not_allowed`) | no |
+| `ask` | yes | after you approve each one on the machine | no |
+| `trusted` | yes | yes | no |
+| `full` | yes | yes | yes |
`observe` is the default. Folders are `roots` (default: the home folder of the account running the Node); a path is
`observe` is the default. Folders are `roots` (default: the home folder of the account running the Node); a path is
checked after symlinks are resolved, so a link out of a folder is outside it.
checked after symlinks are resolved, so a link out of a folder is outside it.
@@ -48,7 +48,7 @@ When the Node runs in your graphical session, a waiting request also shows a des
@@ -48,7 +48,7 @@ When the Node runs in your graphical session, a waiting request also shows a des
that a request is waiting on the machine.
that a request is waiting on the machine.
**Kill switch:** `openvibe-node stop` (and an owner's e-stop) drops the plugin to `observe`: what runs is killed, what
**Kill switch:** `openvibe-node stop` (and an owner's e-stop) drops the plugin to `observe`: what runs is killed, what
-waits is denied, and the core also refuses `exec` and file writes until `openvibe-node resume`. `halt` kills what runs.
+waits is denied, and the core also refuses `exec`, file writes and `input` until `openvibe-node resume`. `halt` kills what runs.
## What it does
## What it does
@@ -58,12 +58,21 @@ waits is denied, and the core also refuses `exec` and file writes until `openvib
@@ -58,12 +58,21 @@ waits is denied, and the core also refuses `exec` and file writes until `openvib
| `fs` | `op` `list` / `stat` / `read` (`offset`, `length`) / `write` (`content_b64`, `mode` `create`, `overwrite` or `append`) / `mkdir`, and an absolute `path` | `entries` (≤ 1000), the stat, `content_b64` (≤ `max_read_bytes`), `written`, `created` |
| `fs` | `op` `list` / `stat` / `read` (`offset`, `length`) / `write` (`content_b64`, `mode` `create`, `overwrite` or `append`) / `mkdir`, and an absolute `path` | `entries` (≤ 1000), the stat, `content_b64` (≤ `max_read_bytes`), `written`, `created` |
| `probe` | `target`: `http(s)://…` (one GET, no redirect followed) or `tcp://host:port`; `timeout_s` | `status`, `content_type`, `etag`, `last_modified`, `body` (≤ 64 KB), `ms`; or `open`, `ms` |
| `probe` | `target`: `http(s)://…` (one GET, no redirect followed) or `tcp://host:port`; `timeout_s` | `status`, `content_type`, `etag`, `last_modified`, `body` (≤ 64 KB), `ms`; or `open`, `ms` |
| `screen` | `max_width` (default 1280) | `format` (`jpeg` with Pillow, else `png` ≤ 300 KB), `width`, `height`, `content_b64` |
| `screen` | `max_width` (default 1280) | `format` (`jpeg` with Pillow, else `png` ≤ 300 KB), `width`, `height`, `content_b64` |
+| `input` | `action`: `move`, `click`, `double_click`, `right_click` (`x`, `y`), `scroll` (`dy` clicks, ±20), `type` (`text`, ≤ 2000), `key` (`keys`: one key or chord, `ctrl+s`, `Return`); `width`, `height`: the picture `x`, `y` are measured in | `done`, `action`, and the real `x`, `y` it used |
A program runs in its own process group with the Node account's rights, at most 2 at a time, and is killed at
A program runs in its own process group with the Node account's rights, at most 2 at a time, and is killed at
`timeout_s` (the owner's `timeout_s` is the cap). A write with `overwrite` replaces the file atomically. Screens come
`timeout_s` (the owner's `timeout_s` is the cap). A write with `overwrite` replaces the file atomically. Screens come
from `screencapture` (macOS), `grim` (Wayland) or `import`, `scrot` or `gnome-screenshot` (X11), so the Node must run in
from `screencapture` (macOS), `grim` (Wayland) or `import`, `scrot` or `gnome-screenshot` (X11), so the Node must run in
your graphical session for `screen`; without one the plugin simply does not offer it.
your graphical session for `screen`; without one the plugin simply does not offer it.
+**Mouse and keyboard** (`input`) is an acting command like running a program: refused at `observe`, approved one by one
+at `ask` (the approval says exactly what it would do: "click at 640, 360 of a 1280×720 picture of the screen",
+"type 12 characters: 'hello there'", "press ctrl+s"). Coordinates are in the space of the screen picture the agent last
+saw (`width` × `height`), scaled to your real screen. It uses `xdotool` on an X11 session and `cliclick` on macOS
+(`brew install cliclick`; no scrolling there); Wayland has no general input tool yet, so it is not offered there. Typed
+text and keys are passed to the tool as arguments, never through a shell, and `keys` takes only letters, digits, `+`,
+`-` and `_`.
+
## Config
## Config
| key | default | meaning |
| key | default | meaning |
@@ -77,8 +86,9 @@ your graphical session for `screen`; without one the plugin simply does not offe
@@ -77,8 +86,9 @@ your graphical session for `screen`; without one the plugin simply does not offe
| `approval_timeout_s` | 120 (≤ 600) | how long a request waits for you |
| `approval_timeout_s` | 120 (≤ 600) | how long a request waits for you |
| `notify` | `true` | a desktop notification when a request waits (graphical session only) |
| `notify` | `true` | a desktop notification when a request waits (graphical session only) |
| `probe` | `true` | `false`: no probes (OpenVibe.Watch's `node` source) |
| `probe` | `true` | `false`: no probes (OpenVibe.Watch's `node` source) |
+| `allow_input` | `true` | `false`: never use the mouse or keyboard (no `input` capability) |
-`describe` reports `terminal`, `files` and `screen` with the level and folders, so Bot shows what the machine allows;
+`describe` reports `terminal`, `files`, `screen` and `input` with the level and folders, so Bot shows what the machine allows;
telemetry carries the sensors `computer_level` (observe while a stop is latched), `computer_waiting` and
telemetry carries the sensors `computer_level` (observe while a stop is latched), `computer_waiting` and
`computer_running`.
`computer_running`.
Every request is logged by the Node (`journalctl -u openvibe-node`), with what it would do, approved or denied.
Every request is logged by the Node (`journalctl -u openvibe-node`), with what it would do, approved or denied.
@@ -218,6 +218,9 @@ func TestComputerLocalStopLeavesObserveOnly(t *testing.T) {
@@ -218,6 +218,9 @@ func TestComputerLocalStopLeavesObserveOnly(t *testing.T) {
if nk := nackOf(t, e.computerCmd("s2", protocol.KindFS, map[string]any{"op": "write", "path": filepath.Join(work, "x"), "content_b64": "eA=="}, 5*time.Second)); nk.FaultCode != protocol.FaultLocalStop {
if nk := nackOf(t, e.computerCmd("s2", protocol.KindFS, map[string]any{"op": "write", "path": filepath.Join(work, "x"), "content_b64": "eA=="}, 5*time.Second)); nk.FaultCode != protocol.FaultLocalStop {
t.Fatalf("write while stopped: %+v", nk)
t.Fatalf("write while stopped: %+v", nk)
}
}
+ if nk := nackOf(t, e.computerCmd("s2i", protocol.KindInput, map[string]any{"action": "key", "keys": "Return"}, 5*time.Second)); nk.FaultCode != protocol.FaultLocalStop {
+ t.Fatalf("input while stopped: %+v", nk)
+ }
if d := dataOf(t, e.computerCmd("s3", protocol.KindFS, map[string]any{"op": "stat", "path": work}, 5*time.Second)); d["type"] != "dir" {
if d := dataOf(t, e.computerCmd("s3", protocol.KindFS, map[string]any{"op": "stat", "path": work}, 5*time.Second)); d["type"] != "dir" {
t.Fatalf("observing while stopped: %v", d)
t.Fatalf("observing while stopped: %v", d)
}
}
@@ -21,6 +21,7 @@ var capabilityFor = map[string]string{
@@ -21,6 +21,7 @@ var capabilityFor = map[string]string{
protocol.KindFS: "files",
protocol.KindFS: "files",
protocol.KindScreen: "screen",
protocol.KindScreen: "screen",
protocol.KindProbe: "probe",
protocol.KindProbe: "probe",
+ protocol.KindInput: "input",
}
}
// Manager supervises every configured plugin.
// Manager supervises every configured plugin.
@@ -164,13 +164,16 @@ const (
@@ -164,13 +164,16 @@ const (
// KindProbe is a read-only check of something on the machine's own network (one GET or a TCP connect to a private,
// KindProbe is a read-only check of something on the machine's own network (one GET or a TCP connect to a private,
// loopback or link-local address): OpenVibe.Watch's node carrier.
// loopback or link-local address): OpenVibe.Watch's node carrier.
KindProbe = "probe"
KindProbe = "probe"
+ // KindInput moves the mouse, clicks, scrolls, types or presses keys on the machine's screen (contracts 0.171.0):
+ // an acting command, so the owner's level decides and a latched stop refuses it.
+ KindInput = "input"
)
)
// Kinds lists every command kind the Node understands.
// Kinds lists every command kind the Node understands.
-var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe}
+var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput}
// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.
// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.
-var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true}
+var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true}
const (
const (
// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps
// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps
@@ -182,11 +185,12 @@ const (
@@ -182,11 +185,12 @@ const (
MaxAckData = 512 * 1024
MaxAckData = 512 * 1024
)
)
-// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file): those are
-// refused while the local stop or an e-stop is latched. Capturing the screen and reading files are not.
+// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file, uses the mouse
+// or keyboard): those are refused while the local stop or an e-stop is latched. Capturing the screen and reading files
+// are not.
func ComputerActs(kind string, value json.RawMessage) bool {
func ComputerActs(kind string, value json.RawMessage) bool {
switch kind {
switch kind {
- case KindExec:
+ case KindExec, KindInput:
return true
return true
case KindFS:
case KindFS:
var v struct {
var v struct {
@@ -1,12 +1,13 @@
@@ -1,12 +1,13 @@
-"""OpenVibe Node computer plugin: the machine's terminal, files and screen for its owner's agents (plan T14/T17).
+"""OpenVibe Node computer plugin: the machine's terminal, files, screen and input for its owner's agents (plan T14/T17).
Off unless its owner lists it in the Node's config. The owner sets the **control level** on the machine itself; a cloud
Off unless its owner lists it in the Node's config. The owner sets the **control level** on the machine itself; a cloud
command can never raise it:
command can never raise it:
* ``observe`` (the default): capture the screen; list, stat and read files inside the roots.
* ``observe`` (the default): capture the screen; list, stat and read files inside the roots.
-* ``ask``: also run programs and write files inside the roots, each only after the owner approves it on the machine
+* ``ask``: also run programs, write files inside the roots and use the mouse and keyboard, each only after the owner
+ approves it on the machine
(``openvibe-node approvals``, ``approve <id>``, ``deny <id>``); unanswered requests expire.
(``openvibe-node approvals``, ``approve <id>``, ``deny <id>``); unanswered requests expire.
-* ``trusted``: run programs and write files inside the roots without asking.
+* ``trusted``: run programs, write files inside the roots and use the mouse and keyboard without asking.
* ``full``: as trusted, anywhere the account running the Node can reach.
* ``full``: as trusted, anywhere the account running the Node can reach.
The local kill switch (``openvibe-node stop``) and a remote e-stop drop the plugin to observe, kill what is running
The local kill switch (``openvibe-node stop``) and a remote e-stop drop the plugin to observe, kill what is running
@@ -19,6 +20,12 @@ Command kinds and their answers (the ack's ``data``):
@@ -19,6 +20,12 @@ Command kinds and their answers (the ack's ``data``):
* ``fs`` ``{"op": "list"|"stat"|"read"|"write"|"mkdir", "path", …}``: ``read`` takes ``offset``/``length`` and answers
* ``fs`` ``{"op": "list"|"stat"|"read"|"write"|"mkdir", "path", …}``: ``read`` takes ``offset``/``length`` and answers
``content_b64``; ``write`` takes ``content_b64`` and ``mode`` (``create``, ``overwrite`` or ``append``).
``content_b64``; ``write`` takes ``content_b64`` and ``mode`` (``create``, ``overwrite`` or ``append``).
* ``screen`` ``{"max_width"?}`` → ``{format, width, height, content_b64}`` (JPEG through Pillow when it is installed).
* ``screen`` ``{"max_width"?}`` → ``{format, width, height, content_b64}`` (JPEG through Pillow when it is installed).
+* ``input`` ``{"action", "x"?, "y"?, "width"?, "height"?, "dy"?, "text"?, "keys"?}`` → ``{done, action, x?, y?}``: move,
+ click, double_click or right_click at a point, scroll by ``dy`` clicks, type ``text`` into whatever has focus, or press
+ one key or chord (``keys``, for example ``ctrl+s``). ``x``/``y`` are in the space of a screen picture ``width`` ×
+ ``height`` (the ``screen`` answer) and are scaled to the real screen. An acting command: refused at observe, approved
+ one by one at ask. xdotool on X11, cliclick on macOS (no scroll there); none on Wayland yet. Config
+ ``allow_input: false`` turns it off.
* ``probe`` ``{"target", "timeout_s"?}`` → for ``http(s)://…`` one GET, no redirect followed: ``{status, content_type,
* ``probe`` ``{"target", "timeout_s"?}`` → for ``http(s)://…`` one GET, no redirect followed: ``{status, content_type,
etag, last_modified, body, truncated, ms}``; for ``tcp://host:port``: ``{open, ms, error?}``. Read-only, so it runs at
etag, last_modified, body, truncated, ms}``; for ``tcp://host:port``: ``{open, ms, error?}``. Read-only, so it runs at
every level, and only towards the machine's own network: every address the name resolves to must be private,
every level, and only towards the machine's own network: every address the name resolves to must be private,
@@ -57,6 +64,17 @@ MAX_WAITING = 8
@@ -57,6 +64,17 @@ MAX_WAITING = 8
MAX_OUTPUT_CAP = 192 * 1024
MAX_OUTPUT_CAP = 192 * 1024
MAX_READ_CAP = 256 * 1024
MAX_READ_CAP = 256 * 1024
SCREEN_MAX_BYTES = 300 * 1024
SCREEN_MAX_BYTES = 300 * 1024
+INPUT_ACTIONS = frozenset({"move", "click", "double_click", "right_click", "scroll", "type", "key"})
+POINT_ACTIONS = frozenset({"move", "click", "double_click", "right_click"})
+MAX_TYPE = 2000
+KEYS_OK = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_+-")
+MODIFIERS = {"ctrl": "ctrl", "control": "ctrl", "alt": "alt", "option": "alt", "shift": "shift", "cmd": "cmd",
+ "command": "cmd", "super": "super", "meta": "super", "win": "super"}
+# xdotool key names → cliclick's kp: names (macOS); a single character is typed with t:.
+CLICLICK_KEYS = {"return": "return", "enter": "enter", "escape": "esc", "esc": "esc", "tab": "tab", "backspace": "delete",
+ "delete": "fwd-delete", "space": "space", "up": "arrow-up", "down": "arrow-down", "left": "arrow-left",
+ "right": "arrow-right", "home": "home", "end": "end", "page_up": "page-up", "prior": "page-up",
+ "page_down": "page-down", "next": "page-down"}
def _int(v, name, lo, hi, default):
def _int(v, name, lo, hi, default):
@@ -69,13 +87,14 @@ def _int(v, name, lo, hi, default):
@@ -69,13 +87,14 @@ def _int(v, name, lo, hi, default):
class Computer(Plugin):
class Computer(Plugin):
driver = "computer"
driver = "computer"
- version = "0.3.0"
+ version = "0.4.0"
motion_kinds = frozenset()
motion_kinds = frozenset()
telemetry_interval_s = 2.0
telemetry_interval_s = 2.0
- def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None):
+ def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None, screen_size=None):
self.which = which
self.which = which
self.popen = popen
self.popen = popen
+ self.screen_size = screen_size or self._screen_size
self.clock = clock
self.clock = clock
self.env = os.environ if env is None else env
self.env = os.environ if env is None else env
self.platform = platform or sys.platform
self.platform = platform or sys.platform
@@ -86,6 +105,7 @@ class Computer(Plugin):
@@ -86,6 +105,7 @@ class Computer(Plugin):
self.max_read = 256 * 1024
self.max_read = 256 * 1024
self.allow_exec = True
self.allow_exec = True
self.allow_probe = True
self.allow_probe = True
+ self.allow_input = True
self.notify = True
self.notify = True
self.approval_timeout_s = 120
self.approval_timeout_s = 120
self.pending = {} # command id -> (run, expires_at, summary)
self.pending = {} # command id -> (run, expires_at, summary)
@@ -111,6 +131,7 @@ class Computer(Plugin):
@@ -111,6 +131,7 @@ class Computer(Plugin):
self.approval_timeout_s = _int(config.get("approval_timeout_s"), "approval_timeout_s", 10, 600, 120)
self.approval_timeout_s = _int(config.get("approval_timeout_s"), "approval_timeout_s", 10, 600, 120)
self.allow_exec = config.get("allow_exec", True) is not False
self.allow_exec = config.get("allow_exec", True) is not False
self.allow_probe = config.get("probe", True) is not False
self.allow_probe = config.get("probe", True) is not False
+ self.allow_input = config.get("allow_input", True) is not False
self.notify = config.get("notify", True) is not False
self.notify = config.get("notify", True) is not False
def describe(self, config):
def describe(self, config):
@@ -126,6 +147,9 @@ class Computer(Plugin):
@@ -126,6 +147,9 @@ class Computer(Plugin):
caps["screen"] = {"level": self.level, "tool": tool[0]}
caps["screen"] = {"level": self.level, "tool": tool[0]}
if self.allow_probe:
if self.allow_probe:
caps["probe"] = {"lan_only": True, "schemes": ["http", "https", "tcp"]}
caps["probe"] = {"lan_only": True, "schemes": ["http", "https", "tcp"]}
+ itool = self._input_tool() if self.allow_input else None
+ if itool:
+ caps["input"] = {"level": self.level, "tool": itool, "scroll": itool == "xdotool"}
return caps
return caps
def setup(self, config):
def setup(self, config):
@@ -221,6 +245,14 @@ class Computer(Plugin):
@@ -221,6 +245,14 @@ class Computer(Plugin):
raise Fault("unsupported", "this machine does not allow probes")
raise Fault("unsupported", "this machine does not allow probes")
target, timeout = self._probe_target(v)
target, timeout = self._probe_target(v)
return self._start(cmd, lambda: self._probe(target, timeout))
return self._start(cmd, lambda: self._probe(target, timeout))
+ if cmd.kind == "input":
+ if not self.allow_input:
+ raise Fault("unsupported", "this machine does not allow mouse and keyboard input")
+ tool = self._input_tool()
+ if not tool:
+ raise Fault("unsupported", "no input tool on this machine (xdotool on X11, cliclick on macOS)")
+ run, summary = self._input_op(tool, v)
+ return self._act(cmd, level, run, summary, threaded=True)
if cmd.kind == "exec":
if cmd.kind == "exec":
if not self.allow_exec:
if not self.allow_exec:
raise Fault("unsupported", "this machine does not allow running programs")
raise Fault("unsupported", "this machine does not allow running programs")
@@ -572,6 +604,138 @@ class Computer(Plugin):
@@ -572,6 +604,138 @@ class Computer(Plugin):
verb = {"create": "create", "overwrite": "overwrite", "append": "append to"}[mode]
verb = {"create": "create", "overwrite": "overwrite", "append": "append to"}[mode]
return run, "%s %s (%d bytes)" % (verb, path, len(data))
return run, "%s %s (%d bytes)" % (verb, path, len(data))
+ # ---- input -----------------------------------------------------------------------------------------------------
+ def _input_tool(self):
+ """xdotool on an X11 session, cliclick on macOS, else None (Wayland has no general input tool yet)."""
+ if self.platform == "darwin" and self.which("cliclick"):
+ return "cliclick"
+ if self.platform.startswith("linux") and self.env.get("DISPLAY") and not self.env.get("WAYLAND_DISPLAY") and self.which("xdotool"):
+ return "xdotool"
+ return None
+
+ def _input_op(self, tool, v):
+ """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask)."""
+ action = v.get("action")
+ if action not in INPUT_ACTIONS:
+ raise Fault("bad_value", "action is one of %s" % ", ".join(sorted(INPUT_ACTIONS)))
+ point = None
+ if action in POINT_ACTIONS or (action == "scroll" and v.get("x") is not None):
+ x = _int(v.get("x"), "x", 0, 16384, None)
+ y = _int(v.get("y"), "y", 0, 16384, None)
+ if x is None or y is None:
+ raise Fault("bad_value", "%s needs x and y" % action)
+ w = _int(v.get("width"), "width", 1, 16384, None)
+ h = _int(v.get("height"), "height", 1, 16384, None)
+ if (w is None) != (h is None):
+ raise Fault("bad_value", "width and height come together")
+ point = (x, y, w, h)
+ if action == "scroll":
+ dy = _int(v.get("dy"), "dy", -20, 20, None)
+ if not dy:
+ raise Fault("bad_value", "scroll needs dy (clicks, -20 to 20, not 0)")
+ if tool != "xdotool":
+ raise Fault("unsupported", "scrolling needs xdotool (Linux X11)")
+ if action == "type":
+ text = v.get("text")
+ if not isinstance(text, str) or not text or len(text) > MAX_TYPE or "\x00" in text:
+ raise Fault("bad_value", "text is 1 to %d characters" % MAX_TYPE)
+ if action == "key":
+ keys = v.get("keys")
+ if not isinstance(keys, str) or not keys or len(keys) > 64 or not set(keys) <= KEYS_OK or keys.startswith("-"):
+ raise Fault("bad_value", "keys is one key or chord, for example Return or ctrl+s")
+
+ def run():
+ at = self._scale(tool, point) if point else None
+ argv = self._input_argv(tool, action, at, v)
+ proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True)
+ try:
+ _, err = proc.communicate(timeout=30)
+ except subprocess.TimeoutExpired:
+ self._kill(proc)
+ raise Fault("hardware", "%s did not finish in 30 s" % tool)
+ if proc.returncode != 0:
+ raise Fault("hardware", "%s failed: %s" % (tool, (err or b"").decode("utf-8", "replace")[:200]))
+ out = {"done": True, "action": action}
+ if at:
+ out["x"], out["y"] = at
+ return out
+
+ if action == "type":
+ text = v["text"]
+ shown = text if len(text) <= 120 else text[:120] + "…"
+ summary = "type %d characters: %r" % (len(text), shown)
+ elif action == "key":
+ summary = "press %s" % v["keys"]
+ elif action == "scroll":
+ summary = "scroll %s %d" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]))
+ else:
+ x, y, w, h = point
+ summary = "%s at %d, %d%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "")
+ return run, summary
+
+ def _screen_size(self, tool):
+ """The screen's size in the units the input tool uses: pixels for xdotool, points for cliclick (a Retina
+ screen's picture is twice as many pixels)."""
+ if tool == "xdotool":
+ out = subprocess.run(["xdotool", "getdisplaygeometry"], capture_output=True, text=True, timeout=10, env=self._tool_env())
+ parts = out.stdout.split()
+ else:
+ out = subprocess.run(["osascript", "-e", "tell application \"Finder\" to get bounds of window of desktop"],
+ capture_output=True, text=True, timeout=10)
+ parts = [p.strip() for p in out.stdout.split(",")][2:4]
+ if out.returncode != 0 or len(parts) != 2 or not all(p.isdigit() for p in parts):
+ raise Fault("hardware", "could not read the screen size")
+ return int(parts[0]), int(parts[1])
+
+ def _scale(self, tool, point):
+ x, y, w, h = point
+ if not w:
+ return x, y
+ sw, sh = self.screen_size(tool)
+ return min(sw - 1, round(x * sw / w)), min(sh - 1, round(y * sh / h))
+
+ def _tool_env(self):
+ env = dict(os.environ)
+ env.update({k: v for k, v in self.env.items() if k in ("DISPLAY", "XAUTHORITY")})
+ return env
+
+ @staticmethod
+ def _input_argv(tool, action, at, v):
+ if tool == "xdotool":
+ move = ["mousemove", "--sync", str(at[0]), str(at[1])] if at else []
+ if action == "move":
+ return ["xdotool"] + move
+ if action in ("click", "double_click", "right_click"):
+ button = "3" if action == "right_click" else "1"
+ repeat = ["--repeat", "2"] if action == "double_click" else []
+ return ["xdotool"] + move + ["click"] + repeat + [button]
+ if action == "scroll":
+ return ["xdotool"] + move + ["click", "--repeat", str(abs(v["dy"])), "4" if v["dy"] < 0 else "5"]
+ if action == "type":
+ return ["xdotool", "type", "--delay", "12", "--", v["text"]]
+ return ["xdotool", "key", "--", v["keys"]]
+ # cliclick (macOS): c: click, dc: double, rc: right, m: move, t: type, kd/ku: modifier down/up, kp: a named key.
+ if action in POINT_ACTIONS:
+ verb = {"move": "m", "click": "c", "double_click": "dc", "right_click": "rc"}[action]
+ return ["cliclick", "%s:%d,%d" % (verb, at[0], at[1])]
+ if action == "type":
+ return ["cliclick", "t:" + v["text"]]
+ parts = v["keys"].split("+")
+ mods = [MODIFIERS.get(p.lower()) for p in parts[:-1]]
+ if None in mods:
+ raise Fault("bad_value", "unknown modifier in %s" % v["keys"])
+ key = parts[-1]
+ if len(key) == 1:
+ press = ["t:" + key]
+ elif key.lower() in CLICLICK_KEYS:
+ press = ["kp:" + CLICLICK_KEYS[key.lower()]]
+ elif key.lower().startswith("f") and key[1:].isdigit() and 1 <= int(key[1:]) <= 16:
+ press = ["kp:" + key.lower()]
+ else:
+ raise Fault("unsupported", "the key %s is not supported on macOS" % key)
+ mods = ["cmd" if m == "super" else m for m in mods]
+ return ["cliclick"] + ["kd:" + m for m in mods] + press + ["ku:" + m for m in reversed(mods)]
+
# ---- screen ----------------------------------------------------------------------------------------------------
# ---- screen ----------------------------------------------------------------------------------------------------
def _screen_tool(self):
def _screen_tool(self):
"""(name, argv builder) for this machine's screen capture, or None."""
"""(name, argv builder) for this machine's screen capture, or None."""
@@ -304,5 +304,92 @@ class ComputerTest(unittest.TestCase):
@@ -304,5 +304,92 @@ class ComputerTest(unittest.TestCase):
self.assertEqual(base64.b64decode(d["content_b64"])[:2], b"\xff\xd8")
self.assertEqual(base64.b64decode(d["content_b64"])[:2], b"\xff\xd8")
+ # ---- input ---------------------------------------------------------------------------------------------------
+ def boot_input(self, level, platform="linux", tool="xdotool", size=(2560, 1440)):
+ self.argvs = []
+
+ class Proc:
+ returncode = 0
+ pid = 1
+
+ def communicate(self_, timeout=None):
+ return b"", b""
+
+ def popen(argv, **kw):
+ self.argvs.append(argv)
+ return Proc()
+
+ self.dir = tempfile.mkdtemp(prefix="ov-computer-")
+ self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True))
+ self.out = Out()
+ env = {"DISPLAY": ":0"} if platform == "linux" else {}
+ self.plugin = Computer(which=lambda n: "/usr/bin/" + n if n == tool else None, popen=popen, env=env,
+ platform=platform, screen_size=lambda t: size)
+ self.rt = Runtime(self.plugin, infile=io.StringIO(""), outfile=self.out)
+ self.rt.dispatch({"op": "hello", "config": {"level": level, "roots": [self.dir], "notify": False}})
+ self.n = 0
+ return self.out.msgs()
+
+ def test_input_tool_detection(self):
+ xdo = lambda n: "/usr/bin/xdotool" if n == "xdotool" else None
+ self.assertEqual(Computer(which=xdo, env={"DISPLAY": ":0"}, platform="linux")._input_tool(), "xdotool")
+ self.assertIsNone(Computer(which=xdo, env={"DISPLAY": ":0", "WAYLAND_DISPLAY": "wayland-0"}, platform="linux")._input_tool(), "not on Wayland")
+ self.assertIsNone(Computer(which=xdo, env={}, platform="linux")._input_tool(), "no session")
+ self.assertEqual(Computer(which=lambda n: "/opt/homebrew/bin/cliclick" if n == "cliclick" else None, env={}, platform="darwin")._input_tool(), "cliclick")
+ caps = self.boot_input("trusted")[0]["capabilities"]
+ self.assertEqual(caps["input"], {"level": "trusted", "tool": "xdotool", "scroll": True})
+
+ def test_input_is_an_acting_command(self):
+ self.boot_input("observe")
+ r = reply(self.out, self.cmd("input", {"action": "click", "x": 10, "y": 10}))
+ self.assertEqual((r["op"], r["fault_code"]), ("nack", "not_allowed"))
+ self.boot_input("ask")
+ cid = self.cmd("input", {"action": "click", "x": 640, "y": 360, "width": 1280, "height": 720})
+ ev = self.out.wait_for(lambda m: m.get("op") == "event" and m.get("name") == "approval")
+ self.assertEqual((ev["kind"], ev["summary"]), ("input", "click at 640, 360 of a 1280\u00d7720 picture of the screen"))
+ self.assertEqual(self.argvs, [], "nothing touches the screen before the owner approves")
+ self.plugin.on_approval(cid, True)
+ self.assertEqual(reply(self.out, cid)["op"], "ack")
+
+ def test_input_scales_the_picture_to_the_screen(self):
+ self.boot_input("trusted")
+ r = reply(self.out, self.cmd("input", {"action": "click", "x": 640, "y": 360, "width": 1280, "height": 720}))
+ self.assertEqual(r["data"], {"done": True, "action": "click", "x": 1280, "y": 720})
+ self.assertEqual(self.argvs[-1], ["xdotool", "mousemove", "--sync", "1280", "720", "click", "1"])
+ reply(self.out, self.cmd("input", {"action": "double_click", "x": 5, "y": 5}))
+ self.assertEqual(self.argvs[-1], ["xdotool", "mousemove", "--sync", "5", "5", "click", "--repeat", "2", "1"], "no picture: screen pixels")
+ reply(self.out, self.cmd("input", {"action": "scroll", "dy": -3}))
+ self.assertEqual(self.argvs[-1], ["xdotool", "click", "--repeat", "3", "4"])
+ reply(self.out, self.cmd("input", {"action": "type", "text": "-hello"}))
+ self.assertEqual(self.argvs[-1], ["xdotool", "type", "--delay", "12", "--", "-hello"], "text is never an option")
+ reply(self.out, self.cmd("input", {"action": "key", "keys": "ctrl+s"}))
+ self.assertEqual(self.argvs[-1], ["xdotool", "key", "--", "ctrl+s"])
+
+ def test_input_refuses_what_it_should_not_send(self):
+ self.boot_input("trusted")
+ before = len(self.argvs)
+ for value, code in [({"action": "key", "keys": "ctrl+s; rm -rf ~"}, "bad_value"),
+ ({"action": "key", "keys": "--help"}, "bad_value"),
+ ({"action": "drag"}, "bad_value"),
+ ({"action": "click"}, "bad_value"),
+ ({"action": "click", "x": 1, "y": 1, "width": 100}, "bad_value"),
+ ({"action": "type", "text": "x" * 2001}, "bad_value"),
+ ({"action": "scroll", "dy": 0}, "bad_value")]:
+ r = reply(self.out, self.cmd("input", value))
+ self.assertEqual((r["op"], r["fault_code"]), ("nack", code), value)
+ self.assertEqual(len(self.argvs), before)
+
+ def test_input_on_macos_uses_cliclick(self):
+ self.boot_input("trusted", platform="darwin", tool="cliclick", size=(1440, 900))
+ reply(self.out, self.cmd("input", {"action": "right_click", "x": 640, "y": 400, "width": 1280, "height": 800}))
+ self.assertEqual(self.argvs[-1], ["cliclick", "rc:720,450"])
+ reply(self.out, self.cmd("input", {"action": "key", "keys": "cmd+shift+s"}))
+ self.assertEqual(self.argvs[-1], ["cliclick", "kd:cmd", "kd:shift", "t:s", "ku:shift", "ku:cmd"])
+ reply(self.out, self.cmd("input", {"action": "key", "keys": "Return"}))
+ self.assertEqual(self.argvs[-1], ["cliclick", "kp:return"])
+ r = reply(self.out, self.cmd("input", {"action": "scroll", "dy": 2}))
+ self.assertEqual((r["op"], r["fault_code"]), ("nack", "unsupported"))
+
+
if __name__ == "__main__":
if __name__ == "__main__":
unittest.main()
unittest.main()