openvibers/openvibe.link Commits
A private desk for agents (desk start/stop/status): an Xvfb display with its own bus and window manager; screen, live screen, input and the browser act there with desk: private, never on the owner's screen (#57)
Changes
@@@ -124,6 +124,25 @@ text and keys are passed to the tool as arguments, never through a shell, and `k- **Closing.** It closes after `browser_idle_s` without a command and at the kill switch.- **Platforms.** Linux and macOS; not offered on Windows yet.++### The private desk (`desk`, Node v0.15.0)++++An agent can work on a desk of its own instead of your screen (plan track K: Vibe's agent mode):++- **What it is.** An Xvfb display on your machine with a window manager (metacity, openbox, xfwm4 or fluxbox, whichever++ is installed) and a D-Bus session of its own, so apps opened there open there, not on your screen.++- **Using it.** `desk` `{"action": "start", "width"?, "height"?}` (1280×800 by default), `stop` or `status`. Then++ `screen`, `screen_stream`, `input` and `browser` take `"desk": "private"`: screenshots and the live screen of the desk,++ the mouse and keyboard on the desk (points scale to the desk's own size), and the agent's browser shown on the desk++ in a profile of its own (`browser_profile` + `-desk`). Without `desk` (or with `main`) they act on your own screen,++ as before.++- **Levels.** Starting and stopping a desk are acting commands (refused at `observe`, approved one by one at `ask`);++ `status` runs at every level. What runs on the desk keeps its own rules: input and the browser still act.++- **Watching and taking over.** Start the live screen with `"desk": "private"` (your desktop is not told, since nothing of++ yours is shown) and drive it the same way.++- **It ends** on `stop`, and at once on `openvibe-node stop` or an e-stop: every process the desk started is ended by++ handle, never by name.++- **Needs** Linux with `Xvfb` (Debian/Ubuntu: `apt install xvfb`), `xdotool` for input and ImageMagick `import` or `scrot`++ for screenshots. Not offered elsewhere; `allow_desk: false` turns it off.++## Config| key | default | meaning |@@@ -138,6 +157,7 @@ text and keys are passed to the tool as arguments, never through a shell, and `k| `notify` | `true` | the plugin's own prompt when a request waits and no tray shows it (graphical session only) || `probe` | `true` | `false`: no probes (OpenVibe.Watch's `node` source) || `allow_input` | `true` | `false`: never use the mouse or keyboard (no `input` capability) |++| `allow_desk` | `true` | `false`: never start a private desk (no `desk` capability) || `allow_browser` | `true` | `false`: no browser (no `browser` capability) || `allow_screen_stream` | `true` | `false`: never show the screen live (`screen.stream` is false) || `browser_path` | found on PATH | the browser to drive (Chromium, Chrome, Edge or Brave) |@@@ -24,6 +24,7 @@ var capabilityFor = map[string]string{protocol.KindProbe: "probe",protocol.KindInput: "input",protocol.KindBrowser: "browser",++ protocol.KindDesk: "desk",}// Manager supervises every configured plugin.@@@ -175,13 +175,17 @@ const (// machine's private OpenRestream stream); no operator sends them. The computer plugin captures the frames, the// core publishes them; a latched stop ends it and refuses a start.KindScreenStream = "screen_stream"++ // KindDesk starts, stops or reports a private desk on the machine (contracts 0.214.0): {"action": "start"|"stop"|++ // "status", "width"?, "height"?}, an Xvfb display of its own where an agent works without touching the owner's++ // screen; screen, screen_stream, input and browser take "desk": "private" to act there. Starting and stopping act.++ KindDesk = "desk")// Kinds lists every command kind the Node understands.--var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream}++var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream, KindDesk}// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.--var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true}++var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true, KindDesk: true}const (// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps@@@ -194,8 +198,8 @@ const ()// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file, uses the mouse--// or keyboard, drives the browser): those are refused while the local stop or an e-stop is latched. Capturing the--// screen, reading files and reading or closing the browser are not.++// or keyboard, drives the browser, starts or stops the private desk): those are refused while the local stop or an++// e-stop is latched. Capturing the screen, reading files, reading or closing the browser and the desk's status are not.func ComputerActs(kind string, value json.RawMessage) bool {switch kind {case KindExec, KindInput:@@@ -212,6 +216,12 @@ func ComputerActs(kind string, value json.RawMessage) bool {}_ = json.Unmarshal(value, &v)return v.Action != "read" && v.Action != "screenshot" && v.Action != "close"++ case KindDesk:++ var v struct {++ Action string `json:"action"`++ }++ _ = json.Unmarshal(value, &v)++ return v.Action != "status"}return false}@@@ -396,6 +396,18 @@ func TestJobUndecodableBody(t *testing.T) {}}++func TestComputerActsDesk(t *testing.T) {++ cases := map[string]bool{"start": true, "stop": true, "status": false}++ for action, acts := range cases {++ if got := ComputerActs(KindDesk, json.RawMessage(`{"action":"`+action+`"}`)); got != acts {++ t.Errorf("desk %s acts = %v, want %v", action, got, acts)++ }++ }++ if !ComputerKinds[KindDesk] {++ t.Error("desk is a computer kind (long deadline, no clamping)")++ }++}++func TestComputerActsBrowser(t *testing.T) {want := map[string]bool{"open": true, "click": true, "fill": true, "press": true, "back": true, "read": false, "screenshot": false, "close": false}for action, acts := range want {@@@ -40,6 +40,12 @@ Command kinds and their answers (the ack's ``data``):way; ``screenshot`` → ``{format, width, height, content_b64}``; ``close``. Reading, screenshots and closing run at everylevel; the rest are acting commands. Below level full, ``open`` refuses an address on the machine's own network.Config ``allow_browser: false`` turns it off; ``browser_path``, ``browser_headless`` (default true).++* ``desk`` ``{"action": "start"|"stop"|"status", "width"?, "height"?}``: a private desk (``desk.py``): an Xvfb display with a++ window manager and a D-Bus session of its own, where the agent works without touching the owner's screen. ``screen``,++ ``screen_stream``, ``input`` and ``browser`` take ``"desk": "private"`` to act there (the browser shown on the desk, in a++ profile of its own, ``browser_profile`` + ``-desk``); ``main`` (the default) is the owner's screen. Starting and stopping++ act (refused at observe, approved at ask), ``status`` runs at every level, and the kill switch stops the desk. Linux++ with Xvfb; config ``allow_desk: false`` turns it off.* ``probe`` ``{"target", "timeout_s"?}`` → for ``http(s)://…`` one GET, no redirect followed: ``{status, content_type,etag, last_modified, body, truncated, ms}``; for ``tcp://host:port``: ``{open, ms, error?}``. Read-only, so it runs atevery level, and only towards the machine's own network: every address the name resolves to must be private,@@@ -67,6 +73,7 @@ import urllib.parsefrom openvibe_plugin import Deferred, Fault, Pluginfrom .browser import KEY_CODES, Browser, BrowserError, find_binary++from .desk import PrivateDeskLEVELS = ("observe", "ask", "trusted", "full")RANK = {name: i for i, name in enumerate(LEVELS)}@@@ -132,12 +139,12 @@ def _int(v, name, lo, hi, default):class Computer(Plugin):driver = "computer"-- version = "0.7.0"++ version = "0.8.0"motion_kinds = frozenset()telemetry_interval_s = 2.0def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None, screen_size=None,-- browser_factory=None, resolve=None):++ browser_factory=None, resolve=None, desk=None):self.which = whichself.popen = popenself.screen_size = screen_size or self._screen_size@@@ -174,7 +181,10 @@ class Computer(Plugin):self.stream_mu = threading.Lock()self.browser_mu = threading.Lock()self.browser_op_mu = threading.Lock()-- self.browser_factory = browser_factory or (lambda binary, profile, headless: Browser(binary, profile, headless))++ self.browser_factory = browser_factory or (lambda binary, profile, headless, env=None: Browser(binary, profile, headless, env=env))++ self.desk = desk or PrivateDesk(which=which, popen=popen)++ self.allow_desk = True++ self.desk_browser = None # the agent's browser on the private desk (its own profile, shown there, not headless)self.resolve = resolve or (lambda host: [ai[4][0] for ai in socket.getaddrinfo(host, None)])# ---- configuration -------------------------------------------------------------------------------------------@@@ -199,6 +209,7 @@ class Computer(Plugin):self.notify = config.get("notify", True) is not Falseself.allow_browser = config.get("allow_browser", True) is not Falseself.allow_screen_stream = config.get("allow_screen_stream", True) is not False++ self.allow_desk = config.get("allow_desk", True) is not Falseself.browser_path = config.get("browser_path") or Noneself.browser_profile = os.path.expanduser(config.get("browser_profile") or "~/.local/share/openvibe-node/browser")self.browser_headless = config.get("browser_headless", True) is not False@@@ -224,6 +235,9 @@ class Computer(Plugin):binary = self._browser_binary()if binary:caps["browser"] = {"level": self.level, "browser": os.path.basename(binary), "headless": self.browser_headless}++ if self.allow_desk and self.platform.startswith("linux") and self.desk.available():++ caps["desk"] = {"level": self.level, "window_manager": self.desk.window_manager(), "input": bool(self.which("xdotool")),++ "screen": bool(self._x11_tool())}return capsdef setup(self, config):@@@ -248,6 +262,7 @@ class Computer(Plugin):self._kill(p)self._close_browser()self._end_stream("stopped on the machine")++ self.desk.stop()for cid in waiting:self._close_prompt(cid)self.emit_event("approval_done", id=cid, approved=False, reason="stopped")@@@ -309,14 +324,17 @@ class Computer(Plugin):if not isinstance(v, dict):raise Fault("bad_value", "value must be an object")level = self.effective_level()++ if cmd.kind == "desk":++ return self._desk_op(cmd, v, level)++ desk_env = self._desk_env(v) if cmd.kind in ("screen", "screen_stream", "input", "browser") else Noneif cmd.kind == "screen":-- tool = self._screen_tool()++ tool = self._x11_tool() if desk_env else self._screen_tool()if not tool:raise Fault("unsupported", "no screen capture tool on this machine")width = _int(v.get("max_width"), "max_width", 320, 3840, 1280)-- return self._start(cmd, lambda: self._screen(tool, width))++ return self._start(cmd, lambda: self._screen(tool, width, env=desk_env))if cmd.kind == "screen_stream":-- return self._screen_stream(v)++ return self._screen_stream(v, env=desk_env)if cmd.kind == "fs":op = v.get("op")if op in READ_OPS:@@@ -333,13 +351,13 @@ class Computer(Plugin):if cmd.kind == "input":if not self.allow_input:raise Fault("unsupported", "this machine does not allow mouse and keyboard input")-- tool = self._input_tool()++ tool = ("xdotool" if self.which("xdotool") else None) if desk_env else self._input_tool()if not tool:raise Fault("unsupported", "no input tool on this machine (xdotool on X11, cliclick on macOS)")-- run, summary = self._input_op(tool, v)++ run, summary = self._input_op(tool, v, env=desk_env)return self._act(cmd, level, self._driving(run), summary, threaded=True)if cmd.kind == "browser":-- run, summary, acting = self._browser_op(v, level)++ run, summary, acting = self._browser_op(v, level, env=desk_env)if acting:return self._act(cmd, level, run, summary, threaded=True)return self._start(cmd, run)@@@ -350,6 +368,51 @@ class Computer(Plugin):return self._act(cmd, level, run, summary, threaded=True)raise Fault("unsupported", "kind %s not supported" % cmd.kind)++ # ---- the private desk --------------------------------------------------------------------------------------------++ def _desk_env(self, v):++ """None for the owner's own screen (desk main, the default); the private desk's environment for desk private."""++ where = v.get("desk", "main")++ if where not in ("main", "private"):++ raise Fault("bad_value", "desk is main or private")++ if where == "main":++ return None++ if not self.allow_desk:++ raise Fault("unsupported", "this machine does not allow a private desk (allow_desk is off)")++ return self.desk.env()++++ def _desk_op(self, cmd, v, level):++ action = v.get("action")++ if action == "status":++ return self.desk.status()++ if action not in ("start", "stop"):++ raise Fault("bad_value", "action is start, stop or status")++ if not self.allow_desk or not self.platform.startswith("linux"):++ raise Fault("unsupported", "this machine does not offer a private desk (Linux with Xvfb, allow_desk on)")++ if action == "start":++ width = _int(v.get("width"), "width", 640, 3840, 1280)++ height = _int(v.get("height"), "height", 480, 2160, 800)++ return self._act(cmd, level, lambda: self.desk.start(width, height),++ "start a private desk (%d×%d) where your agent works without touching your screen" % (width, height),++ threaded=True)++++ def stop():++ self._close_desk_browser()++ return self.desk.stop()++ return self._act(cmd, level, stop, "stop the private desk (and close what runs on it)", threaded=False)++++ def _close_desk_browser(self):++ b, self.desk_browser = self.desk_browser, None++ if b is not None:++ b.close()++++ def _x11_tool(self):++ """An X11 capture tool for the private desk (it is always X11, whatever the owner's session is)."""++ if self.which("import"):++ return ("import", lambda out: ["import", "-window", "root", out])++ if self.which("scrot"):++ return ("scrot", lambda out: ["scrot", "-o", out])++ return None++def _act(self, cmd, level, run, summary, threaded):if RANK[level] < RANK["ask"]:stopped = self.runtime is not None and self.runtime.estopped@@@ -828,7 +891,7 @@ class Computer(Plugin):raise Fault("not_allowed", "%s is on this machine's own network: only at level full" % u.hostname)return raw-- def _browser_op(self, v, level):++ def _browser_op(self, v, level, env=None):"""(run, summary, acting) for one browser action; nothing is opened or clicked before run()."""binary = self._browser_binary()if not binary:@@@ -854,13 +917,17 @@ class Computer(Plugin):def browser():with self.browser_mu:++ if env: # on the private desk: shown there (never headless), in a profile of its own++ if self.desk_browser is None or not self.desk_browser.alive():++ self.desk_browser = self.browser_factory(binary, self.browser_profile + "-desk", False, env=env)++ return self.desk_browserif self.browser is None or not self.browser.alive():self.browser = self.browser_factory(binary, self.browser_profile, self.browser_headless)return self.browserdef run():if action == "close":-- self._close_browser()++ self._close_desk_browser() if env else self._close_browser()return {"done": True}b = browser()try:@@@ -904,8 +971,9 @@ class Computer(Plugin):return "xdotool"return None-- def _input_op(self, tool, v):-- """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask)."""++ def _input_op(self, tool, v, env=None):++ """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask). With env,++ on the private desk (its display; points scale to the desk's own size)."""action = v.get("action")if action not in INPUT_ACTIONS:raise Fault("bad_value", "action is one of %s" % ", ".join(sorted(INPUT_ACTIONS)))@@@ -936,9 +1004,10 @@ class Computer(Plugin):raise Fault("bad_value", "keys is one key or chord, for example Return or ctrl+s")def run():-- at = self._scale(tool, point) if point else None++ at = (self._desk_scale(point) if env else self._scale(tool, point)) if point else Noneargv = self._input_argv(tool, action, at, v)-- proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True)++ proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True,++ **({"env": env} if env else {}))try:_, err = proc.communicate(timeout=30)except subprocess.TimeoutExpired:@@@ -951,19 +1020,27 @@ class Computer(Plugin):out["x"], out["y"] = atreturn out++ where = " on the private desk" if env else ""if action == "type":text = v["text"]shown = text if len(text) <= 120 else text[:120] + "…"-- summary = "type %d characters: %r" % (len(text), shown)++ summary = "type %d characters%s: %r" % (len(text), where, shown)elif action == "key":-- summary = "press %s" % v["keys"]++ summary = "press %s%s" % (v["keys"], where)elif action == "scroll":-- summary = "scroll %s %d" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]))++ summary = "scroll %s %d%s" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]), where)else:x, y, w, h = point-- summary = "%s at %d, %d%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "")++ summary = "%s at %d, %d%s%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "", where)return run, summary++ def _desk_scale(self, point):++ x, y, w, h = point++ st = self.desk.status()++ if not w or not st.get("running"):++ return x, y++ return min(st["width"] - 1, round(x * st["width"] / w)), min(st["height"] - 1, round(y * st["height"] / h))++def _screen_size(self, tool):"""The screen's size in the units the input tool uses: pixels for xdotool, points for cliclick (a Retinascreen's picture is twice as many pixels)."""@@@ -1044,12 +1121,12 @@ class Computer(Plugin):return ("gnome-screenshot", lambda out: ["gnome-screenshot", "-f", out])return None-- def _screen(self, tool, max_width):++ def _screen(self, tool, max_width, env=None):fd, out = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")os.close(fd)try:proc = self.popen(tool[1](out), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,-- stderr=subprocess.PIPE, start_new_session=True)++ stderr=subprocess.PIPE, start_new_session=True, **({"env": env} if env else {}))try:_, err = proc.communicate(timeout=15)except subprocess.TimeoutExpired:@@@ -1088,7 +1165,7 @@ class Computer(Plugin):def _stream_capable(self, tool):return self._have_pillow() or self._native_jpeg(tool, "x.jpg") is not None-- def _screen_stream(self, v):++ def _screen_stream(self, v, env=None):action = v.get("action")if action == "stop":return {"streaming": False, "stopped": self._end_stream("stopped by its owner")}@@@ -1098,7 +1175,7 @@ class Computer(Plugin):raise Fault("unsupported", "this machine does not show its screen live (allow_screen_stream is off)")if self.runtime is not None and self.runtime.estopped:raise Fault("local_stop", "stopped on the machine")-- tool = self._screen_tool()++ tool = self._x11_tool() if env else self._screen_tool()if not tool:raise Fault("unsupported", "no screen capture tool on this machine")if not self._stream_capable(tool):@@@ -1108,12 +1185,14 @@ class Computer(Plugin):self._end_stream("replaced")frames = tempfile.mkdtemp(prefix="openvibe-screen-stream-") # 0700, this account onlystream = {"sid": os.urandom(8).hex(), "until": self.clock() + seconds, "fps": fps, "tool": tool, "dir": frames,-- "stop": threading.Event()}++ "stop": threading.Event(), "env": env}with self.stream_mu:self.stream = streamthreading.Thread(target=self._stream_loop, args=(stream,), name="computer-screen-stream", daemon=True).start()-- self._notify_stream(seconds)-- return {"streaming": True, "sid": stream["sid"], "seconds": seconds, "fps": fps, "tool": tool[0]}++ if not env: # the private desk is not the owner's screen: nothing of theirs goes live++ self._notify_stream(seconds)++ return {"streaming": True, "sid": stream["sid"], "seconds": seconds, "fps": fps, "tool": tool[0],++ "desk": "private" if env else "main"}def _end_stream(self, reason):"""End the live screen, if one runs → whether one did."""@@@ -1135,7 +1214,7 @@ class Computer(Plugin):breakpath = os.path.join(stream["dir"], "frame-%d.jpg" % (n % 3)) # the core reads each one at oncetry:-- self._stream_frame(stream["tool"], path)++ self._stream_frame(stream["tool"], path, env=stream.get("env"))n += 1failures = 0self.emit_video(path, stream="screen")@@@ -1153,13 +1232,13 @@ class Computer(Plugin):# The core ends the stream with the same id (a newer one that replaced it carries its own).self.emit_event("screen_stream_ended", sid=stream["sid"], reason=stream.get("reason") or reason, frames=n)-- def _stream_frame(self, tool, path):++ def _stream_frame(self, tool, path, env=None):"""One JPEG frame of the screen at path, at most STREAM_WIDTH wide when Pillow can resize it."""if self._have_pillow():fd, png = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")os.close(fd)try:-- self._capture(tool[1](png))++ self._capture(tool[1](png), env=env)from PIL import Imagewith Image.open(png) as im:im = im.convert("RGB")@@@ -1171,12 +1250,12 @@ class Computer(Plugin):if os.path.exists(png):os.unlink(png)return-- self._capture(self._native_jpeg(tool, path + ".part.jpg"))++ self._capture(self._native_jpeg(tool, path + ".part.jpg"), env=env)os.replace(path + ".part.jpg", path)-- def _capture(self, argv):++ def _capture(self, argv, env=None):proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,-- start_new_session=True)++ start_new_session=True, **({"env": env} if env else {}))try:_, err = proc.communicate(timeout=10)except subprocess.TimeoutExpired:@@@ -91,8 +91,9 @@ def find_binary(which, platform, configured=None):class Browser:-- def __init__(self, binary, profile, headless=True, timeout_s=15.0, popen=subprocess.Popen, extra_args=()):++ def __init__(self, binary, profile, headless=True, timeout_s=15.0, popen=subprocess.Popen, extra_args=(), env=None):self.binary = binary++ self.env = env # the private desk's display and bus, or None for this process's ownself.extra_args = list(extra_args)self.stderr_tail = []self.profile = profile@@@ -136,7 +137,7 @@ class Browser:os.dup2(w, 4)self.proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,-- pass_fds=(3, 4), preexec_fn=fds, start_new_session=True)++ pass_fds=(3, 4), preexec_fn=fds, start_new_session=True, **({"env": self.env} if self.env else {}))threading.Thread(target=self._stderr, args=(self.proc.stderr,), name="computer-browser-err", daemon=True).start()os.close(to_chrome_r)os.close(from_chrome_w)nnew file mode 100644@@@ -0,0 +1,177 @@++"""The private desk: a display of the machine's own where an agent works without touching its owner's screen.++++An Xvfb display with a window manager (when one is installed) and a D-Bus session of its own. The session matters:++apps already open on the owner's desktop (most browsers and GNOME apps) are single-instance over D-Bus, and without a++separate bus "open it on the private desk" would quietly open a window on the owner's screen instead. The owner watches++it through the plugin's live screen (``screen_stream`` with ``desk: private``) and takes over by driving it the same way.++++Every process the desk starts is its own process group, kept by handle and ended by handle (never by name), so stopping++the desk ends exactly what it started. Nothing here touches the owner's display; its address is never read.++++Brought in from the owner's desktop-mcp (``desks.py``), plan track K step 2 (Vibe's desk on Node).++"""++from __future__ import annotations++++import os++import shutil++import signal++import subprocess++import threading++import time++++from openvibe_plugin import Fault++++WINDOW_MANAGERS = ("metacity", "openbox", "xfwm4", "fluxbox")++DISPLAYS = range(90, 100)++START_TIMEOUT_S = 8.0++++++class PrivateDesk:++ def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.time, sleep=time.sleep,++ exists=os.path.exists, base_env=None, displays=DISPLAYS):++ self.which = which++ self.popen = popen++ self.clock = clock++ self.sleep = sleep++ self.exists = exists++ self.base_env = dict(os.environ if base_env is None else base_env)++ self.displays = displays++ self.mu = threading.Lock()++ self.procs = [] # (name, Popen) in start order: Xvfb, then the bus, then the window manager++ self.display = None++ self.width = None++ self.height = None++ self.since = None++ self.bus = None++ self.wm = None++++ # ---- what this machine has -------------------------------------------------------------------------------------++ def available(self):++ return bool(self.which("Xvfb"))++++ def window_manager(self):++ for name in WINDOW_MANAGERS:++ if self.which(name):++ return name++ return None++++ def running(self):++ with self.mu:++ return self._running()++++ def _running(self):++ return bool(self.procs) and self.procs[0][1].poll() is None++++ # ---- start, stop, status --------------------------------------------------------------------------------------++ def status(self):++ with self.mu:++ if not self._running():++ return {"running": False, "available": self.available()}++ return {"running": True, "display": ":%d" % self.display, "width": self.width, "height": self.height,++ "window_manager": self.wm, "since": int(self.since)}++++ def start(self, width=1280, height=800):++ with self.mu:++ if self._running():++ return self._status_locked()++ self._end_locked()++ if not self.available():++ raise Fault("unsupported", "a private desk needs Xvfb on this machine (Debian/Ubuntu: apt install xvfb)")++ n = self._free_display()++ xvfb = self.popen(["Xvfb", ":%d" % n, "-screen", "0", "%dx%dx24" % (width, height), "-nolisten", "tcp",++ "-noreset"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,++ start_new_session=True)++ self.procs = [("Xvfb", xvfb)]++ if not self._wait_for(lambda: self.exists("/tmp/.X11-unix/X%d" % n) or xvfb.poll() is not None):++ self._end_locked()++ raise Fault("hardware", "the private desk did not start in %d s" % START_TIMEOUT_S)++ if xvfb.poll() is not None:++ self._end_locked()++ raise Fault("hardware", "Xvfb ended as it started (display :%d)" % n)++ self.display, self.width, self.height, self.since = n, width, height, self.clock()++ self.bus = self._start_bus()++ self.wm = self.window_manager()++ if self.wm:++ self.procs.append((self.wm, self.popen([self.wm], env=self._env_locked(), stdin=subprocess.DEVNULL,++ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,++ start_new_session=True)))++ return self._status_locked()++++ def stop(self):++ with self.mu:++ was = self._running()++ self._end_locked()++ return {"running": False, "stopped": was}++++ def env(self):++ """The environment for a tool or app that should run on the desk (its display and its own bus)."""++ with self.mu:++ if not self._running():++ raise Fault("not_ready", "start the private desk first (desk start)")++ return self._env_locked()++++ # ---- internals ------------------------------------------------------------------------------------------------++ def _status_locked(self):++ return {"running": True, "display": ":%d" % self.display, "width": self.width, "height": self.height,++ "window_manager": self.wm, "since": int(self.since)}++++ def _env_locked(self):++ env = {k: v for k, v in self.base_env.items() if k not in ("WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "XAUTHORITY")}++ env["DISPLAY"] = ":%d" % self.display++ if self.bus:++ env["DBUS_SESSION_BUS_ADDRESS"] = self.bus++ return env++++ def _free_display(self):++ for n in self.displays:++ if not self.exists("/tmp/.X%d-lock" % n) and not self.exists("/tmp/.X11-unix/X%d" % n):++ return n++ raise Fault("not_ready", "no free display number for a private desk (:%d to :%d are taken)" % (self.displays[0], self.displays[-1]))++++ def _wait_for(self, ready):++ end = self.clock() + START_TIMEOUT_S++ while self.clock() < end:++ if ready():++ return True++ self.sleep(0.05)++ return ready()++++ def _start_bus(self):++ """A D-Bus session of the desk's own; its address, or None where there is no dbus-daemon (apps then share none)."""++ if not self.which("dbus-daemon"):++ return None++ env = self._env_locked()++ proc = self.popen(["dbus-daemon", "--session", "--nofork", "--print-address=1"], env=env, stdin=subprocess.DEVNULL,++ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, start_new_session=True, text=True)++ self.procs.append(("dbus-daemon", proc))++ got = []++ reader = threading.Thread(target=lambda: got.append(proc.stdout.readline().strip()), daemon=True)++ reader.start()++ reader.join(3.0)++ address = got[0] if got and got[0].startswith(("unix:", "tcp:")) else None++ return address++++ def _end_locked(self):++ for _, proc in reversed(self.procs):++ self._kill(proc)++ self.procs = []++ self.display = self.width = self.height = self.since = self.bus = self.wm = None++++ @staticmethod++ def _kill(proc):++ if proc.poll() is not None:++ return++ try:++ if hasattr(os, "killpg"):++ os.killpg(proc.pid, signal.SIGTERM)++ else:++ proc.terminate()++ proc.wait(timeout=3)++ except Exception: # noqa: BLE001 - still running or already gone: make sure it is gone++ try:++ if hasattr(os, "killpg"):++ os.killpg(proc.pid, signal.SIGKILL)++ else:++ proc.kill()++ except Exception: # noqa: BLE001++ passnnew file mode 100644@@@ -0,0 +1,210 @@++"""The private desk (openvibe_computer/desk.py and the plugin's desk kind, plan track K step 2): it starts an Xvfb display++with its own bus and window manager and stops exactly what it started; the plugin starts and stops it under the++machine's level (refused at observe), reports it at every level, and sends screen, input and the browser there with++desk: private (refused until it runs), never to the owner's screen. A real desk runs where Xvfb is installed."""++import base64++import io++import os++import shutil++import sys++import tempfile++import unittest++++HERE = os.path.dirname(os.path.abspath(__file__))++sys.path.insert(0, HERE)++sys.path.insert(0, os.path.dirname(HERE))++sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(HERE)), "sdk"))++++from openvibe_computer import Computer # noqa: E402++from openvibe_computer.desk import PrivateDesk # noqa: E402++from openvibe_plugin import Fault, Runtime # noqa: E402++from test_computer import Out, reply # noqa: E402++++++class FakeProc:++ def __init__(self, argv, kw):++ self.argv, self.kw = argv, kw++ self.pid = 4242++ self.code = None++ self.stdout = io.StringIO("unix:path=/tmp/dbus-desk,guid=1\n")++++ def poll(self):++ return self.code++++ def wait(self, timeout=None):++ self.code = 0++ return 0++++ def terminate(self):++ self.code = -15++++ kill = terminate++++++class DeskModuleTest(unittest.TestCase):++ def make(self, have=("Xvfb", "dbus-daemon", "metacity")):++ self.started = []++ sockets = set()++++ def popen(argv, **kw):++ p = FakeProc(argv, kw)++ self.started.append(p)++ if argv[0] == "Xvfb":++ sockets.add("/tmp/.X11-unix/X%s" % argv[1][1:])++ return p++ desk = PrivateDesk(which=lambda n: "/usr/bin/" + n if n in have else None, popen=popen, sleep=lambda s: None,++ exists=lambda path: path in sockets, base_env={"WAYLAND_DISPLAY": "wayland-0", "DISPLAY": ":0", "HOME": "/home/a"})++ desk._kill = lambda proc: setattr(proc, "code", -15)++ return desk++++ def test_start_status_env_stop(self):++ desk = self.make()++ st = desk.start(1024, 768)++ self.assertEqual(st["display"], ":90")++ self.assertEqual((st["width"], st["height"], st["window_manager"]), (1024, 768, "metacity"))++ xvfb = self.started[0]++ self.assertEqual(xvfb.argv, ["Xvfb", ":90", "-screen", "0", "1024x768x24", "-nolisten", "tcp", "-noreset"])++ self.assertTrue(xvfb.kw["start_new_session"], "its own process group")++ self.assertEqual([p.argv[0] for p in self.started], ["Xvfb", "dbus-daemon", "metacity"])++ env = desk.env()++ self.assertEqual((env["DISPLAY"], env["DBUS_SESSION_BUS_ADDRESS"]), (":90", "unix:path=/tmp/dbus-desk,guid=1"))++ self.assertNotIn("WAYLAND_DISPLAY", env, "nothing goes to the owner's session")++ self.assertEqual(self.started[2].kw["env"]["DISPLAY"], ":90", "the window manager runs on the desk")++ self.assertEqual(desk.start(1024, 768)["display"], ":90", "starting again keeps the running desk")++ self.assertEqual(len(self.started), 3)++ self.assertEqual(desk.stop(), {"running": False, "stopped": True})++ self.assertTrue(all(p.code is not None for p in self.started), "every process it started has ended")++ self.assertEqual(desk.status(), {"running": False, "available": True})++ with self.assertRaises(Fault) as e:++ desk.env()++ self.assertEqual(e.exception.code, "not_ready")++++ def test_without_xvfb_or_a_window_manager(self):++ with self.assertRaises(Fault) as e:++ self.make(have=()).start()++ self.assertEqual(e.exception.code, "unsupported")++ desk = self.make(have=("Xvfb",))++ st = desk.start()++ self.assertIsNone(st["window_manager"])++ self.assertNotIn("DBUS_SESSION_BUS_ADDRESS", desk.env())++++++class FakeDesk:++ def __init__(self):++ self.on = False++++ def available(self):++ return True++++ def window_manager(self):++ return "metacity"++++ def running(self):++ return self.on++++ def start(self, width, height):++ self.on, self.size = True, (width, height)++ return self.status()++++ def stop(self):++ was, self.on = self.on, False++ return {"running": False, "stopped": was}++++ def status(self):++ return {"running": True, "display": ":91", "width": 1000, "height": 500} if self.on else {"running": False, "available": True}++++ def env(self):++ if not self.on:++ raise Fault("not_ready", "start the private desk first (desk start)")++ return {"DISPLAY": ":91"}++++++class DeskPluginTest(unittest.TestCase):++ def boot(self, level):++ self.out = Out()++ self.desk = FakeDesk()++ self.calls = []++++ def popen(argv, **kw):++ self.calls.append((argv, kw.get("env")))++ p = FakeProc(argv, kw)++ p.returncode = 0++ p.communicate = lambda timeout=None: (b"", b"")++ return p++ self.plugin = Computer(env={"DISPLAY": ":0"}, platform="linux", desk=self.desk, popen=popen,++ which=lambda n: "/usr/bin/" + n, screen_size=lambda tool: (2000, 1000))++ self.rt = Runtime(self.plugin, infile=io.StringIO(""), outfile=self.out)++ self.rt.dispatch({"op": "hello", "config": {"level": level, "roots": [tempfile.gettempdir()], "notify": False}})++ self.n = 0++ return self.out.msgs()++++ def cmd(self, kind, value):++ self.n += 1++ cid = "d%d" % self.n++ self.rt.dispatch({"op": "command", "id": cid, "kind": kind, "value": value, "deadline_ms": 30000})++ return cid++++ def test_describe_offers_the_desk(self):++ caps = self.boot("trusted")[0]["capabilities"]++ self.assertEqual(caps["desk"], {"level": "trusted", "window_manager": "metacity", "input": True, "screen": True})++++ def test_observe_reports_but_does_not_start(self):++ self.boot("observe")++ self.assertEqual(reply(self.out, self.cmd("desk", {"action": "status"}))["data"], {"running": False, "available": True})++ r = reply(self.out, self.cmd("desk", {"action": "start"}))++ self.assertEqual((r["op"], r["fault_code"]), ("nack", "not_allowed"))++ self.assertFalse(self.desk.on)++++ def test_private_desk_input_screen_and_refusals(self):++ self.boot("trusted")++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 10, "y": 10, "desk": "private"}))++ self.assertEqual(r["fault_code"], "not_ready", "the desk has to run first")++ r = reply(self.out, self.cmd("desk", {"action": "start", "width": 1000, "height": 500}))++ self.assertEqual(r["data"]["running"], True)++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 50, "y": 50, "width": 100, "height": 100, "desk": "private"}))++ self.assertEqual(r["op"], "ack")++ argv, env = self.calls[-1]++ self.assertEqual(argv, ["xdotool", "mousemove", "--sync", "500", "250", "click", "1"], "scaled to the desk's own 1000×500")++ self.assertEqual(env["DISPLAY"], ":91", "on the desk, never the owner's :0")++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 50, "y": 50, "width": 100, "height": 100}))++ argv, env = self.calls[-1]++ self.assertEqual(argv[3:5], ["1000", "500"], "desk main is the owner's 2000×1000 screen")++ self.assertIsNone(env)++ r = reply(self.out, self.cmd("screen", {"desk": "neighbour"}))++ self.assertEqual(r["fault_code"], "bad_value")++ r = reply(self.out, self.cmd("desk", {"action": "stop"}))++ self.assertEqual(r["data"], {"running": False, "stopped": True})++++ def test_the_kill_switch_stops_the_desk(self):++ self.boot("trusted")++ reply(self.out, self.cmd("desk", {"action": "start"}))++ self.assertTrue(self.desk.on)++ self.plugin.stop()++ self.assertFalse(self.desk.on)+++++[email protected](all(shutil.which(t) for t in ("Xvfb", "import")) and sys.platform.startswith("linux"), "needs Xvfb and ImageMagick")++class RealDeskTest(unittest.TestCase):++ def test_a_real_desk_shows_its_own_screen(self):++ out = Out()++ plugin = Computer(platform="linux")++ rt = Runtime(plugin, infile=io.StringIO(""), outfile=out)++ rt.dispatch({"op": "hello", "config": {"level": "trusted", "roots": [tempfile.gettempdir()], "notify": False}})++ self.addCleanup(plugin.desk.stop)++ rt.dispatch({"op": "command", "id": "r1", "kind": "desk", "value": {"action": "start", "width": 800, "height": 600}, "deadline_ms": 30000})++ st = reply(out, "r1")["data"]++ self.assertTrue(st["running"], st)++ rt.dispatch({"op": "command", "id": "r2", "kind": "screen", "value": {"desk": "private", "max_width": 800}, "deadline_ms": 30000})++ shot = reply(out, "r2")["data"]++ self.assertEqual((shot["width"], shot["height"]), (800, 600), "the desk's own 800×600, not the owner's screen")++ self.assertTrue(base64.b64decode(shot["content_b64"]))++ pids = [p.pid for _, p in plugin.desk.procs]++ rt.dispatch({"op": "command", "id": "r3", "kind": "desk", "value": {"action": "stop"}, "deadline_ms": 30000})++ self.assertEqual(reply(out, "r3")["data"]["stopped"], True)++ for pid in pids:++ with self.assertRaises(ProcessLookupError):++ os.kill(pid, 0)++++++if __name__ == "__main__":++ unittest.main()
@@ -124,6 +124,25 @@ text and keys are passed to the tool as arguments, never through a shell, and `k
@@ -124,6 +124,25 @@ text and keys are passed to the tool as arguments, never through a shell, and `k
- **Closing.** It closes after `browser_idle_s` without a command and at the kill switch.
- **Closing.** It closes after `browser_idle_s` without a command and at the kill switch.
- **Platforms.** Linux and macOS; not offered on Windows yet.
- **Platforms.** Linux and macOS; not offered on Windows yet.
+### The private desk (`desk`, Node v0.15.0)
+
+An agent can work on a desk of its own instead of your screen (plan track K: Vibe's agent mode):
+- **What it is.** An Xvfb display on your machine with a window manager (metacity, openbox, xfwm4 or fluxbox, whichever
+ is installed) and a D-Bus session of its own, so apps opened there open there, not on your screen.
+- **Using it.** `desk` `{"action": "start", "width"?, "height"?}` (1280×800 by default), `stop` or `status`. Then
+ `screen`, `screen_stream`, `input` and `browser` take `"desk": "private"`: screenshots and the live screen of the desk,
+ the mouse and keyboard on the desk (points scale to the desk's own size), and the agent's browser shown on the desk
+ in a profile of its own (`browser_profile` + `-desk`). Without `desk` (or with `main`) they act on your own screen,
+ as before.
+- **Levels.** Starting and stopping a desk are acting commands (refused at `observe`, approved one by one at `ask`);
+ `status` runs at every level. What runs on the desk keeps its own rules: input and the browser still act.
+- **Watching and taking over.** Start the live screen with `"desk": "private"` (your desktop is not told, since nothing of
+ yours is shown) and drive it the same way.
+- **It ends** on `stop`, and at once on `openvibe-node stop` or an e-stop: every process the desk started is ended by
+ handle, never by name.
+- **Needs** Linux with `Xvfb` (Debian/Ubuntu: `apt install xvfb`), `xdotool` for input and ImageMagick `import` or `scrot`
+ for screenshots. Not offered elsewhere; `allow_desk: false` turns it off.
+
## Config
## Config
| key | default | meaning |
| key | default | meaning |
@@ -138,6 +157,7 @@ text and keys are passed to the tool as arguments, never through a shell, and `k
@@ -138,6 +157,7 @@ text and keys are passed to the tool as arguments, never through a shell, and `k
| `notify` | `true` | the plugin's own prompt when a request waits and no tray shows it (graphical session only) |
| `notify` | `true` | the plugin's own prompt when a request waits and no tray shows it (graphical session only) |
| `probe` | `true` | `false`: no probes (OpenVibe.Watch's `node` source) |
| `probe` | `true` | `false`: no probes (OpenVibe.Watch's `node` source) |
| `allow_input` | `true` | `false`: never use the mouse or keyboard (no `input` capability) |
| `allow_input` | `true` | `false`: never use the mouse or keyboard (no `input` capability) |
+| `allow_desk` | `true` | `false`: never start a private desk (no `desk` capability) |
| `allow_browser` | `true` | `false`: no browser (no `browser` capability) |
| `allow_browser` | `true` | `false`: no browser (no `browser` capability) |
| `allow_screen_stream` | `true` | `false`: never show the screen live (`screen.stream` is false) |
| `allow_screen_stream` | `true` | `false`: never show the screen live (`screen.stream` is false) |
| `browser_path` | found on PATH | the browser to drive (Chromium, Chrome, Edge or Brave) |
| `browser_path` | found on PATH | the browser to drive (Chromium, Chrome, Edge or Brave) |
@@ -24,6 +24,7 @@ var capabilityFor = map[string]string{
@@ -24,6 +24,7 @@ var capabilityFor = map[string]string{
protocol.KindProbe: "probe",
protocol.KindProbe: "probe",
protocol.KindInput: "input",
protocol.KindInput: "input",
protocol.KindBrowser: "browser",
protocol.KindBrowser: "browser",
+ protocol.KindDesk: "desk",
}
}
// Manager supervises every configured plugin.
// Manager supervises every configured plugin.
@@ -175,13 +175,17 @@ const (
@@ -175,13 +175,17 @@ const (
// machine's private OpenRestream stream); no operator sends them. The computer plugin captures the frames, the
// machine's private OpenRestream stream); no operator sends them. The computer plugin captures the frames, the
// core publishes them; a latched stop ends it and refuses a start.
// core publishes them; a latched stop ends it and refuses a start.
KindScreenStream = "screen_stream"
KindScreenStream = "screen_stream"
+ // KindDesk starts, stops or reports a private desk on the machine (contracts 0.214.0): {"action": "start"|"stop"|
+ // "status", "width"?, "height"?}, an Xvfb display of its own where an agent works without touching the owner's
+ // screen; screen, screen_stream, input and browser take "desk": "private" to act there. Starting and stopping act.
+ KindDesk = "desk"
)
)
// Kinds lists every command kind the Node understands.
// Kinds lists every command kind the Node understands.
-var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream}
+var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream, KindDesk}
// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.
// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.
-var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true}
+var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true, KindDesk: true}
const (
const (
// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps
// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps
@@ -194,8 +198,8 @@ const (
@@ -194,8 +198,8 @@ const (
)
)
// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file, uses the mouse
// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file, uses the mouse
-// or keyboard, drives the browser): those are refused while the local stop or an e-stop is latched. Capturing the
-// screen, reading files and reading or closing the browser are not.
+// or keyboard, drives the browser, starts or stops the private desk): those are refused while the local stop or an
+// e-stop is latched. Capturing the screen, reading files, reading or closing the browser and the desk's status are not.
func ComputerActs(kind string, value json.RawMessage) bool {
func ComputerActs(kind string, value json.RawMessage) bool {
switch kind {
switch kind {
case KindExec, KindInput:
case KindExec, KindInput:
@@ -212,6 +216,12 @@ func ComputerActs(kind string, value json.RawMessage) bool {
@@ -212,6 +216,12 @@ func ComputerActs(kind string, value json.RawMessage) bool {
}
}
_ = json.Unmarshal(value, &v)
_ = json.Unmarshal(value, &v)
return v.Action != "read" && v.Action != "screenshot" && v.Action != "close"
return v.Action != "read" && v.Action != "screenshot" && v.Action != "close"
+ case KindDesk:
+ var v struct {
+ Action string `json:"action"`
+ }
+ _ = json.Unmarshal(value, &v)
+ return v.Action != "status"
}
}
return false
return false
}
}
@@ -396,6 +396,18 @@ func TestJobUndecodableBody(t *testing.T) {
@@ -396,6 +396,18 @@ func TestJobUndecodableBody(t *testing.T) {
}
}
}
}
+func TestComputerActsDesk(t *testing.T) {
+ cases := map[string]bool{"start": true, "stop": true, "status": false}
+ for action, acts := range cases {
+ if got := ComputerActs(KindDesk, json.RawMessage(`{"action":"`+action+`"}`)); got != acts {
+ t.Errorf("desk %s acts = %v, want %v", action, got, acts)
+ }
+ }
+ if !ComputerKinds[KindDesk] {
+ t.Error("desk is a computer kind (long deadline, no clamping)")
+ }
+}
+
func TestComputerActsBrowser(t *testing.T) {
func TestComputerActsBrowser(t *testing.T) {
want := map[string]bool{"open": true, "click": true, "fill": true, "press": true, "back": true, "read": false, "screenshot": false, "close": false}
want := map[string]bool{"open": true, "click": true, "fill": true, "press": true, "back": true, "read": false, "screenshot": false, "close": false}
for action, acts := range want {
for action, acts := range want {
@@ -40,6 +40,12 @@ Command kinds and their answers (the ack's ``data``):
@@ -40,6 +40,12 @@ Command kinds and their answers (the ack's ``data``):
way; ``screenshot`` → ``{format, width, height, content_b64}``; ``close``. Reading, screenshots and closing run at every
way; ``screenshot`` → ``{format, width, height, content_b64}``; ``close``. Reading, screenshots and closing run at every
level; the rest are acting commands. Below level full, ``open`` refuses an address on the machine's own network.
level; the rest are acting commands. Below level full, ``open`` refuses an address on the machine's own network.
Config ``allow_browser: false`` turns it off; ``browser_path``, ``browser_headless`` (default true).
Config ``allow_browser: false`` turns it off; ``browser_path``, ``browser_headless`` (default true).
+* ``desk`` ``{"action": "start"|"stop"|"status", "width"?, "height"?}``: a private desk (``desk.py``): an Xvfb display with a
+ window manager and a D-Bus session of its own, where the agent works without touching the owner's screen. ``screen``,
+ ``screen_stream``, ``input`` and ``browser`` take ``"desk": "private"`` to act there (the browser shown on the desk, in a
+ profile of its own, ``browser_profile`` + ``-desk``); ``main`` (the default) is the owner's screen. Starting and stopping
+ act (refused at observe, approved at ask), ``status`` runs at every level, and the kill switch stops the desk. Linux
+ with Xvfb; config ``allow_desk: false`` turns it off.
* ``probe`` ``{"target", "timeout_s"?}`` → for ``http(s)://…`` one GET, no redirect followed: ``{status, content_type,
* ``probe`` ``{"target", "timeout_s"?}`` → for ``http(s)://…`` one GET, no redirect followed: ``{status, content_type,
etag, last_modified, body, truncated, ms}``; for ``tcp://host:port``: ``{open, ms, error?}``. Read-only, so it runs at
etag, last_modified, body, truncated, ms}``; for ``tcp://host:port``: ``{open, ms, error?}``. Read-only, so it runs at
every level, and only towards the machine's own network: every address the name resolves to must be private,
every level, and only towards the machine's own network: every address the name resolves to must be private,
@@ -67,6 +73,7 @@ import urllib.parse
@@ -67,6 +73,7 @@ import urllib.parse
from openvibe_plugin import Deferred, Fault, Plugin
from openvibe_plugin import Deferred, Fault, Plugin
from .browser import KEY_CODES, Browser, BrowserError, find_binary
from .browser import KEY_CODES, Browser, BrowserError, find_binary
+from .desk import PrivateDesk
LEVELS = ("observe", "ask", "trusted", "full")
LEVELS = ("observe", "ask", "trusted", "full")
RANK = {name: i for i, name in enumerate(LEVELS)}
RANK = {name: i for i, name in enumerate(LEVELS)}
@@ -132,12 +139,12 @@ def _int(v, name, lo, hi, default):
@@ -132,12 +139,12 @@ def _int(v, name, lo, hi, default):
class Computer(Plugin):
class Computer(Plugin):
driver = "computer"
driver = "computer"
- version = "0.7.0"
+ version = "0.8.0"
motion_kinds = frozenset()
motion_kinds = frozenset()
telemetry_interval_s = 2.0
telemetry_interval_s = 2.0
def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None, screen_size=None,
def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None, screen_size=None,
- browser_factory=None, resolve=None):
+ browser_factory=None, resolve=None, desk=None):
self.which = which
self.which = which
self.popen = popen
self.popen = popen
self.screen_size = screen_size or self._screen_size
self.screen_size = screen_size or self._screen_size
@@ -174,7 +181,10 @@ class Computer(Plugin):
@@ -174,7 +181,10 @@ class Computer(Plugin):
self.stream_mu = threading.Lock()
self.stream_mu = threading.Lock()
self.browser_mu = threading.Lock()
self.browser_mu = threading.Lock()
self.browser_op_mu = threading.Lock()
self.browser_op_mu = threading.Lock()
- self.browser_factory = browser_factory or (lambda binary, profile, headless: Browser(binary, profile, headless))
+ self.browser_factory = browser_factory or (lambda binary, profile, headless, env=None: Browser(binary, profile, headless, env=env))
+ self.desk = desk or PrivateDesk(which=which, popen=popen)
+ self.allow_desk = True
+ self.desk_browser = None # the agent's browser on the private desk (its own profile, shown there, not headless)
self.resolve = resolve or (lambda host: [ai[4][0] for ai in socket.getaddrinfo(host, None)])
self.resolve = resolve or (lambda host: [ai[4][0] for ai in socket.getaddrinfo(host, None)])
# ---- configuration -------------------------------------------------------------------------------------------
# ---- configuration -------------------------------------------------------------------------------------------
@@ -199,6 +209,7 @@ class Computer(Plugin):
@@ -199,6 +209,7 @@ class Computer(Plugin):
self.notify = config.get("notify", True) is not False
self.notify = config.get("notify", True) is not False
self.allow_browser = config.get("allow_browser", True) is not False
self.allow_browser = config.get("allow_browser", True) is not False
self.allow_screen_stream = config.get("allow_screen_stream", True) is not False
self.allow_screen_stream = config.get("allow_screen_stream", True) is not False
+ self.allow_desk = config.get("allow_desk", True) is not False
self.browser_path = config.get("browser_path") or None
self.browser_path = config.get("browser_path") or None
self.browser_profile = os.path.expanduser(config.get("browser_profile") or "~/.local/share/openvibe-node/browser")
self.browser_profile = os.path.expanduser(config.get("browser_profile") or "~/.local/share/openvibe-node/browser")
self.browser_headless = config.get("browser_headless", True) is not False
self.browser_headless = config.get("browser_headless", True) is not False
@@ -224,6 +235,9 @@ class Computer(Plugin):
@@ -224,6 +235,9 @@ class Computer(Plugin):
binary = self._browser_binary()
binary = self._browser_binary()
if binary:
if binary:
caps["browser"] = {"level": self.level, "browser": os.path.basename(binary), "headless": self.browser_headless}
caps["browser"] = {"level": self.level, "browser": os.path.basename(binary), "headless": self.browser_headless}
+ if self.allow_desk and self.platform.startswith("linux") and self.desk.available():
+ caps["desk"] = {"level": self.level, "window_manager": self.desk.window_manager(), "input": bool(self.which("xdotool")),
+ "screen": bool(self._x11_tool())}
return caps
return caps
def setup(self, config):
def setup(self, config):
@@ -248,6 +262,7 @@ class Computer(Plugin):
@@ -248,6 +262,7 @@ class Computer(Plugin):
self._kill(p)
self._kill(p)
self._close_browser()
self._close_browser()
self._end_stream("stopped on the machine")
self._end_stream("stopped on the machine")
+ self.desk.stop()
for cid in waiting:
for cid in waiting:
self._close_prompt(cid)
self._close_prompt(cid)
self.emit_event("approval_done", id=cid, approved=False, reason="stopped")
self.emit_event("approval_done", id=cid, approved=False, reason="stopped")
@@ -309,14 +324,17 @@ class Computer(Plugin):
@@ -309,14 +324,17 @@ class Computer(Plugin):
if not isinstance(v, dict):
if not isinstance(v, dict):
raise Fault("bad_value", "value must be an object")
raise Fault("bad_value", "value must be an object")
level = self.effective_level()
level = self.effective_level()
+ if cmd.kind == "desk":
+ return self._desk_op(cmd, v, level)
+ desk_env = self._desk_env(v) if cmd.kind in ("screen", "screen_stream", "input", "browser") else None
if cmd.kind == "screen":
if cmd.kind == "screen":
- tool = self._screen_tool()
+ tool = self._x11_tool() if desk_env else self._screen_tool()
if not tool:
if not tool:
raise Fault("unsupported", "no screen capture tool on this machine")
raise Fault("unsupported", "no screen capture tool on this machine")
width = _int(v.get("max_width"), "max_width", 320, 3840, 1280)
width = _int(v.get("max_width"), "max_width", 320, 3840, 1280)
- return self._start(cmd, lambda: self._screen(tool, width))
+ return self._start(cmd, lambda: self._screen(tool, width, env=desk_env))
if cmd.kind == "screen_stream":
if cmd.kind == "screen_stream":
- return self._screen_stream(v)
+ return self._screen_stream(v, env=desk_env)
if cmd.kind == "fs":
if cmd.kind == "fs":
op = v.get("op")
op = v.get("op")
if op in READ_OPS:
if op in READ_OPS:
@@ -333,13 +351,13 @@ class Computer(Plugin):
@@ -333,13 +351,13 @@ class Computer(Plugin):
if cmd.kind == "input":
if cmd.kind == "input":
if not self.allow_input:
if not self.allow_input:
raise Fault("unsupported", "this machine does not allow mouse and keyboard input")
raise Fault("unsupported", "this machine does not allow mouse and keyboard input")
- tool = self._input_tool()
+ tool = ("xdotool" if self.which("xdotool") else None) if desk_env else self._input_tool()
if not tool:
if not tool:
raise Fault("unsupported", "no input tool on this machine (xdotool on X11, cliclick on macOS)")
raise Fault("unsupported", "no input tool on this machine (xdotool on X11, cliclick on macOS)")
- run, summary = self._input_op(tool, v)
+ run, summary = self._input_op(tool, v, env=desk_env)
return self._act(cmd, level, self._driving(run), summary, threaded=True)
return self._act(cmd, level, self._driving(run), summary, threaded=True)
if cmd.kind == "browser":
if cmd.kind == "browser":
- run, summary, acting = self._browser_op(v, level)
+ run, summary, acting = self._browser_op(v, level, env=desk_env)
if acting:
if acting:
return self._act(cmd, level, run, summary, threaded=True)
return self._act(cmd, level, run, summary, threaded=True)
return self._start(cmd, run)
return self._start(cmd, run)
@@ -350,6 +368,51 @@ class Computer(Plugin):
@@ -350,6 +368,51 @@ class Computer(Plugin):
return self._act(cmd, level, run, summary, threaded=True)
return self._act(cmd, level, run, summary, threaded=True)
raise Fault("unsupported", "kind %s not supported" % cmd.kind)
raise Fault("unsupported", "kind %s not supported" % cmd.kind)
+ # ---- the private desk --------------------------------------------------------------------------------------------
+ def _desk_env(self, v):
+ """None for the owner's own screen (desk main, the default); the private desk's environment for desk private."""
+ where = v.get("desk", "main")
+ if where not in ("main", "private"):
+ raise Fault("bad_value", "desk is main or private")
+ if where == "main":
+ return None
+ if not self.allow_desk:
+ raise Fault("unsupported", "this machine does not allow a private desk (allow_desk is off)")
+ return self.desk.env()
+
+ def _desk_op(self, cmd, v, level):
+ action = v.get("action")
+ if action == "status":
+ return self.desk.status()
+ if action not in ("start", "stop"):
+ raise Fault("bad_value", "action is start, stop or status")
+ if not self.allow_desk or not self.platform.startswith("linux"):
+ raise Fault("unsupported", "this machine does not offer a private desk (Linux with Xvfb, allow_desk on)")
+ if action == "start":
+ width = _int(v.get("width"), "width", 640, 3840, 1280)
+ height = _int(v.get("height"), "height", 480, 2160, 800)
+ return self._act(cmd, level, lambda: self.desk.start(width, height),
+ "start a private desk (%d×%d) where your agent works without touching your screen" % (width, height),
+ threaded=True)
+
+ def stop():
+ self._close_desk_browser()
+ return self.desk.stop()
+ return self._act(cmd, level, stop, "stop the private desk (and close what runs on it)", threaded=False)
+
+ def _close_desk_browser(self):
+ b, self.desk_browser = self.desk_browser, None
+ if b is not None:
+ b.close()
+
+ def _x11_tool(self):
+ """An X11 capture tool for the private desk (it is always X11, whatever the owner's session is)."""
+ if self.which("import"):
+ return ("import", lambda out: ["import", "-window", "root", out])
+ if self.which("scrot"):
+ return ("scrot", lambda out: ["scrot", "-o", out])
+ return None
+
def _act(self, cmd, level, run, summary, threaded):
def _act(self, cmd, level, run, summary, threaded):
if RANK[level] < RANK["ask"]:
if RANK[level] < RANK["ask"]:
stopped = self.runtime is not None and self.runtime.estopped
stopped = self.runtime is not None and self.runtime.estopped
@@ -828,7 +891,7 @@ class Computer(Plugin):
@@ -828,7 +891,7 @@ class Computer(Plugin):
raise Fault("not_allowed", "%s is on this machine's own network: only at level full" % u.hostname)
raise Fault("not_allowed", "%s is on this machine's own network: only at level full" % u.hostname)
return raw
return raw
- def _browser_op(self, v, level):
+ def _browser_op(self, v, level, env=None):
"""(run, summary, acting) for one browser action; nothing is opened or clicked before run()."""
"""(run, summary, acting) for one browser action; nothing is opened or clicked before run()."""
binary = self._browser_binary()
binary = self._browser_binary()
if not binary:
if not binary:
@@ -854,13 +917,17 @@ class Computer(Plugin):
@@ -854,13 +917,17 @@ class Computer(Plugin):
def browser():
def browser():
with self.browser_mu:
with self.browser_mu:
+ if env: # on the private desk: shown there (never headless), in a profile of its own
+ if self.desk_browser is None or not self.desk_browser.alive():
+ self.desk_browser = self.browser_factory(binary, self.browser_profile + "-desk", False, env=env)
+ return self.desk_browser
if self.browser is None or not self.browser.alive():
if self.browser is None or not self.browser.alive():
self.browser = self.browser_factory(binary, self.browser_profile, self.browser_headless)
self.browser = self.browser_factory(binary, self.browser_profile, self.browser_headless)
return self.browser
return self.browser
def run():
def run():
if action == "close":
if action == "close":
- self._close_browser()
+ self._close_desk_browser() if env else self._close_browser()
return {"done": True}
return {"done": True}
b = browser()
b = browser()
try:
try:
@@ -904,8 +971,9 @@ class Computer(Plugin):
@@ -904,8 +971,9 @@ class Computer(Plugin):
return "xdotool"
return "xdotool"
return None
return None
- def _input_op(self, tool, v):
- """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask)."""
+ def _input_op(self, tool, v, env=None):
+ """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask). With env,
+ on the private desk (its display; points scale to the desk's own size)."""
action = v.get("action")
action = v.get("action")
if action not in INPUT_ACTIONS:
if action not in INPUT_ACTIONS:
raise Fault("bad_value", "action is one of %s" % ", ".join(sorted(INPUT_ACTIONS)))
raise Fault("bad_value", "action is one of %s" % ", ".join(sorted(INPUT_ACTIONS)))
@@ -936,9 +1004,10 @@ class Computer(Plugin):
@@ -936,9 +1004,10 @@ class Computer(Plugin):
raise Fault("bad_value", "keys is one key or chord, for example Return or ctrl+s")
raise Fault("bad_value", "keys is one key or chord, for example Return or ctrl+s")
def run():
def run():
- at = self._scale(tool, point) if point else None
+ at = (self._desk_scale(point) if env else self._scale(tool, point)) if point else None
argv = self._input_argv(tool, action, at, v)
argv = self._input_argv(tool, action, at, v)
- proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True)
+ proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True,
+ **({"env": env} if env else {}))
try:
try:
_, err = proc.communicate(timeout=30)
_, err = proc.communicate(timeout=30)
except subprocess.TimeoutExpired:
except subprocess.TimeoutExpired:
@@ -951,19 +1020,27 @@ class Computer(Plugin):
@@ -951,19 +1020,27 @@ class Computer(Plugin):
out["x"], out["y"] = at
out["x"], out["y"] = at
return out
return out
+ where = " on the private desk" if env else ""
if action == "type":
if action == "type":
text = v["text"]
text = v["text"]
shown = text if len(text) <= 120 else text[:120] + "…"
shown = text if len(text) <= 120 else text[:120] + "…"
- summary = "type %d characters: %r" % (len(text), shown)
+ summary = "type %d characters%s: %r" % (len(text), where, shown)
elif action == "key":
elif action == "key":
- summary = "press %s" % v["keys"]
+ summary = "press %s%s" % (v["keys"], where)
elif action == "scroll":
elif action == "scroll":
- summary = "scroll %s %d" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]))
+ summary = "scroll %s %d%s" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]), where)
else:
else:
x, y, w, h = point
x, y, w, h = point
- summary = "%s at %d, %d%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "")
+ summary = "%s at %d, %d%s%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "", where)
return run, summary
return run, summary
+ def _desk_scale(self, point):
+ x, y, w, h = point
+ st = self.desk.status()
+ if not w or not st.get("running"):
+ return x, y
+ return min(st["width"] - 1, round(x * st["width"] / w)), min(st["height"] - 1, round(y * st["height"] / h))
+
def _screen_size(self, tool):
def _screen_size(self, tool):
"""The screen's size in the units the input tool uses: pixels for xdotool, points for cliclick (a Retina
"""The screen's size in the units the input tool uses: pixels for xdotool, points for cliclick (a Retina
screen's picture is twice as many pixels)."""
screen's picture is twice as many pixels)."""
@@ -1044,12 +1121,12 @@ class Computer(Plugin):
@@ -1044,12 +1121,12 @@ class Computer(Plugin):
return ("gnome-screenshot", lambda out: ["gnome-screenshot", "-f", out])
return ("gnome-screenshot", lambda out: ["gnome-screenshot", "-f", out])
return None
return None
- def _screen(self, tool, max_width):
+ def _screen(self, tool, max_width, env=None):
fd, out = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")
fd, out = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")
os.close(fd)
os.close(fd)
try:
try:
proc = self.popen(tool[1](out), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
proc = self.popen(tool[1](out), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
- stderr=subprocess.PIPE, start_new_session=True)
+ stderr=subprocess.PIPE, start_new_session=True, **({"env": env} if env else {}))
try:
try:
_, err = proc.communicate(timeout=15)
_, err = proc.communicate(timeout=15)
except subprocess.TimeoutExpired:
except subprocess.TimeoutExpired:
@@ -1088,7 +1165,7 @@ class Computer(Plugin):
@@ -1088,7 +1165,7 @@ class Computer(Plugin):
def _stream_capable(self, tool):
def _stream_capable(self, tool):
return self._have_pillow() or self._native_jpeg(tool, "x.jpg") is not None
return self._have_pillow() or self._native_jpeg(tool, "x.jpg") is not None
- def _screen_stream(self, v):
+ def _screen_stream(self, v, env=None):
action = v.get("action")
action = v.get("action")
if action == "stop":
if action == "stop":
return {"streaming": False, "stopped": self._end_stream("stopped by its owner")}
return {"streaming": False, "stopped": self._end_stream("stopped by its owner")}
@@ -1098,7 +1175,7 @@ class Computer(Plugin):
@@ -1098,7 +1175,7 @@ class Computer(Plugin):
raise Fault("unsupported", "this machine does not show its screen live (allow_screen_stream is off)")
raise Fault("unsupported", "this machine does not show its screen live (allow_screen_stream is off)")
if self.runtime is not None and self.runtime.estopped:
if self.runtime is not None and self.runtime.estopped:
raise Fault("local_stop", "stopped on the machine")
raise Fault("local_stop", "stopped on the machine")
- tool = self._screen_tool()
+ tool = self._x11_tool() if env else self._screen_tool()
if not tool:
if not tool:
raise Fault("unsupported", "no screen capture tool on this machine")
raise Fault("unsupported", "no screen capture tool on this machine")
if not self._stream_capable(tool):
if not self._stream_capable(tool):
@@ -1108,12 +1185,14 @@ class Computer(Plugin):
@@ -1108,12 +1185,14 @@ class Computer(Plugin):
self._end_stream("replaced")
self._end_stream("replaced")
frames = tempfile.mkdtemp(prefix="openvibe-screen-stream-") # 0700, this account only
frames = tempfile.mkdtemp(prefix="openvibe-screen-stream-") # 0700, this account only
stream = {"sid": os.urandom(8).hex(), "until": self.clock() + seconds, "fps": fps, "tool": tool, "dir": frames,
stream = {"sid": os.urandom(8).hex(), "until": self.clock() + seconds, "fps": fps, "tool": tool, "dir": frames,
- "stop": threading.Event()}
+ "stop": threading.Event(), "env": env}
with self.stream_mu:
with self.stream_mu:
self.stream = stream
self.stream = stream
threading.Thread(target=self._stream_loop, args=(stream,), name="computer-screen-stream", daemon=True).start()
threading.Thread(target=self._stream_loop, args=(stream,), name="computer-screen-stream", daemon=True).start()
- self._notify_stream(seconds)
- return {"streaming": True, "sid": stream["sid"], "seconds": seconds, "fps": fps, "tool": tool[0]}
+ if not env: # the private desk is not the owner's screen: nothing of theirs goes live
+ self._notify_stream(seconds)
+ return {"streaming": True, "sid": stream["sid"], "seconds": seconds, "fps": fps, "tool": tool[0],
+ "desk": "private" if env else "main"}
def _end_stream(self, reason):
def _end_stream(self, reason):
"""End the live screen, if one runs → whether one did."""
"""End the live screen, if one runs → whether one did."""
@@ -1135,7 +1214,7 @@ class Computer(Plugin):
@@ -1135,7 +1214,7 @@ class Computer(Plugin):
break
break
path = os.path.join(stream["dir"], "frame-%d.jpg" % (n % 3)) # the core reads each one at once
path = os.path.join(stream["dir"], "frame-%d.jpg" % (n % 3)) # the core reads each one at once
try:
try:
- self._stream_frame(stream["tool"], path)
+ self._stream_frame(stream["tool"], path, env=stream.get("env"))
n += 1
n += 1
failures = 0
failures = 0
self.emit_video(path, stream="screen")
self.emit_video(path, stream="screen")
@@ -1153,13 +1232,13 @@ class Computer(Plugin):
@@ -1153,13 +1232,13 @@ class Computer(Plugin):
# The core ends the stream with the same id (a newer one that replaced it carries its own).
# The core ends the stream with the same id (a newer one that replaced it carries its own).
self.emit_event("screen_stream_ended", sid=stream["sid"], reason=stream.get("reason") or reason, frames=n)
self.emit_event("screen_stream_ended", sid=stream["sid"], reason=stream.get("reason") or reason, frames=n)
- def _stream_frame(self, tool, path):
+ def _stream_frame(self, tool, path, env=None):
"""One JPEG frame of the screen at path, at most STREAM_WIDTH wide when Pillow can resize it."""
"""One JPEG frame of the screen at path, at most STREAM_WIDTH wide when Pillow can resize it."""
if self._have_pillow():
if self._have_pillow():
fd, png = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")
fd, png = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")
os.close(fd)
os.close(fd)
try:
try:
- self._capture(tool[1](png))
+ self._capture(tool[1](png), env=env)
from PIL import Image
from PIL import Image
with Image.open(png) as im:
with Image.open(png) as im:
im = im.convert("RGB")
im = im.convert("RGB")
@@ -1171,12 +1250,12 @@ class Computer(Plugin):
@@ -1171,12 +1250,12 @@ class Computer(Plugin):
if os.path.exists(png):
if os.path.exists(png):
os.unlink(png)
os.unlink(png)
return
return
- self._capture(self._native_jpeg(tool, path + ".part.jpg"))
+ self._capture(self._native_jpeg(tool, path + ".part.jpg"), env=env)
os.replace(path + ".part.jpg", path)
os.replace(path + ".part.jpg", path)
- def _capture(self, argv):
+ def _capture(self, argv, env=None):
proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
- start_new_session=True)
+ start_new_session=True, **({"env": env} if env else {}))
try:
try:
_, err = proc.communicate(timeout=10)
_, err = proc.communicate(timeout=10)
except subprocess.TimeoutExpired:
except subprocess.TimeoutExpired:
@@ -91,8 +91,9 @@ def find_binary(which, platform, configured=None):
@@ -91,8 +91,9 @@ def find_binary(which, platform, configured=None):
class Browser:
class Browser:
- def __init__(self, binary, profile, headless=True, timeout_s=15.0, popen=subprocess.Popen, extra_args=()):
+ def __init__(self, binary, profile, headless=True, timeout_s=15.0, popen=subprocess.Popen, extra_args=(), env=None):
self.binary = binary
self.binary = binary
+ self.env = env # the private desk's display and bus, or None for this process's own
self.extra_args = list(extra_args)
self.extra_args = list(extra_args)
self.stderr_tail = []
self.stderr_tail = []
self.profile = profile
self.profile = profile
@@ -136,7 +137,7 @@ class Browser:
@@ -136,7 +137,7 @@ class Browser:
os.dup2(w, 4)
os.dup2(w, 4)
self.proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
self.proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
- pass_fds=(3, 4), preexec_fn=fds, start_new_session=True)
+ pass_fds=(3, 4), preexec_fn=fds, start_new_session=True, **({"env": self.env} if self.env else {}))
threading.Thread(target=self._stderr, args=(self.proc.stderr,), name="computer-browser-err", daemon=True).start()
threading.Thread(target=self._stderr, args=(self.proc.stderr,), name="computer-browser-err", daemon=True).start()
os.close(to_chrome_r)
os.close(to_chrome_r)
os.close(from_chrome_w)
os.close(from_chrome_w)
new file mode 100644
new file mode 100644
@@ -0,0 +1,177 @@
@@ -0,0 +1,177 @@
+"""The private desk: a display of the machine's own where an agent works without touching its owner's screen.
+
+An Xvfb display with a window manager (when one is installed) and a D-Bus session of its own. The session matters:
+apps already open on the owner's desktop (most browsers and GNOME apps) are single-instance over D-Bus, and without a
+separate bus "open it on the private desk" would quietly open a window on the owner's screen instead. The owner watches
+it through the plugin's live screen (``screen_stream`` with ``desk: private``) and takes over by driving it the same way.
+
+Every process the desk starts is its own process group, kept by handle and ended by handle (never by name), so stopping
+the desk ends exactly what it started. Nothing here touches the owner's display; its address is never read.
+
+Brought in from the owner's desktop-mcp (``desks.py``), plan track K step 2 (Vibe's desk on Node).
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import signal
+import subprocess
+import threading
+import time
+
+from openvibe_plugin import Fault
+
+WINDOW_MANAGERS = ("metacity", "openbox", "xfwm4", "fluxbox")
+DISPLAYS = range(90, 100)
+START_TIMEOUT_S = 8.0
+
+
+class PrivateDesk:
+ def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.time, sleep=time.sleep,
+ exists=os.path.exists, base_env=None, displays=DISPLAYS):
+ self.which = which
+ self.popen = popen
+ self.clock = clock
+ self.sleep = sleep
+ self.exists = exists
+ self.base_env = dict(os.environ if base_env is None else base_env)
+ self.displays = displays
+ self.mu = threading.Lock()
+ self.procs = [] # (name, Popen) in start order: Xvfb, then the bus, then the window manager
+ self.display = None
+ self.width = None
+ self.height = None
+ self.since = None
+ self.bus = None
+ self.wm = None
+
+ # ---- what this machine has -------------------------------------------------------------------------------------
+ def available(self):
+ return bool(self.which("Xvfb"))
+
+ def window_manager(self):
+ for name in WINDOW_MANAGERS:
+ if self.which(name):
+ return name
+ return None
+
+ def running(self):
+ with self.mu:
+ return self._running()
+
+ def _running(self):
+ return bool(self.procs) and self.procs[0][1].poll() is None
+
+ # ---- start, stop, status --------------------------------------------------------------------------------------
+ def status(self):
+ with self.mu:
+ if not self._running():
+ return {"running": False, "available": self.available()}
+ return {"running": True, "display": ":%d" % self.display, "width": self.width, "height": self.height,
+ "window_manager": self.wm, "since": int(self.since)}
+
+ def start(self, width=1280, height=800):
+ with self.mu:
+ if self._running():
+ return self._status_locked()
+ self._end_locked()
+ if not self.available():
+ raise Fault("unsupported", "a private desk needs Xvfb on this machine (Debian/Ubuntu: apt install xvfb)")
+ n = self._free_display()
+ xvfb = self.popen(["Xvfb", ":%d" % n, "-screen", "0", "%dx%dx24" % (width, height), "-nolisten", "tcp",
+ "-noreset"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+ start_new_session=True)
+ self.procs = [("Xvfb", xvfb)]
+ if not self._wait_for(lambda: self.exists("/tmp/.X11-unix/X%d" % n) or xvfb.poll() is not None):
+ self._end_locked()
+ raise Fault("hardware", "the private desk did not start in %d s" % START_TIMEOUT_S)
+ if xvfb.poll() is not None:
+ self._end_locked()
+ raise Fault("hardware", "Xvfb ended as it started (display :%d)" % n)
+ self.display, self.width, self.height, self.since = n, width, height, self.clock()
+ self.bus = self._start_bus()
+ self.wm = self.window_manager()
+ if self.wm:
+ self.procs.append((self.wm, self.popen([self.wm], env=self._env_locked(), stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+ start_new_session=True)))
+ return self._status_locked()
+
+ def stop(self):
+ with self.mu:
+ was = self._running()
+ self._end_locked()
+ return {"running": False, "stopped": was}
+
+ def env(self):
+ """The environment for a tool or app that should run on the desk (its display and its own bus)."""
+ with self.mu:
+ if not self._running():
+ raise Fault("not_ready", "start the private desk first (desk start)")
+ return self._env_locked()
+
+ # ---- internals ------------------------------------------------------------------------------------------------
+ def _status_locked(self):
+ return {"running": True, "display": ":%d" % self.display, "width": self.width, "height": self.height,
+ "window_manager": self.wm, "since": int(self.since)}
+
+ def _env_locked(self):
+ env = {k: v for k, v in self.base_env.items() if k not in ("WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "XAUTHORITY")}
+ env["DISPLAY"] = ":%d" % self.display
+ if self.bus:
+ env["DBUS_SESSION_BUS_ADDRESS"] = self.bus
+ return env
+
+ def _free_display(self):
+ for n in self.displays:
+ if not self.exists("/tmp/.X%d-lock" % n) and not self.exists("/tmp/.X11-unix/X%d" % n):
+ return n
+ raise Fault("not_ready", "no free display number for a private desk (:%d to :%d are taken)" % (self.displays[0], self.displays[-1]))
+
+ def _wait_for(self, ready):
+ end = self.clock() + START_TIMEOUT_S
+ while self.clock() < end:
+ if ready():
+ return True
+ self.sleep(0.05)
+ return ready()
+
+ def _start_bus(self):
+ """A D-Bus session of the desk's own; its address, or None where there is no dbus-daemon (apps then share none)."""
+ if not self.which("dbus-daemon"):
+ return None
+ env = self._env_locked()
+ proc = self.popen(["dbus-daemon", "--session", "--nofork", "--print-address=1"], env=env, stdin=subprocess.DEVNULL,
+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, start_new_session=True, text=True)
+ self.procs.append(("dbus-daemon", proc))
+ got = []
+ reader = threading.Thread(target=lambda: got.append(proc.stdout.readline().strip()), daemon=True)
+ reader.start()
+ reader.join(3.0)
+ address = got[0] if got and got[0].startswith(("unix:", "tcp:")) else None
+ return address
+
+ def _end_locked(self):
+ for _, proc in reversed(self.procs):
+ self._kill(proc)
+ self.procs = []
+ self.display = self.width = self.height = self.since = self.bus = self.wm = None
+
+ @staticmethod
+ def _kill(proc):
+ if proc.poll() is not None:
+ return
+ try:
+ if hasattr(os, "killpg"):
+ os.killpg(proc.pid, signal.SIGTERM)
+ else:
+ proc.terminate()
+ proc.wait(timeout=3)
+ except Exception: # noqa: BLE001 - still running or already gone: make sure it is gone
+ try:
+ if hasattr(os, "killpg"):
+ os.killpg(proc.pid, signal.SIGKILL)
+ else:
+ proc.kill()
+ except Exception: # noqa: BLE001
+ pass
new file mode 100644
new file mode 100644
@@ -0,0 +1,210 @@
@@ -0,0 +1,210 @@
+"""The private desk (openvibe_computer/desk.py and the plugin's desk kind, plan track K step 2): it starts an Xvfb display
+with its own bus and window manager and stops exactly what it started; the plugin starts and stops it under the
+machine's level (refused at observe), reports it at every level, and sends screen, input and the browser there with
+desk: private (refused until it runs), never to the owner's screen. A real desk runs where Xvfb is installed."""
+import base64
+import io
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+sys.path.insert(0, os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(HERE)), "sdk"))
+
+from openvibe_computer import Computer # noqa: E402
+from openvibe_computer.desk import PrivateDesk # noqa: E402
+from openvibe_plugin import Fault, Runtime # noqa: E402
+from test_computer import Out, reply # noqa: E402
+
+
+class FakeProc:
+ def __init__(self, argv, kw):
+ self.argv, self.kw = argv, kw
+ self.pid = 4242
+ self.code = None
+ self.stdout = io.StringIO("unix:path=/tmp/dbus-desk,guid=1\n")
+
+ def poll(self):
+ return self.code
+
+ def wait(self, timeout=None):
+ self.code = 0
+ return 0
+
+ def terminate(self):
+ self.code = -15
+
+ kill = terminate
+
+
+class DeskModuleTest(unittest.TestCase):
+ def make(self, have=("Xvfb", "dbus-daemon", "metacity")):
+ self.started = []
+ sockets = set()
+
+ def popen(argv, **kw):
+ p = FakeProc(argv, kw)
+ self.started.append(p)
+ if argv[0] == "Xvfb":
+ sockets.add("/tmp/.X11-unix/X%s" % argv[1][1:])
+ return p
+ desk = PrivateDesk(which=lambda n: "/usr/bin/" + n if n in have else None, popen=popen, sleep=lambda s: None,
+ exists=lambda path: path in sockets, base_env={"WAYLAND_DISPLAY": "wayland-0", "DISPLAY": ":0", "HOME": "/home/a"})
+ desk._kill = lambda proc: setattr(proc, "code", -15)
+ return desk
+
+ def test_start_status_env_stop(self):
+ desk = self.make()
+ st = desk.start(1024, 768)
+ self.assertEqual(st["display"], ":90")
+ self.assertEqual((st["width"], st["height"], st["window_manager"]), (1024, 768, "metacity"))
+ xvfb = self.started[0]
+ self.assertEqual(xvfb.argv, ["Xvfb", ":90", "-screen", "0", "1024x768x24", "-nolisten", "tcp", "-noreset"])
+ self.assertTrue(xvfb.kw["start_new_session"], "its own process group")
+ self.assertEqual([p.argv[0] for p in self.started], ["Xvfb", "dbus-daemon", "metacity"])
+ env = desk.env()
+ self.assertEqual((env["DISPLAY"], env["DBUS_SESSION_BUS_ADDRESS"]), (":90", "unix:path=/tmp/dbus-desk,guid=1"))
+ self.assertNotIn("WAYLAND_DISPLAY", env, "nothing goes to the owner's session")
+ self.assertEqual(self.started[2].kw["env"]["DISPLAY"], ":90", "the window manager runs on the desk")
+ self.assertEqual(desk.start(1024, 768)["display"], ":90", "starting again keeps the running desk")
+ self.assertEqual(len(self.started), 3)
+ self.assertEqual(desk.stop(), {"running": False, "stopped": True})
+ self.assertTrue(all(p.code is not None for p in self.started), "every process it started has ended")
+ self.assertEqual(desk.status(), {"running": False, "available": True})
+ with self.assertRaises(Fault) as e:
+ desk.env()
+ self.assertEqual(e.exception.code, "not_ready")
+
+ def test_without_xvfb_or_a_window_manager(self):
+ with self.assertRaises(Fault) as e:
+ self.make(have=()).start()
+ self.assertEqual(e.exception.code, "unsupported")
+ desk = self.make(have=("Xvfb",))
+ st = desk.start()
+ self.assertIsNone(st["window_manager"])
+ self.assertNotIn("DBUS_SESSION_BUS_ADDRESS", desk.env())
+
+
+class FakeDesk:
+ def __init__(self):
+ self.on = False
+
+ def available(self):
+ return True
+
+ def window_manager(self):
+ return "metacity"
+
+ def running(self):
+ return self.on
+
+ def start(self, width, height):
+ self.on, self.size = True, (width, height)
+ return self.status()
+
+ def stop(self):
+ was, self.on = self.on, False
+ return {"running": False, "stopped": was}
+
+ def status(self):
+ return {"running": True, "display": ":91", "width": 1000, "height": 500} if self.on else {"running": False, "available": True}
+
+ def env(self):
+ if not self.on:
+ raise Fault("not_ready", "start the private desk first (desk start)")
+ return {"DISPLAY": ":91"}
+
+
+class DeskPluginTest(unittest.TestCase):
+ def boot(self, level):
+ self.out = Out()
+ self.desk = FakeDesk()
+ self.calls = []
+
+ def popen(argv, **kw):
+ self.calls.append((argv, kw.get("env")))
+ p = FakeProc(argv, kw)
+ p.returncode = 0
+ p.communicate = lambda timeout=None: (b"", b"")
+ return p
+ self.plugin = Computer(env={"DISPLAY": ":0"}, platform="linux", desk=self.desk, popen=popen,
+ which=lambda n: "/usr/bin/" + n, screen_size=lambda tool: (2000, 1000))
+ self.rt = Runtime(self.plugin, infile=io.StringIO(""), outfile=self.out)
+ self.rt.dispatch({"op": "hello", "config": {"level": level, "roots": [tempfile.gettempdir()], "notify": False}})
+ self.n = 0
+ return self.out.msgs()
+
+ def cmd(self, kind, value):
+ self.n += 1
+ cid = "d%d" % self.n
+ self.rt.dispatch({"op": "command", "id": cid, "kind": kind, "value": value, "deadline_ms": 30000})
+ return cid
+
+ def test_describe_offers_the_desk(self):
+ caps = self.boot("trusted")[0]["capabilities"]
+ self.assertEqual(caps["desk"], {"level": "trusted", "window_manager": "metacity", "input": True, "screen": True})
+
+ def test_observe_reports_but_does_not_start(self):
+ self.boot("observe")
+ self.assertEqual(reply(self.out, self.cmd("desk", {"action": "status"}))["data"], {"running": False, "available": True})
+ r = reply(self.out, self.cmd("desk", {"action": "start"}))
+ self.assertEqual((r["op"], r["fault_code"]), ("nack", "not_allowed"))
+ self.assertFalse(self.desk.on)
+
+ def test_private_desk_input_screen_and_refusals(self):
+ self.boot("trusted")
+ r = reply(self.out, self.cmd("input", {"action": "click", "x": 10, "y": 10, "desk": "private"}))
+ self.assertEqual(r["fault_code"], "not_ready", "the desk has to run first")
+ r = reply(self.out, self.cmd("desk", {"action": "start", "width": 1000, "height": 500}))
+ self.assertEqual(r["data"]["running"], True)
+ r = reply(self.out, self.cmd("input", {"action": "click", "x": 50, "y": 50, "width": 100, "height": 100, "desk": "private"}))
+ self.assertEqual(r["op"], "ack")
+ argv, env = self.calls[-1]
+ self.assertEqual(argv, ["xdotool", "mousemove", "--sync", "500", "250", "click", "1"], "scaled to the desk's own 1000×500")
+ self.assertEqual(env["DISPLAY"], ":91", "on the desk, never the owner's :0")
+ r = reply(self.out, self.cmd("input", {"action": "click", "x": 50, "y": 50, "width": 100, "height": 100}))
+ argv, env = self.calls[-1]
+ self.assertEqual(argv[3:5], ["1000", "500"], "desk main is the owner's 2000×1000 screen")
+ self.assertIsNone(env)
+ r = reply(self.out, self.cmd("screen", {"desk": "neighbour"}))
+ self.assertEqual(r["fault_code"], "bad_value")
+ r = reply(self.out, self.cmd("desk", {"action": "stop"}))
+ self.assertEqual(r["data"], {"running": False, "stopped": True})
+
+ def test_the_kill_switch_stops_the_desk(self):
+ self.boot("trusted")
+ reply(self.out, self.cmd("desk", {"action": "start"}))
+ self.assertTrue(self.desk.on)
+ self.plugin.stop()
+ self.assertFalse(self.desk.on)
+
+
[email protected](all(shutil.which(t) for t in ("Xvfb", "import")) and sys.platform.startswith("linux"), "needs Xvfb and ImageMagick")
+class RealDeskTest(unittest.TestCase):
+ def test_a_real_desk_shows_its_own_screen(self):
+ out = Out()
+ plugin = Computer(platform="linux")
+ rt = Runtime(plugin, infile=io.StringIO(""), outfile=out)
+ rt.dispatch({"op": "hello", "config": {"level": "trusted", "roots": [tempfile.gettempdir()], "notify": False}})
+ self.addCleanup(plugin.desk.stop)
+ rt.dispatch({"op": "command", "id": "r1", "kind": "desk", "value": {"action": "start", "width": 800, "height": 600}, "deadline_ms": 30000})
+ st = reply(out, "r1")["data"]
+ self.assertTrue(st["running"], st)
+ rt.dispatch({"op": "command", "id": "r2", "kind": "screen", "value": {"desk": "private", "max_width": 800}, "deadline_ms": 30000})
+ shot = reply(out, "r2")["data"]
+ self.assertEqual((shot["width"], shot["height"]), (800, 600), "the desk's own 800×600, not the owner's screen")
+ self.assertTrue(base64.b64decode(shot["content_b64"]))
+ pids = [p.pid for _, p in plugin.desk.procs]
+ rt.dispatch({"op": "command", "id": "r3", "kind": "desk", "value": {"action": "stop"}, "deadline_ms": 30000})
+ self.assertEqual(reply(out, "r3")["data"]["stopped"], True)
+ for pid in pids:
+ with self.assertRaises(ProcessLookupError):
+ os.kill(pid, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()