openvibers/openvibe.link Commits

A private desk for agents (desk start/stop/status): an Xvfb display with its own bus and window manager; screen, live screen, input and the browser act there with desk: private, never on the owner's screen (#57)

OP OpenVibers committed 19 days ago · 621458f8c0 · 8 files, +548 −38

Changes

ddiff --git a/docs/computer.md b/docs/computer.md
iindex ad2b6c5..6cc6ab1 100644
---- a/docs/computer.md
++++ b/docs/computer.md
@@@ -124,6 +124,25 @@ text and keys are passed to the tool as arguments, never through a shell, and `k
- **Closing.** It closes after `browser_idle_s` without a command and at the kill switch.
- **Platforms.** Linux and macOS; not offered on Windows yet.
++### The private desk (`desk`, Node v0.15.0)
++
++An agent can work on a desk of its own instead of your screen (plan track K: Vibe's agent mode):
++- **What it is.** An Xvfb display on your machine with a window manager (metacity, openbox, xfwm4 or fluxbox, whichever
++ is installed) and a D-Bus session of its own, so apps opened there open there, not on your screen.
++- **Using it.** `desk` `{"action": "start", "width"?, "height"?}` (1280×800 by default), `stop` or `status`. Then
++ `screen`, `screen_stream`, `input` and `browser` take `"desk": "private"`: screenshots and the live screen of the desk,
++ the mouse and keyboard on the desk (points scale to the desk's own size), and the agent's browser shown on the desk
++ in a profile of its own (`browser_profile` + `-desk`). Without `desk` (or with `main`) they act on your own screen,
++ as before.
++- **Levels.** Starting and stopping a desk are acting commands (refused at `observe`, approved one by one at `ask`);
++ `status` runs at every level. What runs on the desk keeps its own rules: input and the browser still act.
++- **Watching and taking over.** Start the live screen with `"desk": "private"` (your desktop is not told, since nothing of
++ yours is shown) and drive it the same way.
++- **It ends** on `stop`, and at once on `openvibe-node stop` or an e-stop: every process the desk started is ended by
++ handle, never by name.
++- **Needs** Linux with `Xvfb` (Debian/Ubuntu: `apt install xvfb`), `xdotool` for input and ImageMagick `import` or `scrot`
++ for screenshots. Not offered elsewhere; `allow_desk: false` turns it off.
++
## Config
| key | default | meaning |
@@@ -138,6 +157,7 @@ text and keys are passed to the tool as arguments, never through a shell, and `k
| `notify` | `true` | the plugin's own prompt when a request waits and no tray shows it (graphical session only) |
| `probe` | `true` | `false`: no probes (OpenVibe.Watch's `node` source) |
| `allow_input` | `true` | `false`: never use the mouse or keyboard (no `input` capability) |
++| `allow_desk` | `true` | `false`: never start a private desk (no `desk` capability) |
| `allow_browser` | `true` | `false`: no browser (no `browser` capability) |
| `allow_screen_stream` | `true` | `false`: never show the screen live (`screen.stream` is false) |
| `browser_path` | found on PATH | the browser to drive (Chromium, Chrome, Edge or Brave) |
ddiff --git a/internal/plugins/manager.go b/internal/plugins/manager.go
iindex 2291466..9d6a302 100644
---- a/internal/plugins/manager.go
++++ b/internal/plugins/manager.go
@@@ -24,6 +24,7 @@ var capabilityFor = map[string]string{
protocol.KindProbe: "probe",
protocol.KindInput: "input",
protocol.KindBrowser: "browser",
++ protocol.KindDesk: "desk",
}
// Manager supervises every configured plugin.
ddiff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
iindex fb75b95..9c4121b 100644
---- a/internal/protocol/protocol.go
++++ b/internal/protocol/protocol.go
@@@ -175,13 +175,17 @@ const (
// machine's private OpenRestream stream); no operator sends them. The computer plugin captures the frames, the
// core publishes them; a latched stop ends it and refuses a start.
KindScreenStream = "screen_stream"
++ // KindDesk starts, stops or reports a private desk on the machine (contracts 0.214.0): {"action": "start"|"stop"|
++ // "status", "width"?, "height"?}, an Xvfb display of its own where an agent works without touching the owner's
++ // screen; screen, screen_stream, input and browser take "desk": "private" to act there. Starting and stopping act.
++ KindDesk = "desk"
)
// Kinds lists every command kind the Node understands.
--var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream}
++var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream, KindDesk}
// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.
--var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true}
++var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true, KindDesk: true}
const (
// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps
@@@ -194,8 +198,8 @@ const (
)
// ComputerActs reports whether a computer command changes the machine (runs a program, writes a file, uses the mouse
--// or keyboard, drives the browser): those are refused while the local stop or an e-stop is latched. Capturing the
--// screen, reading files and reading or closing the browser are not.
++// or keyboard, drives the browser, starts or stops the private desk): those are refused while the local stop or an
++// e-stop is latched. Capturing the screen, reading files, reading or closing the browser and the desk's status are not.
func ComputerActs(kind string, value json.RawMessage) bool {
switch kind {
case KindExec, KindInput:
@@@ -212,6 +216,12 @@ func ComputerActs(kind string, value json.RawMessage) bool {
}
_ = json.Unmarshal(value, &v)
return v.Action != "read" && v.Action != "screenshot" && v.Action != "close"
++ case KindDesk:
++ var v struct {
++ Action string `json:"action"`
++ }
++ _ = json.Unmarshal(value, &v)
++ return v.Action != "status"
}
return false
}
ddiff --git a/internal/protocol/protocol_test.go b/internal/protocol/protocol_test.go
iindex a19442c..16f7ad3 100644
---- a/internal/protocol/protocol_test.go
++++ b/internal/protocol/protocol_test.go
@@@ -396,6 +396,18 @@ func TestJobUndecodableBody(t *testing.T) {
}
}
++func TestComputerActsDesk(t *testing.T) {
++ cases := map[string]bool{"start": true, "stop": true, "status": false}
++ for action, acts := range cases {
++ if got := ComputerActs(KindDesk, json.RawMessage(`{"action":"`+action+`"}`)); got != acts {
++ t.Errorf("desk %s acts = %v, want %v", action, got, acts)
++ }
++ }
++ if !ComputerKinds[KindDesk] {
++ t.Error("desk is a computer kind (long deadline, no clamping)")
++ }
++}
++
func TestComputerActsBrowser(t *testing.T) {
want := map[string]bool{"open": true, "click": true, "fill": true, "press": true, "back": true, "read": false, "screenshot": false, "close": false}
for action, acts := range want {
ddiff --git a/plugins/computer/openvibe_computer/__init__.py b/plugins/computer/openvibe_computer/__init__.py
iindex 19c82c3..0072658 100644
---- a/plugins/computer/openvibe_computer/__init__.py
++++ b/plugins/computer/openvibe_computer/__init__.py
@@@ -40,6 +40,12 @@ Command kinds and their answers (the ack's ``data``):
way; ``screenshot`` → ``{format, width, height, content_b64}``; ``close``. Reading, screenshots and closing run at every
level; the rest are acting commands. Below level full, ``open`` refuses an address on the machine's own network.
Config ``allow_browser: false`` turns it off; ``browser_path``, ``browser_headless`` (default true).
++* ``desk`` ``{"action": "start"|"stop"|"status", "width"?, "height"?}``: a private desk (``desk.py``): an Xvfb display with a
++ window manager and a D-Bus session of its own, where the agent works without touching the owner's screen. ``screen``,
++ ``screen_stream``, ``input`` and ``browser`` take ``"desk": "private"`` to act there (the browser shown on the desk, in a
++ profile of its own, ``browser_profile`` + ``-desk``); ``main`` (the default) is the owner's screen. Starting and stopping
++ act (refused at observe, approved at ask), ``status`` runs at every level, and the kill switch stops the desk. Linux
++ with Xvfb; config ``allow_desk: false`` turns it off.
* ``probe`` ``{"target", "timeout_s"?}`` → for ``http(s)://…`` one GET, no redirect followed: ``{status, content_type,
etag, last_modified, body, truncated, ms}``; for ``tcp://host:port``: ``{open, ms, error?}``. Read-only, so it runs at
every level, and only towards the machine's own network: every address the name resolves to must be private,
@@@ -67,6 +73,7 @@ import urllib.parse
from openvibe_plugin import Deferred, Fault, Plugin
from .browser import KEY_CODES, Browser, BrowserError, find_binary
++from .desk import PrivateDesk
LEVELS = ("observe", "ask", "trusted", "full")
RANK = {name: i for i, name in enumerate(LEVELS)}
@@@ -132,12 +139,12 @@ def _int(v, name, lo, hi, default):
class Computer(Plugin):
driver = "computer"
-- version = "0.7.0"
++ version = "0.8.0"
motion_kinds = frozenset()
telemetry_interval_s = 2.0
def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.monotonic, env=None, platform=None, screen_size=None,
-- browser_factory=None, resolve=None):
++ browser_factory=None, resolve=None, desk=None):
self.which = which
self.popen = popen
self.screen_size = screen_size or self._screen_size
@@@ -174,7 +181,10 @@ class Computer(Plugin):
self.stream_mu = threading.Lock()
self.browser_mu = threading.Lock()
self.browser_op_mu = threading.Lock()
-- self.browser_factory = browser_factory or (lambda binary, profile, headless: Browser(binary, profile, headless))
++ self.browser_factory = browser_factory or (lambda binary, profile, headless, env=None: Browser(binary, profile, headless, env=env))
++ self.desk = desk or PrivateDesk(which=which, popen=popen)
++ self.allow_desk = True
++ self.desk_browser = None # the agent's browser on the private desk (its own profile, shown there, not headless)
self.resolve = resolve or (lambda host: [ai[4][0] for ai in socket.getaddrinfo(host, None)])
# ---- configuration -------------------------------------------------------------------------------------------
@@@ -199,6 +209,7 @@ class Computer(Plugin):
self.notify = config.get("notify", True) is not False
self.allow_browser = config.get("allow_browser", True) is not False
self.allow_screen_stream = config.get("allow_screen_stream", True) is not False
++ self.allow_desk = config.get("allow_desk", True) is not False
self.browser_path = config.get("browser_path") or None
self.browser_profile = os.path.expanduser(config.get("browser_profile") or "~/.local/share/openvibe-node/browser")
self.browser_headless = config.get("browser_headless", True) is not False
@@@ -224,6 +235,9 @@ class Computer(Plugin):
binary = self._browser_binary()
if binary:
caps["browser"] = {"level": self.level, "browser": os.path.basename(binary), "headless": self.browser_headless}
++ if self.allow_desk and self.platform.startswith("linux") and self.desk.available():
++ caps["desk"] = {"level": self.level, "window_manager": self.desk.window_manager(), "input": bool(self.which("xdotool")),
++ "screen": bool(self._x11_tool())}
return caps
def setup(self, config):
@@@ -248,6 +262,7 @@ class Computer(Plugin):
self._kill(p)
self._close_browser()
self._end_stream("stopped on the machine")
++ self.desk.stop()
for cid in waiting:
self._close_prompt(cid)
self.emit_event("approval_done", id=cid, approved=False, reason="stopped")
@@@ -309,14 +324,17 @@ class Computer(Plugin):
if not isinstance(v, dict):
raise Fault("bad_value", "value must be an object")
level = self.effective_level()
++ if cmd.kind == "desk":
++ return self._desk_op(cmd, v, level)
++ desk_env = self._desk_env(v) if cmd.kind in ("screen", "screen_stream", "input", "browser") else None
if cmd.kind == "screen":
-- tool = self._screen_tool()
++ tool = self._x11_tool() if desk_env else self._screen_tool()
if not tool:
raise Fault("unsupported", "no screen capture tool on this machine")
width = _int(v.get("max_width"), "max_width", 320, 3840, 1280)
-- return self._start(cmd, lambda: self._screen(tool, width))
++ return self._start(cmd, lambda: self._screen(tool, width, env=desk_env))
if cmd.kind == "screen_stream":
-- return self._screen_stream(v)
++ return self._screen_stream(v, env=desk_env)
if cmd.kind == "fs":
op = v.get("op")
if op in READ_OPS:
@@@ -333,13 +351,13 @@ class Computer(Plugin):
if cmd.kind == "input":
if not self.allow_input:
raise Fault("unsupported", "this machine does not allow mouse and keyboard input")
-- tool = self._input_tool()
++ tool = ("xdotool" if self.which("xdotool") else None) if desk_env else self._input_tool()
if not tool:
raise Fault("unsupported", "no input tool on this machine (xdotool on X11, cliclick on macOS)")
-- run, summary = self._input_op(tool, v)
++ run, summary = self._input_op(tool, v, env=desk_env)
return self._act(cmd, level, self._driving(run), summary, threaded=True)
if cmd.kind == "browser":
-- run, summary, acting = self._browser_op(v, level)
++ run, summary, acting = self._browser_op(v, level, env=desk_env)
if acting:
return self._act(cmd, level, run, summary, threaded=True)
return self._start(cmd, run)
@@@ -350,6 +368,51 @@ class Computer(Plugin):
return self._act(cmd, level, run, summary, threaded=True)
raise Fault("unsupported", "kind %s not supported" % cmd.kind)
++ # ---- the private desk --------------------------------------------------------------------------------------------
++ def _desk_env(self, v):
++ """None for the owner's own screen (desk main, the default); the private desk's environment for desk private."""
++ where = v.get("desk", "main")
++ if where not in ("main", "private"):
++ raise Fault("bad_value", "desk is main or private")
++ if where == "main":
++ return None
++ if not self.allow_desk:
++ raise Fault("unsupported", "this machine does not allow a private desk (allow_desk is off)")
++ return self.desk.env()
++
++ def _desk_op(self, cmd, v, level):
++ action = v.get("action")
++ if action == "status":
++ return self.desk.status()
++ if action not in ("start", "stop"):
++ raise Fault("bad_value", "action is start, stop or status")
++ if not self.allow_desk or not self.platform.startswith("linux"):
++ raise Fault("unsupported", "this machine does not offer a private desk (Linux with Xvfb, allow_desk on)")
++ if action == "start":
++ width = _int(v.get("width"), "width", 640, 3840, 1280)
++ height = _int(v.get("height"), "height", 480, 2160, 800)
++ return self._act(cmd, level, lambda: self.desk.start(width, height),
++ "start a private desk (%d×%d) where your agent works without touching your screen" % (width, height),
++ threaded=True)
++
++ def stop():
++ self._close_desk_browser()
++ return self.desk.stop()
++ return self._act(cmd, level, stop, "stop the private desk (and close what runs on it)", threaded=False)
++
++ def _close_desk_browser(self):
++ b, self.desk_browser = self.desk_browser, None
++ if b is not None:
++ b.close()
++
++ def _x11_tool(self):
++ """An X11 capture tool for the private desk (it is always X11, whatever the owner's session is)."""
++ if self.which("import"):
++ return ("import", lambda out: ["import", "-window", "root", out])
++ if self.which("scrot"):
++ return ("scrot", lambda out: ["scrot", "-o", out])
++ return None
++
def _act(self, cmd, level, run, summary, threaded):
if RANK[level] < RANK["ask"]:
stopped = self.runtime is not None and self.runtime.estopped
@@@ -828,7 +891,7 @@ class Computer(Plugin):
raise Fault("not_allowed", "%s is on this machine's own network: only at level full" % u.hostname)
return raw
-- def _browser_op(self, v, level):
++ def _browser_op(self, v, level, env=None):
"""(run, summary, acting) for one browser action; nothing is opened or clicked before run()."""
binary = self._browser_binary()
if not binary:
@@@ -854,13 +917,17 @@ class Computer(Plugin):
def browser():
with self.browser_mu:
++ if env: # on the private desk: shown there (never headless), in a profile of its own
++ if self.desk_browser is None or not self.desk_browser.alive():
++ self.desk_browser = self.browser_factory(binary, self.browser_profile + "-desk", False, env=env)
++ return self.desk_browser
if self.browser is None or not self.browser.alive():
self.browser = self.browser_factory(binary, self.browser_profile, self.browser_headless)
return self.browser
def run():
if action == "close":
-- self._close_browser()
++ self._close_desk_browser() if env else self._close_browser()
return {"done": True}
b = browser()
try:
@@@ -904,8 +971,9 @@ class Computer(Plugin):
return "xdotool"
return None
-- def _input_op(self, tool, v):
-- """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask)."""
++ def _input_op(self, tool, v, env=None):
++ """(run, summary) for one input action; nothing touches the screen until run() (after approval at ask). With env,
++ on the private desk (its display; points scale to the desk's own size)."""
action = v.get("action")
if action not in INPUT_ACTIONS:
raise Fault("bad_value", "action is one of %s" % ", ".join(sorted(INPUT_ACTIONS)))
@@@ -936,9 +1004,10 @@ class Computer(Plugin):
raise Fault("bad_value", "keys is one key or chord, for example Return or ctrl+s")
def run():
-- at = self._scale(tool, point) if point else None
++ at = (self._desk_scale(point) if env else self._scale(tool, point)) if point else None
argv = self._input_argv(tool, action, at, v)
-- proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True)
++ proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True,
++ **({"env": env} if env else {}))
try:
_, err = proc.communicate(timeout=30)
except subprocess.TimeoutExpired:
@@@ -951,19 +1020,27 @@ class Computer(Plugin):
out["x"], out["y"] = at
return out
++ where = " on the private desk" if env else ""
if action == "type":
text = v["text"]
shown = text if len(text) <= 120 else text[:120] + "…"
-- summary = "type %d characters: %r" % (len(text), shown)
++ summary = "type %d characters%s: %r" % (len(text), where, shown)
elif action == "key":
-- summary = "press %s" % v["keys"]
++ summary = "press %s%s" % (v["keys"], where)
elif action == "scroll":
-- summary = "scroll %s %d" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]))
++ summary = "scroll %s %d%s" % ("up" if v["dy"] < 0 else "down", abs(v["dy"]), where)
else:
x, y, w, h = point
-- summary = "%s at %d, %d%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "")
++ summary = "%s at %d, %d%s%s" % (action.replace("_", " "), x, y, " of a %d×%d picture of the screen" % (w, h) if w else "", where)
return run, summary
++ def _desk_scale(self, point):
++ x, y, w, h = point
++ st = self.desk.status()
++ if not w or not st.get("running"):
++ return x, y
++ return min(st["width"] - 1, round(x * st["width"] / w)), min(st["height"] - 1, round(y * st["height"] / h))
++
def _screen_size(self, tool):
"""The screen's size in the units the input tool uses: pixels for xdotool, points for cliclick (a Retina
screen's picture is twice as many pixels)."""
@@@ -1044,12 +1121,12 @@ class Computer(Plugin):
return ("gnome-screenshot", lambda out: ["gnome-screenshot", "-f", out])
return None
-- def _screen(self, tool, max_width):
++ def _screen(self, tool, max_width, env=None):
fd, out = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")
os.close(fd)
try:
proc = self.popen(tool[1](out), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
-- stderr=subprocess.PIPE, start_new_session=True)
++ stderr=subprocess.PIPE, start_new_session=True, **({"env": env} if env else {}))
try:
_, err = proc.communicate(timeout=15)
except subprocess.TimeoutExpired:
@@@ -1088,7 +1165,7 @@ class Computer(Plugin):
def _stream_capable(self, tool):
return self._have_pillow() or self._native_jpeg(tool, "x.jpg") is not None
-- def _screen_stream(self, v):
++ def _screen_stream(self, v, env=None):
action = v.get("action")
if action == "stop":
return {"streaming": False, "stopped": self._end_stream("stopped by its owner")}
@@@ -1098,7 +1175,7 @@ class Computer(Plugin):
raise Fault("unsupported", "this machine does not show its screen live (allow_screen_stream is off)")
if self.runtime is not None and self.runtime.estopped:
raise Fault("local_stop", "stopped on the machine")
-- tool = self._screen_tool()
++ tool = self._x11_tool() if env else self._screen_tool()
if not tool:
raise Fault("unsupported", "no screen capture tool on this machine")
if not self._stream_capable(tool):
@@@ -1108,12 +1185,14 @@ class Computer(Plugin):
self._end_stream("replaced")
frames = tempfile.mkdtemp(prefix="openvibe-screen-stream-") # 0700, this account only
stream = {"sid": os.urandom(8).hex(), "until": self.clock() + seconds, "fps": fps, "tool": tool, "dir": frames,
-- "stop": threading.Event()}
++ "stop": threading.Event(), "env": env}
with self.stream_mu:
self.stream = stream
threading.Thread(target=self._stream_loop, args=(stream,), name="computer-screen-stream", daemon=True).start()
-- self._notify_stream(seconds)
-- return {"streaming": True, "sid": stream["sid"], "seconds": seconds, "fps": fps, "tool": tool[0]}
++ if not env: # the private desk is not the owner's screen: nothing of theirs goes live
++ self._notify_stream(seconds)
++ return {"streaming": True, "sid": stream["sid"], "seconds": seconds, "fps": fps, "tool": tool[0],
++ "desk": "private" if env else "main"}
def _end_stream(self, reason):
"""End the live screen, if one runs → whether one did."""
@@@ -1135,7 +1214,7 @@ class Computer(Plugin):
break
path = os.path.join(stream["dir"], "frame-%d.jpg" % (n % 3)) # the core reads each one at once
try:
-- self._stream_frame(stream["tool"], path)
++ self._stream_frame(stream["tool"], path, env=stream.get("env"))
n += 1
failures = 0
self.emit_video(path, stream="screen")
@@@ -1153,13 +1232,13 @@ class Computer(Plugin):
# The core ends the stream with the same id (a newer one that replaced it carries its own).
self.emit_event("screen_stream_ended", sid=stream["sid"], reason=stream.get("reason") or reason, frames=n)
-- def _stream_frame(self, tool, path):
++ def _stream_frame(self, tool, path, env=None):
"""One JPEG frame of the screen at path, at most STREAM_WIDTH wide when Pillow can resize it."""
if self._have_pillow():
fd, png = tempfile.mkstemp(prefix="openvibe-screen-", suffix=".png")
os.close(fd)
try:
-- self._capture(tool[1](png))
++ self._capture(tool[1](png), env=env)
from PIL import Image
with Image.open(png) as im:
im = im.convert("RGB")
@@@ -1171,12 +1250,12 @@ class Computer(Plugin):
if os.path.exists(png):
os.unlink(png)
return
-- self._capture(self._native_jpeg(tool, path + ".part.jpg"))
++ self._capture(self._native_jpeg(tool, path + ".part.jpg"), env=env)
os.replace(path + ".part.jpg", path)
-- def _capture(self, argv):
++ def _capture(self, argv, env=None):
proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
-- start_new_session=True)
++ start_new_session=True, **({"env": env} if env else {}))
try:
_, err = proc.communicate(timeout=10)
except subprocess.TimeoutExpired:
ddiff --git a/plugins/computer/openvibe_computer/browser.py b/plugins/computer/openvibe_computer/browser.py
iindex 8b13600..8ae04ab 100644
---- a/plugins/computer/openvibe_computer/browser.py
++++ b/plugins/computer/openvibe_computer/browser.py
@@@ -91,8 +91,9 @@ def find_binary(which, platform, configured=None):
class Browser:
-- def __init__(self, binary, profile, headless=True, timeout_s=15.0, popen=subprocess.Popen, extra_args=()):
++ def __init__(self, binary, profile, headless=True, timeout_s=15.0, popen=subprocess.Popen, extra_args=(), env=None):
self.binary = binary
++ self.env = env # the private desk's display and bus, or None for this process's own
self.extra_args = list(extra_args)
self.stderr_tail = []
self.profile = profile
@@@ -136,7 +137,7 @@ class Browser:
os.dup2(w, 4)
self.proc = self.popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
-- pass_fds=(3, 4), preexec_fn=fds, start_new_session=True)
++ pass_fds=(3, 4), preexec_fn=fds, start_new_session=True, **({"env": self.env} if self.env else {}))
threading.Thread(target=self._stderr, args=(self.proc.stderr,), name="computer-browser-err", daemon=True).start()
os.close(to_chrome_r)
os.close(from_chrome_w)
ddiff --git a/plugins/computer/openvibe_computer/desk.py b/plugins/computer/openvibe_computer/desk.py
nnew file mode 100644
iindex 0000000..a484bcd
---- /dev/null
++++ b/plugins/computer/openvibe_computer/desk.py
@@@ -0,0 +1,177 @@
++"""The private desk: a display of the machine's own where an agent works without touching its owner's screen.
++
++An Xvfb display with a window manager (when one is installed) and a D-Bus session of its own. The session matters:
++apps already open on the owner's desktop (most browsers and GNOME apps) are single-instance over D-Bus, and without a
++separate bus "open it on the private desk" would quietly open a window on the owner's screen instead. The owner watches
++it through the plugin's live screen (``screen_stream`` with ``desk: private``) and takes over by driving it the same way.
++
++Every process the desk starts is its own process group, kept by handle and ended by handle (never by name), so stopping
++the desk ends exactly what it started. Nothing here touches the owner's display; its address is never read.
++
++Brought in from the owner's desktop-mcp (``desks.py``), plan track K step 2 (Vibe's desk on Node).
++"""
++from __future__ import annotations
++
++import os
++import shutil
++import signal
++import subprocess
++import threading
++import time
++
++from openvibe_plugin import Fault
++
++WINDOW_MANAGERS = ("metacity", "openbox", "xfwm4", "fluxbox")
++DISPLAYS = range(90, 100)
++START_TIMEOUT_S = 8.0
++
++
++class PrivateDesk:
++ def __init__(self, which=shutil.which, popen=subprocess.Popen, clock=time.time, sleep=time.sleep,
++ exists=os.path.exists, base_env=None, displays=DISPLAYS):
++ self.which = which
++ self.popen = popen
++ self.clock = clock
++ self.sleep = sleep
++ self.exists = exists
++ self.base_env = dict(os.environ if base_env is None else base_env)
++ self.displays = displays
++ self.mu = threading.Lock()
++ self.procs = [] # (name, Popen) in start order: Xvfb, then the bus, then the window manager
++ self.display = None
++ self.width = None
++ self.height = None
++ self.since = None
++ self.bus = None
++ self.wm = None
++
++ # ---- what this machine has -------------------------------------------------------------------------------------
++ def available(self):
++ return bool(self.which("Xvfb"))
++
++ def window_manager(self):
++ for name in WINDOW_MANAGERS:
++ if self.which(name):
++ return name
++ return None
++
++ def running(self):
++ with self.mu:
++ return self._running()
++
++ def _running(self):
++ return bool(self.procs) and self.procs[0][1].poll() is None
++
++ # ---- start, stop, status --------------------------------------------------------------------------------------
++ def status(self):
++ with self.mu:
++ if not self._running():
++ return {"running": False, "available": self.available()}
++ return {"running": True, "display": ":%d" % self.display, "width": self.width, "height": self.height,
++ "window_manager": self.wm, "since": int(self.since)}
++
++ def start(self, width=1280, height=800):
++ with self.mu:
++ if self._running():
++ return self._status_locked()
++ self._end_locked()
++ if not self.available():
++ raise Fault("unsupported", "a private desk needs Xvfb on this machine (Debian/Ubuntu: apt install xvfb)")
++ n = self._free_display()
++ xvfb = self.popen(["Xvfb", ":%d" % n, "-screen", "0", "%dx%dx24" % (width, height), "-nolisten", "tcp",
++ "-noreset"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
++ start_new_session=True)
++ self.procs = [("Xvfb", xvfb)]
++ if not self._wait_for(lambda: self.exists("/tmp/.X11-unix/X%d" % n) or xvfb.poll() is not None):
++ self._end_locked()
++ raise Fault("hardware", "the private desk did not start in %d s" % START_TIMEOUT_S)
++ if xvfb.poll() is not None:
++ self._end_locked()
++ raise Fault("hardware", "Xvfb ended as it started (display :%d)" % n)
++ self.display, self.width, self.height, self.since = n, width, height, self.clock()
++ self.bus = self._start_bus()
++ self.wm = self.window_manager()
++ if self.wm:
++ self.procs.append((self.wm, self.popen([self.wm], env=self._env_locked(), stdin=subprocess.DEVNULL,
++ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
++ start_new_session=True)))
++ return self._status_locked()
++
++ def stop(self):
++ with self.mu:
++ was = self._running()
++ self._end_locked()
++ return {"running": False, "stopped": was}
++
++ def env(self):
++ """The environment for a tool or app that should run on the desk (its display and its own bus)."""
++ with self.mu:
++ if not self._running():
++ raise Fault("not_ready", "start the private desk first (desk start)")
++ return self._env_locked()
++
++ # ---- internals ------------------------------------------------------------------------------------------------
++ def _status_locked(self):
++ return {"running": True, "display": ":%d" % self.display, "width": self.width, "height": self.height,
++ "window_manager": self.wm, "since": int(self.since)}
++
++ def _env_locked(self):
++ env = {k: v for k, v in self.base_env.items() if k not in ("WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "XAUTHORITY")}
++ env["DISPLAY"] = ":%d" % self.display
++ if self.bus:
++ env["DBUS_SESSION_BUS_ADDRESS"] = self.bus
++ return env
++
++ def _free_display(self):
++ for n in self.displays:
++ if not self.exists("/tmp/.X%d-lock" % n) and not self.exists("/tmp/.X11-unix/X%d" % n):
++ return n
++ raise Fault("not_ready", "no free display number for a private desk (:%d to :%d are taken)" % (self.displays[0], self.displays[-1]))
++
++ def _wait_for(self, ready):
++ end = self.clock() + START_TIMEOUT_S
++ while self.clock() < end:
++ if ready():
++ return True
++ self.sleep(0.05)
++ return ready()
++
++ def _start_bus(self):
++ """A D-Bus session of the desk's own; its address, or None where there is no dbus-daemon (apps then share none)."""
++ if not self.which("dbus-daemon"):
++ return None
++ env = self._env_locked()
++ proc = self.popen(["dbus-daemon", "--session", "--nofork", "--print-address=1"], env=env, stdin=subprocess.DEVNULL,
++ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, start_new_session=True, text=True)
++ self.procs.append(("dbus-daemon", proc))
++ got = []
++ reader = threading.Thread(target=lambda: got.append(proc.stdout.readline().strip()), daemon=True)
++ reader.start()
++ reader.join(3.0)
++ address = got[0] if got and got[0].startswith(("unix:", "tcp:")) else None
++ return address
++
++ def _end_locked(self):
++ for _, proc in reversed(self.procs):
++ self._kill(proc)
++ self.procs = []
++ self.display = self.width = self.height = self.since = self.bus = self.wm = None
++
++ @staticmethod
++ def _kill(proc):
++ if proc.poll() is not None:
++ return
++ try:
++ if hasattr(os, "killpg"):
++ os.killpg(proc.pid, signal.SIGTERM)
++ else:
++ proc.terminate()
++ proc.wait(timeout=3)
++ except Exception: # noqa: BLE001 - still running or already gone: make sure it is gone
++ try:
++ if hasattr(os, "killpg"):
++ os.killpg(proc.pid, signal.SIGKILL)
++ else:
++ proc.kill()
++ except Exception: # noqa: BLE001
++ pass
ddiff --git a/plugins/computer/tests/test_desk.py b/plugins/computer/tests/test_desk.py
nnew file mode 100644
iindex 0000000..51df8b6
---- /dev/null
++++ b/plugins/computer/tests/test_desk.py
@@@ -0,0 +1,210 @@
++"""The private desk (openvibe_computer/desk.py and the plugin's desk kind, plan track K step 2): it starts an Xvfb display
++with its own bus and window manager and stops exactly what it started; the plugin starts and stops it under the
++machine's level (refused at observe), reports it at every level, and sends screen, input and the browser there with
++desk: private (refused until it runs), never to the owner's screen. A real desk runs where Xvfb is installed."""
++import base64
++import io
++import os
++import shutil
++import sys
++import tempfile
++import unittest
++
++HERE = os.path.dirname(os.path.abspath(__file__))
++sys.path.insert(0, HERE)
++sys.path.insert(0, os.path.dirname(HERE))
++sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(HERE)), "sdk"))
++
++from openvibe_computer import Computer # noqa: E402
++from openvibe_computer.desk import PrivateDesk # noqa: E402
++from openvibe_plugin import Fault, Runtime # noqa: E402
++from test_computer import Out, reply # noqa: E402
++
++
++class FakeProc:
++ def __init__(self, argv, kw):
++ self.argv, self.kw = argv, kw
++ self.pid = 4242
++ self.code = None
++ self.stdout = io.StringIO("unix:path=/tmp/dbus-desk,guid=1\n")
++
++ def poll(self):
++ return self.code
++
++ def wait(self, timeout=None):
++ self.code = 0
++ return 0
++
++ def terminate(self):
++ self.code = -15
++
++ kill = terminate
++
++
++class DeskModuleTest(unittest.TestCase):
++ def make(self, have=("Xvfb", "dbus-daemon", "metacity")):
++ self.started = []
++ sockets = set()
++
++ def popen(argv, **kw):
++ p = FakeProc(argv, kw)
++ self.started.append(p)
++ if argv[0] == "Xvfb":
++ sockets.add("/tmp/.X11-unix/X%s" % argv[1][1:])
++ return p
++ desk = PrivateDesk(which=lambda n: "/usr/bin/" + n if n in have else None, popen=popen, sleep=lambda s: None,
++ exists=lambda path: path in sockets, base_env={"WAYLAND_DISPLAY": "wayland-0", "DISPLAY": ":0", "HOME": "/home/a"})
++ desk._kill = lambda proc: setattr(proc, "code", -15)
++ return desk
++
++ def test_start_status_env_stop(self):
++ desk = self.make()
++ st = desk.start(1024, 768)
++ self.assertEqual(st["display"], ":90")
++ self.assertEqual((st["width"], st["height"], st["window_manager"]), (1024, 768, "metacity"))
++ xvfb = self.started[0]
++ self.assertEqual(xvfb.argv, ["Xvfb", ":90", "-screen", "0", "1024x768x24", "-nolisten", "tcp", "-noreset"])
++ self.assertTrue(xvfb.kw["start_new_session"], "its own process group")
++ self.assertEqual([p.argv[0] for p in self.started], ["Xvfb", "dbus-daemon", "metacity"])
++ env = desk.env()
++ self.assertEqual((env["DISPLAY"], env["DBUS_SESSION_BUS_ADDRESS"]), (":90", "unix:path=/tmp/dbus-desk,guid=1"))
++ self.assertNotIn("WAYLAND_DISPLAY", env, "nothing goes to the owner's session")
++ self.assertEqual(self.started[2].kw["env"]["DISPLAY"], ":90", "the window manager runs on the desk")
++ self.assertEqual(desk.start(1024, 768)["display"], ":90", "starting again keeps the running desk")
++ self.assertEqual(len(self.started), 3)
++ self.assertEqual(desk.stop(), {"running": False, "stopped": True})
++ self.assertTrue(all(p.code is not None for p in self.started), "every process it started has ended")
++ self.assertEqual(desk.status(), {"running": False, "available": True})
++ with self.assertRaises(Fault) as e:
++ desk.env()
++ self.assertEqual(e.exception.code, "not_ready")
++
++ def test_without_xvfb_or_a_window_manager(self):
++ with self.assertRaises(Fault) as e:
++ self.make(have=()).start()
++ self.assertEqual(e.exception.code, "unsupported")
++ desk = self.make(have=("Xvfb",))
++ st = desk.start()
++ self.assertIsNone(st["window_manager"])
++ self.assertNotIn("DBUS_SESSION_BUS_ADDRESS", desk.env())
++
++
++class FakeDesk:
++ def __init__(self):
++ self.on = False
++
++ def available(self):
++ return True
++
++ def window_manager(self):
++ return "metacity"
++
++ def running(self):
++ return self.on
++
++ def start(self, width, height):
++ self.on, self.size = True, (width, height)
++ return self.status()
++
++ def stop(self):
++ was, self.on = self.on, False
++ return {"running": False, "stopped": was}
++
++ def status(self):
++ return {"running": True, "display": ":91", "width": 1000, "height": 500} if self.on else {"running": False, "available": True}
++
++ def env(self):
++ if not self.on:
++ raise Fault("not_ready", "start the private desk first (desk start)")
++ return {"DISPLAY": ":91"}
++
++
++class DeskPluginTest(unittest.TestCase):
++ def boot(self, level):
++ self.out = Out()
++ self.desk = FakeDesk()
++ self.calls = []
++
++ def popen(argv, **kw):
++ self.calls.append((argv, kw.get("env")))
++ p = FakeProc(argv, kw)
++ p.returncode = 0
++ p.communicate = lambda timeout=None: (b"", b"")
++ return p
++ self.plugin = Computer(env={"DISPLAY": ":0"}, platform="linux", desk=self.desk, popen=popen,
++ which=lambda n: "/usr/bin/" + n, screen_size=lambda tool: (2000, 1000))
++ self.rt = Runtime(self.plugin, infile=io.StringIO(""), outfile=self.out)
++ self.rt.dispatch({"op": "hello", "config": {"level": level, "roots": [tempfile.gettempdir()], "notify": False}})
++ self.n = 0
++ return self.out.msgs()
++
++ def cmd(self, kind, value):
++ self.n += 1
++ cid = "d%d" % self.n
++ self.rt.dispatch({"op": "command", "id": cid, "kind": kind, "value": value, "deadline_ms": 30000})
++ return cid
++
++ def test_describe_offers_the_desk(self):
++ caps = self.boot("trusted")[0]["capabilities"]
++ self.assertEqual(caps["desk"], {"level": "trusted", "window_manager": "metacity", "input": True, "screen": True})
++
++ def test_observe_reports_but_does_not_start(self):
++ self.boot("observe")
++ self.assertEqual(reply(self.out, self.cmd("desk", {"action": "status"}))["data"], {"running": False, "available": True})
++ r = reply(self.out, self.cmd("desk", {"action": "start"}))
++ self.assertEqual((r["op"], r["fault_code"]), ("nack", "not_allowed"))
++ self.assertFalse(self.desk.on)
++
++ def test_private_desk_input_screen_and_refusals(self):
++ self.boot("trusted")
++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 10, "y": 10, "desk": "private"}))
++ self.assertEqual(r["fault_code"], "not_ready", "the desk has to run first")
++ r = reply(self.out, self.cmd("desk", {"action": "start", "width": 1000, "height": 500}))
++ self.assertEqual(r["data"]["running"], True)
++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 50, "y": 50, "width": 100, "height": 100, "desk": "private"}))
++ self.assertEqual(r["op"], "ack")
++ argv, env = self.calls[-1]
++ self.assertEqual(argv, ["xdotool", "mousemove", "--sync", "500", "250", "click", "1"], "scaled to the desk's own 1000×500")
++ self.assertEqual(env["DISPLAY"], ":91", "on the desk, never the owner's :0")
++ r = reply(self.out, self.cmd("input", {"action": "click", "x": 50, "y": 50, "width": 100, "height": 100}))
++ argv, env = self.calls[-1]
++ self.assertEqual(argv[3:5], ["1000", "500"], "desk main is the owner's 2000×1000 screen")
++ self.assertIsNone(env)
++ r = reply(self.out, self.cmd("screen", {"desk": "neighbour"}))
++ self.assertEqual(r["fault_code"], "bad_value")
++ r = reply(self.out, self.cmd("desk", {"action": "stop"}))
++ self.assertEqual(r["data"], {"running": False, "stopped": True})
++
++ def test_the_kill_switch_stops_the_desk(self):
++ self.boot("trusted")
++ reply(self.out, self.cmd("desk", {"action": "start"}))
++ self.assertTrue(self.desk.on)
++ self.plugin.stop()
++ self.assertFalse(self.desk.on)
++
++
+[email protected](all(shutil.which(t) for t in ("Xvfb", "import")) and sys.platform.startswith("linux"), "needs Xvfb and ImageMagick")
++class RealDeskTest(unittest.TestCase):
++ def test_a_real_desk_shows_its_own_screen(self):
++ out = Out()
++ plugin = Computer(platform="linux")
++ rt = Runtime(plugin, infile=io.StringIO(""), outfile=out)
++ rt.dispatch({"op": "hello", "config": {"level": "trusted", "roots": [tempfile.gettempdir()], "notify": False}})
++ self.addCleanup(plugin.desk.stop)
++ rt.dispatch({"op": "command", "id": "r1", "kind": "desk", "value": {"action": "start", "width": 800, "height": 600}, "deadline_ms": 30000})
++ st = reply(out, "r1")["data"]
++ self.assertTrue(st["running"], st)
++ rt.dispatch({"op": "command", "id": "r2", "kind": "screen", "value": {"desk": "private", "max_width": 800}, "deadline_ms": 30000})
++ shot = reply(out, "r2")["data"]
++ self.assertEqual((shot["width"], shot["height"]), (800, 600), "the desk's own 800×600, not the owner's screen")
++ self.assertTrue(base64.b64decode(shot["content_b64"]))
++ pids = [p.pid for _, p in plugin.desk.procs]
++ rt.dispatch({"op": "command", "id": "r3", "kind": "desk", "value": {"action": "stop"}, "deadline_ms": 30000})
++ self.assertEqual(reply(out, "r3")["data"]["stopped"], True)
++ for pid in pids:
++ with self.assertRaises(ProcessLookupError):
++ os.kill(pid, 0)
++
++
++if __name__ == "__main__":
++ unittest.main()