openvibers/openvibe.link Commits
Agent Streams S1: the desk stream, curtain and latency probe (#109)
Changes
@@@ -84,6 +84,9 @@ Usage:openvibe-node memory scan [--tools a,b] [--project DIR …] [--json]read the memory files of the coding tools on this machine(read-only, local, secrets removed)++ openvibe-node stream --probe --whip-url URL [--seconds N] [--size WxH] [--json]++ publish a QR of the millisecond clock for the headless latency viewer (the ticket comes from++ OPENVIBE_STREAM_TICKET, never argv)openvibe-node share --key ovc_… --port N put this machine's port N on the web (OpenVibe Connect),behind OpenVibe sign-in, until Ctrl-Copenvibe-node share add|list|remove … the shares the Node service keeps up (openvibe-node share)@@@ -148,6 +151,9 @@ func runWithStdin(args []string, stdin io.Reader, stdout, stderr io.Writer) intif cmd == "desk-sandbox" { // its own flags, up to "--": a browser's flags after it are never the Node'sreturn cmdDeskSandbox(rest, stderr)}++ if cmd == "stream" { // its own flags: --probe and the latency probe's++ return cmdStream(rest, stdin, stdout, stderr)++ }fs := flag.NewFlagSet(cmd, flag.ContinueOnError)fs.SetOutput(stderr)g := &globals{}nnew file mode 100644@@@ -0,0 +1,97 @@++package main++++import (++ "context"++ "encoding/json"++ "errors"++ "flag"++ "fmt"++ "io"++ "os"++ "os/signal"++ "strings"++ "syscall"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++ "github.com/OpenVibers/OpenVibe.Node/internal/stream"++)++++// cmdStream is `openvibe-node stream --probe`: it publishes a QR of the millisecond clock over WHIP, so a headless++// viewer can report the glass-to-glass latency. The ticket comes from the environment or stdin, never from argv.++func cmdStream(args []string, stdin io.Reader, stdout, stderr io.Writer) int {++ fs := flag.NewFlagSet("stream", flag.ContinueOnError)++ fs.SetOutput(stderr)++ probe := fs.Bool("probe", false, "publish the QR millisecond clock (the latency probe)")++ whip := fs.String("whip-url", "", "the stream's WHIP URL (https://whip.openvibe.live/whip/agent/ags_…)")++ seconds := fs.Int("seconds", stream.ProbeDefaultSeconds, "how long to publish (seconds)")++ size := fs.String("size", "640x360", "picture size, WxH")++ fps := fs.Int("fps", stream.ProbeDefaultFPS, "frames per second")++ loopback := fs.Bool("loopback", false, "offer loopback ICE candidates (a WHIP endpoint on this machine)")++ ticketStdin := fs.Bool("ticket-stdin", false, "read the publish ticket from one line on stdin")++ asJSON := fs.Bool("json", false, "print JSON")++ if err := fs.Parse(args); err != nil {++ return 2++ }++ if fs.NArg() > 0 {++ fmt.Fprintln(stderr, "error: stream takes no arguments")++ return 2++ }++ if !*probe {++ fmt.Fprintln(stderr, "usage: openvibe-node stream --probe --whip-url URL [--seconds N] [--size WxH] [--json]")++ fmt.Fprintln(stderr, "the publish ticket is read from OPENVIBE_STREAM_TICKET, or with --ticket-stdin from stdin (never argv)")++ return 2++ }++ width, height, err := parseSize(*size)++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return 2++ }++ ticket, err := probeTicket(stdin, *ticketStdin)++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return 2++ }++ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)++ defer stop()++ n, err := stream.Probe(ctx, stream.ProbeOptions{WHIPURL: *whip, Ticket: ticket, Seconds: *seconds,++ Width: width, Height: height, FPS: *fps, Loopback: *loopback})++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return 1++ }++ if *asJSON {++ b, _ := json.Marshal(map[string]any{"frames": n, "seconds": *seconds, "size": fmt.Sprintf("%dx%d", width, height),++ "whip_url": *whip})++ fmt.Fprintln(stdout, string(b))++ return 0++ }++ fmt.Fprintf(stdout, "probe: published %d frames in %ds at %dx%d\n", n, *seconds, width, height)++ fmt.Fprintln(stdout, "the latency is measured by scripts/stream-viewer.py against the same stream; it reports p50/p95/max")++ return 0++}++++// probeTicket reads the publish ticket. It is a secret: the environment or stdin, never a flag that `ps` can show.++func probeTicket(stdin io.Reader, fromStdin bool) (credentials.Secret, error) {++ t := strings.TrimSpace(os.Getenv("OPENVIBE_STREAM_TICKET"))++ if t == "" && fromStdin && stdin != nil {++ // Read at most one line, so a pipe cannot hang the probe.++ buf := make([]byte, 256)++ n, _ := stdin.Read(buf)++ t = strings.TrimSpace(string(buf[:n]))++ }++ if t == "" {++ return credentials.Secret{}, errors.New("set OPENVIBE_STREAM_TICKET to the stream's wpt_… ticket (or pass --ticket-stdin)")++ }++ if !strings.HasPrefix(t, "wpt_") || len(t) < 8 {++ return credentials.Secret{}, errors.New("the publish ticket must be a wpt_… value (OPENVIBE_STREAM_TICKET)")++ }++ return credentials.NewSecret(t), nil++}++++func parseSize(s string) (int, int, error) {++ var w, h int++ if _, err := fmt.Sscanf(s, "%dx%d", &w, &h); err != nil || w < 160 || h < 90 || w > 3840 || h > 2160 {++ return 0, 0, fmt.Errorf("size %q is WxH between 160x90 and 3840x2160", s)++ }++ return w &^ 15, h &^ 15, nil++}nnew file mode 100644@@@ -0,0 +1,53 @@++package main++++import (++ "bytes"++ "strings"++ "testing"++)++++func TestParseSize(t *testing.T) {++ if w, h, err := parseSize("640x360"); err != nil || w != 640 || h != 352 {++ t.Fatalf("parseSize = %d,%d,%v", w, h, err)++ }++ if _, _, err := parseSize("1x1"); err == nil {++ t.Fatal("a tiny size was accepted")++ }++ if _, _, err := parseSize("nonsense"); err == nil {++ t.Fatal("nonsense was accepted")++ }++}++++func TestTheProbeTicketComesFromTheEnvironmentOrStdinNeverArgv(t *testing.T) {++ t.Setenv("OPENVIBE_STREAM_TICKET", "wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG")++ tk, err := probeTicket(nil, false)++ if err != nil || tk.Reveal() == "" {++ t.Fatalf("env ticket: %v", err)++ }++ t.Setenv("OPENVIBE_STREAM_TICKET", "")++ tk, err = probeTicket(strings.NewReader("wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG\n"), true)++ if err != nil || tk.Reveal() == "" {++ t.Fatalf("stdin ticket: %v", err)++ }++ if _, err := probeTicket(nil, false); err == nil {++ t.Fatal("no ticket was accepted")++ }++ if _, err := probeTicket(strings.NewReader("not-a-ticket\n"), true); err == nil {++ t.Fatal("a bad ticket was accepted")++ }++}++++func TestCmdStreamNeedsAURLAndATicket(t *testing.T) {++ t.Setenv("OPENVIBE_STREAM_TICKET", "")++ var out, errb bytes.Buffer++ if code := cmdStream([]string{"--probe", "--seconds", "1"}, nil, &out, &errb); code != 2 {++ t.Fatalf("missing URL exit = %d", code)++ }++ if strings.Contains(out.String()+errb.String(), "wpt_0123456789") {++ t.Fatal("a ticket was printed")++ }++ if code := cmdStream([]string{"--probe", "--whip-url", "https://whip.openvibe.live/whip/agent/ags_x", "--seconds", "1"},++ nil, &out, &errb); code != 2 {++ t.Fatalf("missing ticket exit = %d", code)++ }++}nnew file mode 100644@@@ -0,0 +1,129 @@++# The desk stream (Agent Streams S1, Node's part)++++An agent's desk can stream on OpenVibe.Live. The Node captures the desk, encodes it with the screen recipe, publishes++it over WHIP with a publish ticket, sends a snapshot while nobody watches, blanks the picture behind the privacy++curtain, and measures the glass-to-glass latency with a QR probe. The design is++`plan-state-2026-10-22/AGENT-STREAMS-S1-DESIGN-2026-11-06.md`; this document is the Node side of it.++++## What runs where++++| Piece | Where |++|---|---|++| The stream, its ticket, the state poll, snapshots, renewal | the core (`internal/stream`, `internal/node/stream.go`) |++| Capture selection, the browser desk's page capture, the curtain detectors, the hotkey | the computer plugin (`desk.stream`, `plugins/computer/openvibe_computer/stream.py` and `curtain.py`) |++| Encoding | ffmpeg (`libx264`), started by the core |++| Publishing | WHIP with pion (`internal/video`), video + a silent Opus track |++++The core mints the stream and its `wpt_…` ticket on Live with this machine's Network node token (audience++`openvibe.live`, capability `live.agent-stream.publish`). **The plugin never sees the WHIP URL or the ticket**, and++neither does a log line, an error, `openvibe-node status` or the command's ack. The ticket is a++`credentials.Secret`.++++## The command++++`desk.stream` is a computer kind routed to the computer plugin, with the core doing the orchestration:++++```json++{"action": "start", "title": "fix the failing tests",++ "task": {"source": "codes", "ref": "job_123", "title": "fix the tests", "url": "https://openvibe.codes/…"},++ "visibility": "private", "allow_sites": ["https://github.com"], "desk": "desk-a"}++{"action": "stop"}++{"action": "curtain", "up": true, "reason": "manual"}++{"action": "status"}++```++++`start` answers `{streaming, id, page_url, visibility, expires_at, capture}`. `desk` may be `main` (the machine's own++screen), `private`, or `desk-<name>`; a browser desk is captured from its own page. `visibility` defaults to++`private`. `stream.live_url` in the Node config moves the API off production.++++## The encoder recipe++++`internal/stream/encode.go` builds the ffmpeg command line from the design:++++* H.264 **main**, `-preset ultrafast`, `-tune zerolatency,stillimage`, `-bf 0`;++* a keyframe every 2 s of wall clock (`-force_key_frames expr:gte(t,n_forced*2)`, `-g 60` at 30 fps);++* variable rate: `mpdecimate` drops unchanged frames and `-fps_mode vfr` keeps the capture's timestamps (about 5 fps on++ a still screen, up to 30 when it moves);++* 1080p30 (or the desk's size), with a 720p rung: after two WHIP retries the core restarts the capture at 720p++ (`EncodeOptions.LowBandwidth`);++* CBR capped at 4 Mb/s (`-b:v 4000k -maxrate 4000k -bufsize 4000k -x264-params nal-hrd=cbr`).++++Capture is `x11grab` or (ffmpeg 7.1+) `pipewire` on Linux, `ddagrab` on Windows, `avfoundation` on macOS, and++`-f image2pipe` for a browser desk's page (the plugin writes JPEGs into a 0600 FIFO at about 5 fps, the floor of the++VFR range). A browser desk has no display to grab.++++A silent Opus track is always negotiated (a 20 ms silence frame every 20 ms), so S4's voice needs no renegotiation.++++## Live's state and snapshots++++`GET /api/streams/agent/:id/state` is polled every 10 s with the ticket. `encode: false` stops the capture and pauses++video on the WHIP session (the audio keeps it alive); a snapshot is `PUT` every 5 s, JPEG or WebP, at most 200 KB,++taken through the plugin's own `screen` command. `encode: true` restarts the capture. The ticket is renewed at **3 h**++(sooner if it would expire first), and a `DELETE /api/streams/agent/:id/ticket` is the kill switch.++++## The curtain++++The plugin's detectors answer `None` (clear), a reason (raise), or raise (fail closed). Every one of these raises it:++++* `password_field` — the focused AT-SPI element is a password field (UI Automation on Windows, the AX API on macOS;++ a missing accessibility stack is *unsure*, which raises it);++* `secret_file` — the focused window's title or class looks secret, or a secret-named file was read through `fs` or++ named in an `exec` (`/.env`, `*credentials*`, `id_rsa`, `*.pem`, `*.key`, a password manager);++* `env_print` — an `exec` ran `env`/`printenv` or printed three or more environment assignments;++* `manual` — the hotkey (Ctrl+Alt+P on X11, `SIGUSR1` for a service), Curtain now, or the `curtain` command;++* `site` — a browser desk navigated outside its `allow_sites` (checked on the agent's own actions and on every++ main-frame navigation the browser reports);++* `detector` — any detector failed or returned something it could not decide.++++The curtain starts **up** and only comes down after every detector has been clear for 2 s, or when the owner lowers++it. While it is up the core feeds the publisher a pre-encoded "Private screen" frame instead of the capture, mutes the++audio track, and reports `POST /api/streams/agent/:id/curtain {up, reason}`; Live pauses every viewer's consumer as++well. Viewers never see the reason. `desk_stream_detectors` in the plugin config can disable a detector a deployment++genuinely cannot run (a deliberate choice, not a silent failure).++++**A lost plugin is a curtain, not a gap.** The plugin sends its curtain state every second. If the computer plugin++exits, crashes, is not ready (the supervisor's state), or goes silent for two seconds while a stream is up, the core++raises the curtain itself at once (reason `detector`), blanks the video and mutes the audio; a "curtain down" command++is refused while the plugin is lost. It comes down only after the plugin is back **and** has reported clear for two++seconds, so a restarted plugin's first "clear" cannot uncover the desk.++++## The latency probe and the viewer++++```++# Live hands the desk a WHIP URL and a wpt_ ticket (create + state routes).++OPENVIBE_STREAM_TICKET=wpt_… openvibe-node stream --probe \++ --whip-url https://whip.openvibe.live/whip/agent/ags_… --seconds 60++++# The same ticket's view side, on a WHEP endpoint:++OPENVIBE_VIEW_TICKET=wvt_… scripts/stream-viewer.sh --whep https://openvibe.live/whep/agent/ags_… --seconds 60++```++++The probe draws a QR of `time.Now().UnixMilli()` in the corner of every frame with the built-in PCM encoder (no ffmpeg,++no display) and publishes it with the same WHIP path and silent audio. The viewer joins over WHEP, decodes the QR from++each frame with OpenCV, and reports p50/p95/max; the S1 pass line is **p95 under 1000 ms over 60 s**. `--self-test`++checks the statistics and the decoder without a network.++++**The production number is not measured yet.** Live's S1 routes are being built in++`live-agent-streams-s1`; the viewer needs a WHEP endpoint (the design allows one for third-party players), and Live's++S1 player speaks mediasoup over `/ws/broadcast`. Once that endpoint exists, run the two commands above and put the p95++in this file and in the PR.++++## Tests++++Go (`go test ./internal/stream ./internal/video`): the recipe's arguments, each capture kind, the publisher's silent++audio and pause, the state poll turning encoding off into snapshots, renewal at 3 h, the curtain blanking every++captured frame, the QR probe round-trip, and that a ticket never appears in JSON or an error.++++Python (`plugins/computer/tests/test_stream.py`): every curtain trigger, a detector exception and an unsure detector++raising it, the two-second clear hold, the owner lowering it, the capture each platform chooses, a browser desk's++JPEG frames reaching the FIFO, and the plugin's start/status/stop/refuse path.++++## Platform notes and follow-ups++++* Linux desktops use `x11grab`; PipeWire needs ffmpeg 7.1+ and `OPENVIBE_PIPEWIRE_NODE` set to the screen-cast node.++ The target ffmpeg here is 6.1, so `x11grab` is the default.++* A browser desk streams its page at 5 fps (an ffmpeg-friendly floor); a real DevTools screencast at up to 30 fps is the++ next step.++* The AT-SPI detector sees the accessibility stack of the Node's session; on a private Xvfb desk the desk's own++ `at-spi2-registryd` must be running for it to see the desk's apps.++* Simulcast (S2) and the owner's page (Live's part) are the next pieces.@@@ -7,10 +7,12 @@ require (github.com/godbus/dbus/v5 v5.1.0github.com/gorilla/websocket v1.5.3github.com/kardianos/service v1.3.0++ github.com/makiuchi-d/gozxing v0.1.1github.com/pion/ice/v4 v4.4.4github.com/pion/interceptor v0.1.49github.com/pion/rtcp v1.2.18github.com/pion/webrtc/v4 v4.2.22++ github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686egolang.org/x/sys v0.41.0)@@@ -31,5 +33,7 @@ require (github.com/wlynxg/anet v0.0.5 // indirectgolang.org/x/crypto v0.48.0 // indirectgolang.org/x/net v0.50.0 // indirect++ golang.org/x/text v0.34.0 // indirectgolang.org/x/time v0.14.0 // indirect++ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect)@@@ -8,6 +8,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNgithub.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI=github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=++github.com/makiuchi-d/gozxing v0.1.1 h1:xxqijhoedi+/lZlhINteGbywIrewVdVv2wl9r5O9S1I=++github.com/makiuchi-d/gozxing v0.1.1/go.mod h1:eRIHbOjX7QWxLIDJoQuMLhuXg9LAuw6znsUtRkNw9DU=github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs=github.com/pion/datachannel v1.6.3/go.mod h1:oRmkXVVXxkUbqa8MatgdNHW29kvtQY3HUdjJQsfv2GE=github.com/pion/dtls/v3 v3.1.9 h1:rpeycmLIkc4krpk1IxP7+39o11QdCXbmV9+FGi9yZJ8=@@@ -40,6 +42,8 @@ github.com/pion/turn/v5 v5.1.2 h1:acEOO+D5txJ52vKfcilmSMItfP23Q9K6pJ1LM5P2VV0=github.com/pion/turn/v5 v5.1.2/go.mod h1:57Oadc8fHnac15VFyCd6nEaBTVZBxQe2POUMD4lsHRA=github.com/pion/webrtc/v4 v4.2.22 h1:eoRTHzLKW5tI9o5iahw3s4KL4IHPESsfIF06zWi3JAo=github.com/pion/webrtc/v4 v4.2.22/go.mod h1:JBu1ba8lUEX4jbYFKofglMQJG7f0rJk7UepKjI3GaLE=++github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=++github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=@@@ -52,5 +56,9 @@ golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=++golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=++golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=++golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=++golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=@@@ -110,6 +110,7 @@ type Config struct {DeviceKind string `json:"device_kind,omitempty"`Plugins []PluginConfig `json:"plugins"`Video VideoConfig `json:"video"`++ Stream StreamConfig `json:"stream,omitzero"`// Limits are the local owner's caps, applied on top of the server's (the stricter value wins).Limits LocalLimits `json:"limits"`// Python is the interpreter for plugins without an explicit command (default: <state>/venv/bin/python, then python3).@@@ -334,6 +335,20 @@ type PluginConfig struct {StrictTiming bool `json:"strict_timing,omitempty"`}++// StreamConfig is the agent desk stream (Agent Streams S1). LiveURL is where POST /api/streams/agent and the state,++// ticket, curtain and frame calls go; the default is production.++type StreamConfig struct {++ LiveURL string `json:"live_url,omitempty"`++}++++// LiveOrigin is the agent-stream API origin, production unless configured.++func (c StreamConfig) LiveOrigin() string {++ if c.LiveURL != "" {++ return c.LiveURL++ }++ return "https://openvibe.live"++}++// VideoConfig selects the camera source published over WHIP.type VideoConfig struct {// Source: "auto" (default: a plugin's h264_command if its program exists, else a plugin's JPEG frames, else the@@@ -450,6 +465,9 @@ func (c *Config) Validate() error {if c.Video.WHIPURL != "" && !strings.HasPrefix(c.Video.WHIPURL, "https://") && !strings.HasPrefix(c.Video.WHIPURL, "http://") {return errors.New("config: video.whip_url must be an https:// URL")}++ if c.Stream.LiveURL != "" && !strings.HasPrefix(c.Stream.LiveURL, "https://") {++ return errors.New("config: stream.live_url must be an https:// origin")++ }for _, l := range []*float64{c.Limits.MaxSpeed, c.Limits.MaxTurn} {if l != nil && (*l < 0 || *l > 1) {return errors.New("config: limits must be between 0 and 1")@@@ -35,6 +35,7 @@ import ("github.com/OpenVibers/OpenVibe.Node/internal/plugins""github.com/OpenVibers/OpenVibe.Node/internal/protocol""github.com/OpenVibers/OpenVibe.Node/internal/safety"++ "github.com/OpenVibers/OpenVibe.Node/internal/stream""github.com/OpenVibers/OpenVibe.Node/internal/video""github.com/OpenVibers/OpenVibe.Node/internal/worker")@@@ -79,6 +80,11 @@ type Node struct {pub *video.Publisher// screen is the owner's live view of the machine's screen (screen.go), apart from a robot's camera.screen screenState++ // deskStream is the agent's desk stream (stream.go, Agent Streams S1); one at a time.++ deskMu sync.Mutex++ deskStream *stream.Agent++ // liveTokens buys node tokens for openvibe.live (Agent Streams S1). Nil unless this machine is Network-paired.++ liveTokens *link.TokenSourcectx context.Context// quit ends Run on `openvibe-node quit` (control), and quitting tells Run to answer ErrQuit.@@@ -228,6 +234,11 @@ func New(opt Options) (*Node, error) {}}n.link = link.New(lo, n)++ if opt.Creds.NetworkPaired() {++ // Agent Streams S1: a Node principal acting for its owner calls openvibe.live. Network grants it++ // live.agent-stream.publish for its owner.++ n.liveTokens = link.NewTokenSourceFor(opt.Creds, nil, lo.UserAgent, "openvibe.live")++ }// The WHIP endpoint is the pairing's whip_url (built by Bot from the publish key) unless the config sets one.whip := opt.Config.Video.WHIPURLif whip == "" {@@@ -302,6 +313,7 @@ func (n *Node) Run(parent context.Context) error {}<-ctx.Done()n.log.Info("shutting down: stopping every actuator")++ n.stopDeskStream(false)n.mgr.StopAll(plugins.StopShutdown)if n.worker != nil {n.worker.Close()@@@ -747,6 +759,10 @@ func (n *Node) computer(c protocol.Command, entry *safety.Entry, sentMS int64, rn.screenCommand(c, entry, received, nack)return}++ if c.Kind == protocol.KindDeskStream {++ n.deskStreamCommand(c, entry, received, nack)++ return++ }if protocol.ComputerActs(c.Kind, c.Value) {st := n.latch.State()if st.Local {@@@ -928,6 +944,9 @@ func (n *Node) onLatch(st safety.LatchState) {n.log.Warn("stop cleared; motion allowed again")}n.applyPluginLatch(st, true)++ if st.Stopped() || st.Computer {++ n.stopDeskStream(true) // the kill switch: Live ends the WHIP session at once++ }n.sendEstopState()n.sendStatus()@@@ -1009,6 +1028,12 @@ func (n *Node) output(o plugins.Output) {n.tellVoice(n.mgr.Get(o.Plugin))return}++ if name == "curtain" {++ n.deskCurtainEvent(o.Plugin, o.Data)++ if heartbeat, _ := o.Data["heartbeat"].(bool); heartbeat {++ return // the stream's one-second liveness beat, not an event for the log or Bot++ }++ }fields := map[string]any{}for k, v := range o.Data {if k != "name" {@@@ -1052,7 +1077,11 @@ func (n *Node) output(o plugins.Output) {if _, err := n.latch.SetLocalWith(reason); err != nil {n.log.Error("a plugin latched the local stop, but the latch could not be saved", "plugin", o.Plugin, "err", err)}-- case plugins.OutFault, plugins.OutState:++ case plugins.OutState:++ n.deskPluginState(o)++ n.sendStatus()++ case plugins.OutFault:++ n.deskPluginFault(o)n.sendStatus()}}@@@ -1221,21 +1250,23 @@ func strSlice(v any) []string {// Status is what `openvibe-node status` prints. It never contains the credential or the publish key.type Status struct {-- Version string `json:"version"`-- DeviceID string `json:"device_id,omitempty"`-- Principal string `json:"principal,omitempty"` // nod_… of a Network-paired machine-- Server string `json:"server,omitempty"`-- Paired bool `json:"paired"`-- Uptime string `json:"uptime"`-- Link LinkStatus `json:"link"`-- Latch safety.LatchState `json:"latch"`-- Limits safety.Limits `json:"limits"`-- Plugins []PluginStatus `json:"plugins"`-- Video VideoStatus `json:"video"`-- Screen ScreenStatus `json:"screen"`-- ConfigDir string `json:"config_dir"`-- StateDir string `json:"state_dir"`-- Telemetry protocol.Telemetry `json:"telemetry"`++ Version string `json:"version"`++ DeviceID string `json:"device_id,omitempty"`++ Principal string `json:"principal,omitempty"` // nod_… of a Network-paired machine++ Server string `json:"server,omitempty"`++ Paired bool `json:"paired"`++ Uptime string `json:"uptime"`++ Link LinkStatus `json:"link"`++ Latch safety.LatchState `json:"latch"`++ Limits safety.Limits `json:"limits"`++ Plugins []PluginStatus `json:"plugins"`++ Video VideoStatus `json:"video"`++ Screen ScreenStatus `json:"screen"`++ // Desk stream is the agent's own stream, when one is running (Agent Streams S1).++ DeskStream *stream.Status `json:"desk_stream,omitempty"`++ ConfigDir string `json:"config_dir"`++ StateDir string `json:"state_dir"`++ Telemetry protocol.Telemetry `json:"telemetry"`// Jobs are the worker's jobs not ended yet; empty when the worker is off.Jobs []worker.JobInfo `json:"jobs"`// Roles are what the machine does for the network, as the owner set them on the hub (plan L34): the ids of the@@@ -1296,6 +1327,7 @@ func (n *Node) Status() Status {s.Jobs = n.worker.Jobs()}s.Screen = n.screenStatus()++ s.DeskStream = n.deskStreamStatus()s.Video.State = video.StateOffif n.pub != nil {s.Video.State, s.Video.Frames, s.Video.LastError = n.pub.State(), n.pub.Frames(), n.pub.LastError()nnew file mode 100644@@@ -0,0 +1,363 @@++package node++++import (++ "bytes"++ "context"++ "encoding/base64"++ "encoding/json"++ "errors"++ "fmt"++ "image"++ "image/jpeg"++ "image/png"++ "strings"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/mono"++ "github.com/OpenVibers/OpenVibe.Node/internal/plugins"++ "github.com/OpenVibers/OpenVibe.Node/internal/protocol"++ "github.com/OpenVibers/OpenVibe.Node/internal/safety"++ "github.com/OpenVibers/OpenVibe.Node/internal/stream"++)++++// The agent desk stream (Agent Streams S1, live.agent-stream@1): the core mints the stream and its wpt_ ticket on++// OpenVibe.Live with this machine's node token, asks the computer plugin for the desk's capture source, then publishes++// over WHIP. The ticket never goes to the plugin, into a log line or into status.++//++// Commands:++//++// {"action":"start","title":…,"task":{source,ref,title,url?},"visibility"?,"allow_sites"?,"desk"?}++// {"action":"stop"}++// {"action":"curtain","up":true|false,"reason"?:…}++// {"action":"status"}++type deskStreamValue struct {++ Action string `json:"action"`++ Title string `json:"title"`++ Task stream.Task `json:"task"`++ Visibility string `json:"visibility"`++ AllowSites []string `json:"allow_sites"`++ Desk string `json:"desk"`++ Up bool `json:"up"`++ Reason string `json:"reason"`++}++++// deskStreamCommand answers one desk.stream command. start creates the stream on Live and publishes; stop ends it;++// curtain blanks it; status reads it. None of them ever carries the ticket.++func (n *Node) deskStreamCommand(c protocol.Command, entry *safety.Entry, received mono.Instant, nack func(code, msg string)) {++ var v deskStreamValue++ if err := json.Unmarshal(c.Value, &v); err != nil {++ nack(protocol.FaultBadValue, "value must be {action, …}")++ return++ }++ switch v.Action {++ case "status":++ var data []byte++ if st := n.deskStreamStatus(); st != nil {++ data, _ = json.Marshal(map[string]any{"running": true, "stream": st})++ } else {++ data, _ = json.Marshal(map[string]any{"running": false})++ }++ r := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds(), Data: data}++ n.dedup.Complete(entry, r)++ n.send(r)++ return++ case "stop":++ stopped := n.stopDeskStream(false)++ p := n.mgr.Route(protocol.KindDeskStream, c.Target, "")++ if p != nil {++ // Best effort: the core's answer does not depend on the plugin.++ wait := p.Begin(n.ctx, c.ID+":stop", protocol.KindDeskStream, json.RawMessage(`{"action":"stop"}`), 10000)++ go wait()++ }++ data, _ := json.Marshal(map[string]any{"stopping": stopped})++ r := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds(), Data: data}++ n.dedup.Complete(entry, r)++ n.send(r)++ return++ case "curtain":++ n.deskMu.Lock()++ a := n.deskStream++ n.deskMu.Unlock()++ if a == nil {++ nack(protocol.FaultNotReady, "no desk stream is running")++ return++ }++ reason := v.Reason++ if reason == "" {++ reason = "manual"++ }++ a.SetCurtain(v.Up, reason)++ p := n.mgr.Route(protocol.KindDeskStream, c.Target, "")++ if p != nil {++ body, _ := json.Marshal(map[string]any{"action": "curtain", "up": v.Up, "reason": reason})++ wait := p.Begin(n.ctx, c.ID, protocol.KindDeskStream, body, 10000)++ go wait()++ }++ r := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds()}++ n.dedup.Complete(entry, r)++ n.send(r)++ return++ case "start":++ default:++ nack(protocol.FaultBadValue, "action is start, stop, curtain or status")++ return++ }++++ st := n.latch.State()++ if st.Stopped() || st.Computer {++ nack(protocol.FaultLocalStop, "stopped on the machine: `openvibe-node resume` on this machine is required")++ return++ }++ if n.liveTokens == nil {++ nack(protocol.FaultUnsupported, "an agent stream needs this machine paired through OpenVibe.Network (pair again with --network)")++ return++ }++ if p := n.mgr.Route(protocol.KindDeskStream, c.Target, ""); p == nil {++ nack(protocol.FaultUnsupported, "the computer plugin is off: no desk to stream")++ return++ }++ if v.Visibility == "" {++ v.Visibility = "private"++ }++ if v.Title == "" {++ v.Title = "AI agent"++ }++ if v.Task.Source == "" {++ v.Task.Source = "desk"++ }++ if v.Task.Ref == "" {++ v.Task.Ref = v.Desk++ if v.Task.Ref == "" {++ v.Task.Ref = "desk"++ }++ }++ if v.Task.Title == "" {++ v.Task.Title = v.Title++ }++ ua := "openvibe-node/" + n.opt.Version++ client, err := stream.NewClient(stream.ClientOptions{BaseURL: n.opt.Config.Stream.LiveOrigin(),++ Token: n.liveTokens.Token, UserAgent: ua})++ if err != nil {++ nack(protocol.FaultBadValue, err.Error())++ return++ }++ created, err := client.Create(n.ctx, stream.CreateRequest{Title: v.Title, Task: v.Task,++ Visibility: v.Visibility, AllowSites: v.AllowSites})++ if err != nil {++ n.log.Warn("desk stream create failed", "err", err)++ nack(protocol.FaultHardware, err.Error())++ return++ }++ // The plugin is asked after Live minted the ticket: if it cannot start, the stream is ended at once.++ p := n.mgr.Route(protocol.KindDeskStream, c.Target, "")++ toPlugin, _ := json.Marshal(map[string]any{"action": "start", "desk": v.Desk, "allow_sites": v.AllowSites,++ "visibility": v.Visibility, "title": v.Title})++ go func() {++ wait := p.Begin(n.ctx, c.ID, protocol.KindDeskStream, toPlugin, 120_000)++ r := wait()++ if !r.OK {++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)++ _ = client.End(ctx, created.Stream.ID, created.Publish.Ticket)++ cancel()++ nack(r.FaultCode, r.Message)++ return++ }++ var got struct {++ Capture stream.Capture `json:"capture"`++ Curtain struct {++ Up bool `json:"up"`++ Reason string `json:"reason"`++ } `json:"curtain"`++ }++ _ = json.Unmarshal(r.Data, &got)++ if got.Capture.Kind == "" {++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)++ _ = client.End(ctx, created.Stream.ID, created.Publish.Ticket)++ cancel()++ nack(protocol.FaultHardware, "the desk plugin returned no capture source")++ return++ }++ enc := stream.DefaultEncode()++ if got.Capture.Width > 0 && got.Capture.Height > 0 {++ enc.Width, enc.Height = got.Capture.Width, got.Capture.Height++ }++ agent, err := stream.NewAgent(stream.AgentOptions{++ Client: client, ID: created.Stream.ID, Publish: created.Publish, Capture: got.Capture, Encode: enc,++ FFmpeg: n.opt.Config.Video.FFmpeg, Log: n.log, Plugin: p.Name(),++ Snapshot: n.deskSnapshotFunc(p, v.Desk),++ })++ if err != nil {++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)++ _ = client.End(ctx, created.Stream.ID, created.Publish.Ticket)++ cancel()++ nack(protocol.FaultHardware, err.Error())++ return++ }++ // The plugin starts its curtain up (fail closed) before it answers; its change event can arrive before this++ // agent exists, so take the initial state from the ack and never show the desk unblanked.++ if got.Curtain.Up {++ reason := got.Curtain.Reason++ if reason == "" {++ reason = "detector"++ }++ agent.SetCurtain(true, reason)++ }++ n.stopDeskStream(false)++ n.deskMu.Lock()++ n.deskStream = agent++ n.deskMu.Unlock()++ go func() {++ if err := agent.Run(n.ctx); err != nil {++ n.log.Warn("desk stream ended", "stream", created.Stream.ID, "err", err)++ }++ }()++ data, _ := json.Marshal(map[string]any{"streaming": true, "id": created.Stream.ID,++ "page_url": created.Stream.PageURL, "visibility": created.Stream.Visibility,++ "expires_at": created.Publish.ExpiresAt.UTC().Format(time.RFC3339), "capture": got.Capture.Kind})++ r2 := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds(), Data: data}++ n.dedup.Complete(entry, r2)++ n.send(r2)++ }()++}++++// stopDeskStream ends the running desk stream, if any → whether one ran. revoke is the kill switch: Live ends the WHIP++// session at once. Otherwise the stream ends and its card stays.++func (n *Node) stopDeskStream(revoke bool) bool {++ n.deskMu.Lock()++ a := n.deskStream++ n.deskStream = nil++ n.deskMu.Unlock()++ if a == nil {++ return false++ }++ if revoke {++ a.Revoke()++ } else {++ a.Stop()++ }++ return true++}++++func (n *Node) deskStreamStatus() *stream.Status {++ n.deskMu.Lock()++ a := n.deskStream++ n.deskMu.Unlock()++ if a == nil {++ return nil++ }++ st := a.Status()++ return &st++}++++// deskSnapshotFunc takes a picture for the card while nobody watches: the computer plugin's own screen command, which++// works the same for a desk's display and a browser desk's page (and never touches the owner's screen).++func (n *Node) deskSnapshotFunc(p *plugins.Plugin, desk string) func(context.Context) ([]byte, error) {++ return func(ctx context.Context) ([]byte, error) {++ value := map[string]any{"max_width": 1280}++ if desk != "" {++ value["desk"] = desk++ }++ body, _ := json.Marshal(value)++ id := fmt.Sprintf("desk-stream-frame-%d", time.Now().UnixNano())++ r := p.Begin(ctx, id, protocol.KindScreen, body, 20_000)()++ if !r.OK {++ return nil, fmt.Errorf("the desk plugin could not take a snapshot: %s %s", r.FaultCode, r.Message)++ }++ var got struct {++ Format string `json:"format"`++ ContentB64 string `json:"content_b64"`++ }++ if err := json.Unmarshal(r.Data, &got); err != nil || got.ContentB64 == "" {++ return nil, errors.New("the desk plugin returned no picture")++ }++ raw, err := base64.StdEncoding.DecodeString(got.ContentB64)++ if err != nil {++ return nil, errors.New("the desk plugin returned a bad picture")++ }++ if strings.EqualFold(got.Format, "png") {++ return pngToJPEG(raw)++ }++ if len(raw) > stream.MaxFrameBytes {++ return nil, fmt.Errorf("the snapshot is larger than %d bytes", stream.MaxFrameBytes)++ }++ return raw, nil++ }++}++++// pngToJPEG turns a capture tool's PNG into the JPEG the frame endpoint takes.++func pngToJPEG(raw []byte) ([]byte, error) {++ img, err := png.Decode(bytes.NewReader(raw))++ if err != nil {++ return nil, errors.New("the desk plugin returned a bad PNG")++ }++ // Downscale to at most 1280 wide with a simple nearest-neighbour pass; snapshots are a card, not the stream.++ b := img.Bounds()++ w, h := b.Dx(), b.Dy()++ if w > 1280 {++ h = h * 1280 / w++ w = 1280++ }++ dst := image.NewRGBA(image.Rect(0, 0, w, h))++ for y := 0; y < h; y++ {++ for x := 0; x < w; x++ {++ dst.Set(x, y, img.At(b.Min.X+x*b.Dx()/w, b.Min.Y+y*b.Dy()/h))++ }++ }++ var buf bytes.Buffer++ if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 80}); err != nil {++ return nil, err++ }++ if buf.Len() > stream.MaxFrameBytes {++ return nil, fmt.Errorf("the snapshot is larger than %d bytes", stream.MaxFrameBytes)++ }++ return buf.Bytes(), nil++}++++// deskCurtainEvent is the computer plugin's own curtain (detectors, hotkey, Curtain now): the desk reports it to Live++// and blanks the picture. A detector that failed already raised it in the plugin.++func (n *Node) deskCurtainEvent(plugin string, data map[string]any) {++ n.deskMu.Lock()++ a := n.deskStream++ n.deskMu.Unlock()++ if a == nil || (a.PluginName() != "" && a.PluginName() != plugin) {++ return++ }++ up, _ := data["up"].(bool)++ reason, _ := data["reason"].(string)++ if reason == "" {++ reason = "detector"++ }++ a.ObserveCurtain(up, reason)++}++++// deskPluginState is the supervisor's state for the plugin that owns the desk's detectors: anything but ready is a++// lost plugin, so the curtain goes up and the audio mutes at once (fail closed).++func (n *Node) deskPluginState(o plugins.Output) {++ a := n.currentDeskStream()++ if a == nil || a.PluginName() == "" || a.PluginName() != o.Plugin {++ return++ }++ state, _ := o.Data["state"].(string)++ if state == plugins.StateReady {++ a.PluginBack()++ return++ }++ a.PluginLost()++}++++// deskPluginFault is the plugin going down between state reports (plugin_down): the same fail-closed curtain.++func (n *Node) deskPluginFault(o plugins.Output) {++ a := n.currentDeskStream()++ if a == nil || a.PluginName() != o.Plugin {++ return++ }++ if code, _ := o.Data["fault_code"].(string); code == protocol.FaultPluginDown {++ a.PluginLost()++ }++}++++func (n *Node) currentDeskStream() *stream.Agent {++ n.deskMu.Lock()++ defer n.deskMu.Unlock()++ return n.deskStream++}nnew file mode 100644@@@ -0,0 +1,53 @@++package node++++import (++ "bytes"++ "image"++ "image/color"++ "image/jpeg"++ "image/png"++ "testing"++)++++// jpegConfig decodes the JPEG the snapshot path produced (it is a normal image, not an encoder detail).++func jpegConfig(b []byte) (image.Config, error) {++ return jpeg.DecodeConfig(bytes.NewReader(b))++}++++func TestDeskStreamStatusAndStopAreSafeBeforeAStream(t *testing.T) {++ n := &Node{}++ if st := n.deskStreamStatus(); st != nil {++ t.Fatalf("status before a stream = %+v", st)++ }++ if n.stopDeskStream(false) || n.stopDeskStream(true) {++ t.Fatal("stop reported a stream that never ran")++ }++ n.deskCurtainEvent("computer", map[string]any{"up": true, "reason": "manual"})++}++++func TestPNGToJPEG(t *testing.T) {++ src := image.NewRGBA(image.Rect(0, 0, 2000, 1000))++ for y := 0; y < 1000; y++ {++ for x := 0; x < 2000; x++ {++ src.Set(x, y, color.RGBA{R: uint8(x), G: uint8(y), B: 40, A: 255})++ }++ }++ var buf bytes.Buffer++ if err := png.Encode(&buf, src); err != nil {++ t.Fatal(err)++ }++ out, err := pngToJPEG(buf.Bytes())++ if err != nil {++ t.Fatal(err)++ }++ cfg, err := jpegConfig(out)++ if err != nil {++ t.Fatal(err)++ }++ if cfg.Width != 1280 {++ t.Fatalf("snapshot width = %d, want 1280", cfg.Width)++ }++ if len(out) > 200*1024 {++ t.Fatalf("snapshot is %d bytes", len(out))++ }++}nnew file mode 100644@@@ -0,0 +1,81 @@++package node++++import (++ "context"++ "io"++ "log/slog"++ "testing"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++ "github.com/OpenVibers/OpenVibe.Node/internal/plugins"++ "github.com/OpenVibers/OpenVibe.Node/internal/protocol"++ "github.com/OpenVibers/OpenVibe.Node/internal/stream"++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++type watchFakeLive struct{}++++func (watchFakeLive) State(context.Context, string, credentials.Secret) (stream.State, error) {++ return stream.State{Encode: true}, nil++}++func (watchFakeLive) Renew(context.Context, string, credentials.Secret) (stream.Publish, error) {++ return stream.Publish{}, nil++}++func (watchFakeLive) Curtain(context.Context, string, credentials.Secret, bool, string) error {++ return nil++}++func (watchFakeLive) PostFrame(context.Context, string, credentials.Secret, []byte) error { return nil }++func (watchFakeLive) End(context.Context, string, credentials.Secret) error { return nil }++func (watchFakeLive) Revoke(context.Context, string, credentials.Secret) error { return nil }++++func newWatchAgent(t *testing.T) (*Node, *stream.Agent) {++ t.Helper()++ a, err := stream.NewAgent(stream.AgentOptions{Client: watchFakeLive{}, ID: "ags_01J8Z4M2Q0R7T9YV3K6N8P1W2X",++ Publish: stream.Publish{WHIPURL: "https://whip.example/whip/agent/ags_01J8Z4M2Q0R7T9YV3K6N8P1W2X",++ Ticket: credentials.NewSecret("wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG"), ExpiresAt: time.Now().Add(time.Hour)},++ Plugin: "computer", Log: slog.New(slog.NewTextHandler(io.Discard, nil)),++ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}})++ if err != nil {++ t.Fatal(err)++ }++ n := &Node{}++ n.deskStream = a++ return n, a++}++++func TestAPluginStateThatIsNotReadyRaisesTheCurtain(t *testing.T) {++ n, a := newWatchAgent(t)++ if a.Status().Curtain {++ t.Fatal("the stream started with the curtain up")++ }++ n.deskPluginState(plugins.Output{Plugin: "computer", Data: map[string]any{"state": plugins.StateDown}})++ if !a.Status().Curtain {++ t.Fatal("the plugin going down did not raise the curtain")++ }++ // Another plugin's state is not this stream's business.++ n.deskPluginState(plugins.Output{Plugin: "dryrun", Data: map[string]any{"state": plugins.StateReady}})++ if !a.Status().Curtain {++ t.Fatal("another plugin's state lowered it")++ }++ n.deskPluginFault(plugins.Output{Plugin: "computer", Data: map[string]any{"fault_code": protocol.FaultPluginDown}})++ if !a.Status().Curtain {++ t.Fatal("plugin_down did not keep it up")++ }++}++++func TestACurtainHeartbeatFromThePluginIsObserved(t *testing.T) {++ n, a := newWatchAgent(t)++ n.deskCurtainEvent("dryrun", map[string]any{"up": true, "reason": "password_field"})++ if a.Status().Curtain {++ t.Fatal("another plugin's curtain moved this stream")++ }++ n.deskCurtainEvent("computer", map[string]any{"up": true, "reason": "password_field", "heartbeat": true})++ if !a.Status().Curtain {++ t.Fatal("a heartbeat saying up did not raise the curtain")++ }++ n.deskCurtainEvent("computer", map[string]any{"up": false, "reason": "detector", "heartbeat": true})++ if !a.Status().Curtain {++ t.Fatal("one clear heartbeat lowered the curtain without the hold")++ }++}@@@ -25,6 +25,7 @@ var capabilityFor = map[string]string{protocol.KindInput: "input",protocol.KindBrowser: "browser",protocol.KindDesk: "desk",++ protocol.KindDeskStream: "desk.stream",protocol.KindAI: "ai",}@@@ -192,13 +192,19 @@ const (// "messages", "max_tokens"?, "temperature"?} → {model, message, usage?}. It reads and changes nothing on the// machine, so it never acts and runs while a stop is latched like a read.KindAI = "ai"++ // KindDeskStream is an agent's own desk stream (Agent Streams S1, live.agent-stream@1): {"action": "start", "title",++ // "task": {source, ref, title, url?}, "visibility": "private"|"unlisted"|"public", "allow_sites"?}, or++ // {"action": "stop"|"status"|"curtain", "up"?, "reason"?}. The core mints the stream and its wpt_ ticket on Live,++ // routes start to the computer plugin for the desk's capture source, then publishes over WHIP. Starting and++ // stopping act; status does not.++ KindDeskStream = "desk.stream")// Kinds lists every command kind the Node understands.--var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream, KindDesk, KindAI}++var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream, KindDesk, KindDeskStream, KindAI}// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.--var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true, KindDesk: true, KindAI: true}++var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true, KindDesk: true, KindDeskStream: true, KindAI: true}const (// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps@@@ -235,6 +241,12 @@ func ComputerActs(kind string, value json.RawMessage) bool {}_ = json.Unmarshal(value, &v)return v.Action != "status" && v.Action != "list"++ case KindDeskStream:++ var v struct {++ Action string `json:"action"`++ }++ _ = json.Unmarshal(value, &v)++ return v.Action != "status"}return false}nnew file mode 100644@@@ -0,0 +1,700 @@++package stream++++import (++ "context"++ "errors"++ "log/slog"++ "sync"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++// liveAPI is the part of Client the agent uses; tests substitute it.++type liveAPI interface {++ State(context.Context, string, credentials.Secret) (State, error)++ Renew(context.Context, string, credentials.Secret) (Publish, error)++ Curtain(context.Context, string, credentials.Secret, bool, string) error++ PostFrame(context.Context, string, credentials.Secret, []byte) error++ End(context.Context, string, credentials.Secret) error++ Revoke(context.Context, string, credentials.Secret) error++}++++// publisher is the part of video.Publisher the agent uses; tests substitute it.++type publisher interface {++ Run(context.Context, video.Source)++ SetPaused(bool)++ SetMuted(bool)++ SetPublishKey(credentials.Secret)++ State() string++ Frames() int64++ LastError() string++}++++// AgentOptions configure one desk stream.++type AgentOptions struct {++ Client liveAPI++ ID string++ Publish Publish++ Capture Capture++ Encode EncodeOptions++ FFmpeg string++ Log *slog.Logger++++ // Snapshot returns a JPEG of the desk for the card while nobody watches. The node wires it to the computer plugin++ // (a browser desk's page) or to a one-shot ffmpeg capture (a display).++ Snapshot func(context.Context) ([]byte, error)++ // CurtainJPEG is the "Private screen" picture posted as the snapshot while the curtain is up.++ CurtainJPEG []byte++ // CurtainFrames are pre-encoded H.264 access units of the curtain, looped while it is up.++ CurtainFrames []video.Sample++++ StateEvery time.Duration // default 10 s++ SnapshotEvery time.Duration // default 5 s++ RenewAfter time.Duration // default 3 h (the design's renewal point)++ Now func() time.Time++ // Plugin is the computer plugin that owns the desk's detectors. Its exit or silence raises the curtain; the++ // watchdog runs every WatchEvery, a loss is declared after CurtainLostAfter, and clear states must last++ // CurtainClearHold before the curtain comes down.++ Plugin string++ WatchEvery time.Duration++ CurtainLostAfter time.Duration++ CurtainClearHold time.Duration++++ // NewPublisher and NewSource are for tests; production uses video.NewPublisher and FFmpegSource.++ NewPublisher func(video.Options) publisher++ NewSource func(Capture, EncodeOptions) video.Source++}++++// Agent is one desk stream: a WHIP publisher, the capture pipeline, Live's state, the curtain and the ticket.++type Agent struct {++ opt AgentOptions++ log *slog.Logger++++ ctx context.Context++ cancel context.CancelFunc++ pub publisher++ ds *deskSource++++ mu sync.Mutex++ ticket credentials.Secret++ encode bool++ curtain bool++ reason string++ viewers int++ ended bool++ revoked bool++ expiresAt time.Time++ retries int++ downshifted bool++ // The plugin's curtain state: when it was last heard, whether the supervisor says it is down, and since when it++ // has been reporting clear while the curtain is up.++ lastPluginAt time.Time++ pluginDown bool++ clearSince time.Time++++ snapMu sync.Mutex++}++++// NewAgent validates and returns an agent; Run starts it.++func NewAgent(o AgentOptions) (*Agent, error) {++ if o.Client == nil || o.ID == "" {++ return nil, errors.New("stream: an agent needs a client and a stream id")++ }++ if err := checkID(o.ID); err != nil {++ return nil, err++ }++ if o.Publish.WHIPURL == "" || o.Publish.Ticket.IsZero() {++ return nil, errors.New("stream: an agent needs a WHIP URL and a publish ticket")++ }++ if o.Publish.ExpiresAt.IsZero() {++ return nil, errors.New("stream: the publish part has no expiry")++ }++ if o.StateEvery <= 0 {++ o.StateEvery = 10 * time.Second++ }++ if o.SnapshotEvery <= 0 {++ o.SnapshotEvery = 5 * time.Second++ }++ if o.RenewAfter <= 0 {++ o.RenewAfter = 3 * time.Hour++ }++ if o.WatchEvery <= 0 {++ o.WatchEvery = 250 * time.Millisecond++ }++ if o.CurtainLostAfter <= 0 {++ o.CurtainLostAfter = 2 * time.Second++ }++ if o.CurtainClearHold <= 0 {++ o.CurtainClearHold = 2 * time.Second++ }++ if o.Now == nil {++ o.Now = time.Now++ }++ if o.Log == nil {++ o.Log = slog.Default()++ }++ if o.NewPublisher == nil {++ o.NewPublisher = func(vo video.Options) publisher { return video.NewPublisher(vo) }++ }++ if o.NewSource == nil {++ o.NewSource = func(c Capture, e EncodeOptions) video.Source {++ return &FFmpegSource{FFmpeg: o.FFmpeg, Capture: c, Options: e, Log: o.Log}++ }++ }++ a := &Agent{opt: o, log: o.Log.With("component", "desk-stream"), ticket: o.Publish.Ticket, encode: true,++ expiresAt: o.Publish.ExpiresAt, lastPluginAt: o.Now()}++ a.ds = &deskSource{a: a}++ return a, nil++}++++// Run publishes until ctx ends or the stream ends. It ends the stream at Live (so the card stays), unless Revoke was++// called, which is the kill switch.++func (a *Agent) Run(parent context.Context) error {++ ctx, cancel := context.WithCancel(parent)++ a.mu.Lock()++ a.ctx, a.cancel = ctx, cancel++ revoked := a.revoked++ a.mu.Unlock()++ if revoked {++ cancel()++ return nil++ }++ if len(a.opt.CurtainFrames) == 0 || len(a.opt.CurtainJPEG) == 0 {++ cctx, ccancel := context.WithTimeout(ctx, 20*time.Second)++ frames, jpg := MakeCurtain(cctx, a.opt.FFmpeg, a.opt.Capture.Width, a.opt.Capture.Height)++ ccancel()++ if len(a.opt.CurtainFrames) == 0 {++ a.opt.CurtainFrames = frames++ }++ if len(a.opt.CurtainJPEG) == 0 {++ a.opt.CurtainJPEG = jpg++ }++ }++ pub := a.opt.NewPublisher(video.Options{WHIPURL: a.opt.Publish.WHIPURL, PublishKey: a.ticket,++ Audio: true, UserAgent: "openvibe-node/stream", Log: a.opt.Log})++ a.mu.Lock()++ a.pub = pub++ curtainUp := a.curtain++ a.mu.Unlock()++ if curtainUp {++ pub.SetMuted(true)++ }++ defer cancel()++ go pub.Run(ctx, a.ds)++++ state := time.NewTicker(a.opt.StateEvery)++ defer state.Stop()++ snap := time.NewTicker(a.opt.SnapshotEvery)++ defer snap.Stop()++ renew := time.NewTimer(a.renewDelay())++ defer renew.Stop()++ watch := time.NewTicker(a.opt.WatchEvery)++ defer watch.Stop()++++ a.log.Info("desk stream started", "stream", a.opt.ID, "capture", a.opt.Capture.Kind)++ a.postSnapshot(ctx) // the card has a face before the first state poll++++ for {++ select {++ case <-ctx.Done():++ a.finish()++ return nil++ case <-renew.C:++ a.renew(ctx)++ renew.Reset(a.renewDelay())++ case <-state.C:++ if err := a.poll(ctx); err != nil {++ a.log.Warn("desk stream state poll failed", "stream", a.opt.ID, "err", err)++ }++ case <-snap.C:++ if !a.encodeNow() {++ a.postSnapshot(ctx)++ }++ case <-watch.C:++ a.watchCurtain()++ }++ }++}++++func (a *Agent) renewDelay() time.Duration {++ // The design renews at three hours; never wait past the ticket's own expiry (with a margin for the round trip).++ a.mu.Lock()++ exp := a.expiresAt++ a.mu.Unlock()++ if d := a.opt.RenewAfter; d > 0 {++ if left := exp.Sub(a.opt.Now()) - 15*time.Minute; left > 0 && left < d {++ return left++ }++ return d++ }++ return 3 * time.Hour++}++++func (a *Agent) poll(ctx context.Context) error {++ st, err := a.opt.Client.State(ctx, a.opt.ID, a.ticketNow())++ if err != nil {++ return err++ }++ a.mu.Lock()++ a.viewers = st.Viewers++ curtainUp := a.curtain++ a.mu.Unlock()++ if st.Curtain.Up && !curtainUp {++ // The owner raised it on another surface. Only the desk may bring it down.++ a.SetCurtain(true, reasonOr(st.Curtain.Reason))++ }++ if st.Encode != a.encodeNow() {++ a.SetEncode(st.Encode)++ }++ a.checkBandwidth()++ if st.EndsAt != nil && !st.EndsAt.After(a.opt.Now()) {++ a.log.Info("desk stream ended at Live", "stream", a.opt.ID)++ a.cancel()++ }++ return nil++}++++func reasonOr(r string) string {++ if validReason(r) {++ return r++ }++ return "detector"++}++++func (a *Agent) renew(ctx context.Context) {++ p, err := a.opt.Client.Renew(ctx, a.opt.ID, a.ticketNow())++ if err != nil {++ a.log.Warn("desk stream ticket renewal failed; retrying in a minute", "stream", a.opt.ID, "err", err)++ a.mu.Lock()++ runCtx := a.ctx++ a.mu.Unlock()++ if runCtx != nil {++ time.AfterFunc(time.Minute, func() {++ if runCtx.Err() == nil {++ a.renew(runCtx)++ }++ })++ }++ return++ }++ a.mu.Lock()++ a.ticket = p.Ticket++ if !p.ExpiresAt.IsZero() {++ a.expiresAt = p.ExpiresAt++ }++ a.mu.Unlock()++ a.pub.SetPublishKey(p.Ticket)++ a.log.Info("desk stream ticket renewed", "stream", a.opt.ID, "expires_at", p.ExpiresAt.UTC().Format(time.RFC3339))++}++++// checkBandwidth is the design's 720p rung: two WHIP retries close together mean the link is struggling, so the++// capture restarts at 720p. It never goes back up (S2's simulcast is the real answer).++func (a *Agent) checkBandwidth() {++ pub := a.pub++ if pub == nil {++ return++ }++ if pub.State() != video.StateRetrying {++ a.mu.Lock()++ a.retries = 0++ a.mu.Unlock()++ return++ }++ a.mu.Lock()++ if a.downshifted {++ a.mu.Unlock()++ return++ }++ a.retries++++ down := a.retries >= 2++ if down {++ a.downshifted = true++ a.opt.Encode = a.opt.Encode.LowBandwidth()++ }++ a.mu.Unlock()++ if down {++ a.log.Info("desk stream downshifted to 720p after repeated WHIP retries", "stream", a.opt.ID)++ a.ds.restart()++ }++}++++func (a *Agent) ticketNow() credentials.Secret {++ a.mu.Lock()++ defer a.mu.Unlock()++ return a.ticket++}++++func (a *Agent) encodeNow() bool {++ a.mu.Lock()++ defer a.mu.Unlock()++ return a.encode++}++++func (a *Agent) curtainNow() (bool, []byte) {++ a.mu.Lock()++ defer a.mu.Unlock()++ return a.curtain, a.opt.CurtainJPEG++}++++// SetEncode is Live's encode flag: false stops the capture and turns the stream into snapshots. The WHIP session and++// its silent audio stay up, so Live does not see the publisher drop.++func (a *Agent) SetEncode(on bool) {++ a.mu.Lock()++ if a.encode == on {++ a.mu.Unlock()++ return++ }++ a.encode = on++ a.mu.Unlock()++ a.ds.setEncode(on)++ a.pub.SetPaused(!on)++ a.log.Info("desk stream encoding changed", "stream", a.opt.ID, "encode", on)++}++++// SetCurtain is the owner's own switch (the curtain command, or the page's Curtain now): it takes effect at once,++// except that it never lowers a curtain while the plugin that owns the detectors is lost or silent (fail closed).++func (a *Agent) SetCurtain(up bool, reason string) {++ if !up && a.pluginLost() {++ a.log.Warn("desk stream curtain down refused: the computer plugin is lost or silent", "stream", a.opt.ID)++ return++ }++ a.applyCurtain(up, reason)++}++++// ObserveCurtain is one curtain state from the desk plugin: a change, or its one-second heartbeat. A clear state only++// lowers the curtain after the plugin has kept reporting it for CurtainClearHold, so a plugin that has just come back++// cannot uncover the desk with its first "clear".++func (a *Agent) ObserveCurtain(up bool, reason string) {++ now := a.opt.Now()++ a.mu.Lock()++ a.lastPluginAt = now++ // pluginDown is the supervisor's verdict, not the plugin's own word: a faulted plugin that still beats cannot++ // clear it. Only PluginBack does.++ if up {++ a.clearSince = time.Time{}++ } else if a.curtain && a.clearSince.IsZero() {++ a.clearSince = now++ }++ a.mu.Unlock()++ if up {++ a.applyCurtain(true, reason)++ }++}++++// PluginLost is the supervisor saying the computer plugin is not ready (it exited, crashed or faulted): the picture++// blanks and the audio mutes now, without waiting for the heartbeat to go quiet.++func (a *Agent) PluginLost() {++ a.mu.Lock()++ a.pluginDown = true++ a.clearSince = time.Time{}++ a.mu.Unlock()++ a.applyCurtain(true, "detector")++}++++// PluginBack says the plugin is ready again. The curtain stays up until its clear states have lasted the hold.++func (a *Agent) PluginBack() {++ a.mu.Lock()++ a.pluginDown = false++ a.lastPluginAt = a.opt.Now()++ a.mu.Unlock()++}++++// PluginName is the computer plugin the stream depends on (the supervisor's state changes are matched on it).++func (a *Agent) PluginName() string { return a.opt.Plugin }++++func (a *Agent) pluginLost() bool {++ a.mu.Lock()++ defer a.mu.Unlock()++ return a.pluginDown || a.opt.Now().Sub(a.lastPluginAt) > a.opt.CurtainLostAfter++}++++// watchCurtain is the fail-closed watchdog: a plugin that is down, or silent for CurtainLostAfter, blanks the desk at++// once; once it is back, only CurtainClearHold of clear states brings the curtain down.++func (a *Agent) watchCurtain() {++ now := a.opt.Now()++ a.mu.Lock()++ lost := a.pluginDown || now.Sub(a.lastPluginAt) > a.opt.CurtainLostAfter++ up := a.curtain++ clear := up && !a.clearSince.IsZero() && now.Sub(a.clearSince) >= a.opt.CurtainClearHold++ a.mu.Unlock()++ if lost {++ a.mu.Lock()++ a.clearSince = time.Time{}++ a.mu.Unlock()++ a.applyCurtain(true, "detector")++ return++ }++ if clear {++ a.applyCurtain(false, "detector")++ }++}++++// applyCurtain changes the picture and the audio and tells Live. It is idempotent, so the watchdog can call it every++// tick.++func (a *Agent) applyCurtain(up bool, reason string) {++ a.mu.Lock()++ if a.curtain == up {++ if up && reason != "" && validReason(reason) && reason != a.reason {++ a.reason = reason++ }++ a.mu.Unlock()++ return++ }++ a.curtain = up++ a.reason = reasonOr(reason)++ r := a.reason++ pub := a.pub++ a.mu.Unlock()++ a.ds.curtainChanged(up)++ if pub != nil {++ pub.SetMuted(up)++ }++ go func() {++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)++ defer cancel()++ if err := a.opt.Client.Curtain(ctx, a.opt.ID, a.ticketNow(), up, r); err != nil {++ a.log.Warn("desk stream curtain report failed", "stream", a.opt.ID, "up", up, "err", err)++ }++ }()++ a.log.Info("desk stream curtain", "stream", a.opt.ID, "up", up, "reason", r)++}++++// Stop ends an ordinary stream: Live keeps the card, with its last frame and worked time.++func (a *Agent) Stop() {++ a.mu.Lock()++ cancel := a.cancel++ a.mu.Unlock()++ if cancel != nil {++ cancel()++ }++}++++// Revoke is the kill switch: Live ends the WHIP session at once and no End is sent afterwards.++func (a *Agent) Revoke() {++ a.mu.Lock()++ a.revoked = true++ a.mu.Unlock()++ // Best effort: the caller is a latch, so it must not wait on the network.++ go func() {++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)++ defer cancel()++ if err := a.opt.Client.Revoke(ctx, a.opt.ID, a.ticketNow()); err != nil {++ a.log.Warn("desk stream revoke failed", "stream", a.opt.ID, "err", err)++ }++ }()++ a.mu.Lock()++ cancel := a.cancel++ a.mu.Unlock()++ if cancel != nil {++ cancel()++ }++}++++func (a *Agent) finish() {++ a.mu.Lock()++ if a.ended || a.revoked {++ a.mu.Unlock()++ return++ }++ a.ended = true++ a.mu.Unlock()++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)++ defer cancel()++ if err := a.opt.Client.End(ctx, a.opt.ID, a.ticketNow()); err != nil {++ a.log.Warn("desk stream end failed", "stream", a.opt.ID, "err", err)++ }++}++++func (a *Agent) postSnapshot(ctx context.Context) {++ if a.opt.Snapshot == nil {++ return++ }++ if !a.snapMu.TryLock() {++ return // a slow capture must not pile snapshots up++ }++ defer a.snapMu.Unlock()++ var (++ frame []byte++ err error++ )++ if up, blank := a.curtainNow(); up {++ frame = blank++ } else {++ frame, err = a.opt.Snapshot(ctx)++ }++ if err != nil || len(frame) == 0 {++ if err != nil {++ a.log.Warn("desk stream snapshot failed", "stream", a.opt.ID, "err", err)++ }++ return++ }++ if err := a.opt.Client.PostFrame(ctx, a.opt.ID, a.ticketNow(), frame); err != nil {++ a.log.Warn("desk stream snapshot post failed", "stream", a.opt.ID, "err", err)++ }++}++++// Status is what `openvibe-node status` shows for one stream.++type Status struct {++ ID string `json:"id"`++ State string `json:"state"`++ Encode bool `json:"encode"`++ Curtain bool `json:"curtain"`++ Viewers int `json:"viewers"`++ Frames int64 `json:"frames"`++ LastError string `json:"last_error,omitempty"`++}++++// Status reports the stream without a ticket.++func (a *Agent) Status() Status {++ a.mu.Lock()++ st := Status{ID: a.opt.ID, Encode: a.encode, Curtain: a.curtain, Viewers: a.viewers}++ pub := a.pub++ a.mu.Unlock()++ if pub != nil {++ st.State = pub.State()++ st.Frames = pub.Frames()++ st.LastError = pub.LastError()++ }++ return st++}++++// ---- the source the publisher runs ----++++// deskSource feeds the publisher. It runs the capture only while Live wants video, and loops the pre-encoded curtain++// while the curtain is up, so no captured frame can slip through.++type deskSource struct {++ a *Agent++++ mu sync.Mutex++ runCtx context.Context++ sink func(video.Sample)++ encStop context.CancelFunc++ encGen int++ encRun bool++ up bool++}++++func (d *deskSource) Name() string { return "desk-stream" }++func (d *deskSource) RequestKeyframe() { d.a.ds.restart() }++++func (d *deskSource) Run(ctx context.Context, sink func(video.Sample)) error {++ d.mu.Lock()++ d.runCtx, d.sink = ctx, sink++ up := d.up++ enc := d.a.encodeNow()++ d.mu.Unlock()++ if enc && !up {++ d.startEncoder()++ }++ // While the curtain is up the publisher gets only the curtain loop; the capture keeps running so the picture is++ // back the instant the detectors clear.++ tick := time.NewTicker(100 * time.Millisecond)++ defer tick.Stop()++ for {++ select {++ case <-ctx.Done():++ d.stopEncoder()++ return nil++ case <-tick.C:++ if d.isUp() {++ d.forwardCurtain()++ }++ }++ }++}++++func (d *deskSource) isUp() bool {++ d.mu.Lock()++ defer d.mu.Unlock()++ return d.up++}++++func (d *deskSource) forwardCurtain() {++ d.mu.Lock()++ sink := d.sink++ frames := d.a.opt.CurtainFrames++ d.mu.Unlock()++ if sink == nil || len(frames) == 0 {++ return++ }++ d.mu.Lock()++ f := frames[0]++ d.mu.Unlock()++ sink(f)++}++++func (d *deskSource) startEncoder() {++ d.mu.Lock()++ if d.encRun || d.runCtx == nil || d.up {++ d.mu.Unlock()++ return++ }++ ctx, cancel := context.WithCancel(d.runCtx)++ d.encGen++++ gen := d.encGen++ d.encRun, d.encStop = true, cancel++ d.a.mu.Lock()++ enc := d.a.opt.Encode++ d.a.mu.Unlock()++ src := d.a.opt.NewSource(d.a.opt.Capture, enc)++ d.mu.Unlock()++ go func() {++ _ = src.Run(ctx, func(s video.Sample) {++ d.mu.Lock()++ up, out := d.up, d.sink++ d.mu.Unlock()++ if up || out == nil {++ return++ }++ out(s)++ })++ d.mu.Lock()++ if d.encGen == gen {++ d.encRun, d.encStop = false, nil++ }++ d.mu.Unlock()++ }()++}++++func (d *deskSource) stopEncoder() {++ d.mu.Lock()++ cancel := d.encStop++ d.encRun, d.encStop = false, nil++ d.mu.Unlock()++ if cancel != nil {++ cancel()++ }++}++++func (d *deskSource) restart() {++ d.mu.Lock()++ was := d.encRun++ d.mu.Unlock()++ if !was {++ return++ }++ d.stopEncoder()++ time.Sleep(10 * time.Millisecond) // let the old ffmpeg release the display++ d.startEncoder()++}++++func (d *deskSource) setEncode(on bool) {++ if on {++ d.startEncoder()++ } else {++ d.stopEncoder()++ }++}++++func (d *deskSource) curtainChanged(up bool) {++ d.mu.Lock()++ d.up = up++ d.mu.Unlock()++ if up {++ d.stopEncoder()++ } else if d.a.encodeNow() {++ d.startEncoder()++ }++}nnew file mode 100644@@@ -0,0 +1,528 @@++package stream++++import (++ "bytes"++ "context"++ "encoding/json"++ "fmt"++ "io"++ "log/slog"++ "strings"++ "sync"++ "sync/atomic"++ "testing"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++type fakeAPI struct {++ mu sync.Mutex++ state func() State++ renews []credentials.Secret++ newKey string++ curtains []string++ frames int32++ ended int32++ revoked int32++}++++func (f *fakeAPI) State(context.Context, string, credentials.Secret) (State, error) {++ if f.state != nil {++ return f.state(), nil++ }++ return State{Encode: true, Curtain: Curtain{}}, nil++}++func (f *fakeAPI) Renew(_ context.Context, _ string, old credentials.Secret) (Publish, error) {++ f.mu.Lock()++ f.renews = append(f.renews, old)++ f.mu.Unlock()++ return Publish{WHIPURL: "https://whip.example/x", Ticket: credentials.NewSecret(f.newKey),++ ExpiresAt: time.Now().Add(4 * time.Hour)}, nil++}++func (f *fakeAPI) Curtain(_ context.Context, _ string, _ credentials.Secret, up bool, reason string) error {++ f.mu.Lock()++ f.curtains = append(f.curtains, reason)++ f.mu.Unlock()++ return nil++}++func (f *fakeAPI) PostFrame(context.Context, string, credentials.Secret, []byte) error {++ atomic.AddInt32(&f.frames, 1)++ return nil++}++func (f *fakeAPI) End(context.Context, string, credentials.Secret) error {++ atomic.AddInt32(&f.ended, 1)++ return nil++}++func (f *fakeAPI) Revoke(context.Context, string, credentials.Secret) error {++ atomic.AddInt32(&f.revoked, 1)++ return nil++}++++type fakePub struct {++ paused atomic.Bool++ muted atomic.Bool++ key atomic.Value++ state atomic.Value++}++++func (p *fakePub) Run(ctx context.Context, src video.Source) {++ go func() { _ = src.Run(ctx, func(video.Sample) {}) }()++ <-ctx.Done()++}++func (p *fakePub) SetPaused(v bool) { p.paused.Store(v) }++func (p *fakePub) SetMuted(v bool) { p.muted.Store(v) }++func (p *fakePub) SetPublishKey(k credentials.Secret) {++ if !k.IsZero() {++ p.key.Store(k)++ }++}++func (p *fakePub) State() string {++ if s, _ := p.state.Load().(string); s != "" {++ return s++ }++ return video.StateLive++}++func (p *fakePub) Frames() int64 { return 1 }++func (p *fakePub) LastError() string { return "" }++++type fakeSource struct {++ out chan string++ once sync.Once++}++++func (s *fakeSource) Name() string { return "fake" }++func (s *fakeSource) RequestKeyframe() {}++func (s *fakeSource) Run(ctx context.Context, sink func(video.Sample)) error {++ for {++ select {++ case <-ctx.Done():++ return nil++ case v := <-s.out:++ sink(video.Sample{Data: []byte(v), Duration: 10 * time.Millisecond})++ }++ }++}++++func newTestAgent(t *testing.T, api *fakeAPI, o AgentOptions) (*Agent, *fakePub) {++ t.Helper()++ if o.ID == "" {++ o.ID = testID++ }++ if o.Publish.WHIPURL == "" {++ o.Publish = Publish{WHIPURL: "https://whip.example/whip/agent/" + testID, Ticket: credentials.NewSecret(testTicket),++ ExpiresAt: time.Now().Add(4 * time.Hour)}++ }++ if o.Capture.Kind == "" {++ o.Capture = Capture{Kind: CaptureTest, Width: 320, Height: 240}++ }++ if o.Log == nil {++ o.Log = slog.New(slog.NewTextHandler(io.Discard, nil))++ }++ if o.Snapshot == nil {++ o.Snapshot = func(context.Context) ([]byte, error) { return blackJPEG(64, 36), nil }++ }++ var pub *fakePub++ if o.NewPublisher == nil {++ pub = &fakePub{}++ o.NewPublisher = func(video.Options) publisher { return pub }++ }++ if o.NewSource == nil {++ o.NewSource = func(Capture, EncodeOptions) video.Source { return &fakeSource{out: make(chan string, 8)} }++ }++ a, err := NewAgent(o)++ if err != nil {++ t.Fatal(err)++ }++ return a, pub++}++++func TestAgentSnapshotModeWhenLiveSaysNoEncode(t *testing.T) {++ var no atomic.Bool++ api := &fakeAPI{}++ api.state = func() State {++ if no.Load() {++ return State{Encode: false}++ }++ return State{Encode: true}++ }++ a, pub := newTestAgent(t, api, AgentOptions{Client: api, StateEvery: 10 * time.Millisecond,++ SnapshotEvery: 15 * time.Millisecond, CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}},++ CurtainJPEG: blackJPEG(16, 9)})++ ctx, cancel := context.WithCancel(context.Background())++ done := make(chan struct{})++ go func() { a.Run(ctx); close(done) }()++ no.Store(true)++ deadline := time.Now().Add(5 * time.Second)++ for a.Status().Encode {++ if time.Now().After(deadline) {++ t.Fatal("encode never went false")++ }++ time.Sleep(5 * time.Millisecond)++ }++ time.Sleep(30 * time.Millisecond)++ before := atomic.LoadInt32(&api.frames)++ time.Sleep(80 * time.Millisecond)++ after := atomic.LoadInt32(&api.frames)++ if after <= before {++ t.Fatalf("no snapshot while encode is false (%d → %d)", before, after)++ }++ if !pub.paused.Load() {++ t.Fatal("the publisher was not paused in snapshot mode")++ }++ cancel()++ <-done++ if atomic.LoadInt32(&api.ended) != 1 {++ t.Fatalf("End called %d times", api.ended)++ }++}++++func TestAgentRenewsAtThreeHours(t *testing.T) {++ api := &fakeAPI{newKey: testTicket[:len(testTicket)-1] + "Z"}++ a, pub := newTestAgent(t, api, AgentOptions{Client: api, Now: func() time.Time { return time.Unix(1_800_000_000, 0) }})++ if d := a.renewDelay(); d != 3*time.Hour {++ t.Fatalf("renewDelay = %v, want 3h", d)++ }++ // A ticket that dies sooner than three hours is renewed before it does.++ a.expiresAt = a.opt.Now().Add(1 * time.Hour)++ if d := a.renewDelay(); d != 45*time.Minute {++ t.Fatalf("renewDelay with a one-hour ticket = %v, want 45m", d)++ }++ // And the renewal replaces the key the next WHIP session reconnects with.++ a2, pub2 := newTestAgent(t, api, AgentOptions{Client: api, RenewAfter: 20 * time.Millisecond,++ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})++ ctx, cancel := context.WithCancel(context.Background())++ go a2.Run(ctx)++ time.Sleep(80 * time.Millisecond)++ cancel()++ api.mu.Lock()++ n := len(api.renews)++ api.mu.Unlock()++ if n == 0 {++ t.Fatal("the ticket was never renewed")++ }++ got, _ := pub2.key.Load().(credentials.Secret)++ if got.Reveal() != api.newKey {++ t.Fatalf("publisher key after renewal = %v", got)++ }++ _ = pub++}++++func TestDeskSourceCurtainBlanksEveryFrame(t *testing.T) {++ api := &fakeAPI{}++ src := &fakeSource{out: make(chan string, 8)}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api,++ NewSource: func(Capture, EncodeOptions) video.Source { return src },++ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true, Duration: 100 * time.Millisecond}}})++ ctx, cancel := context.WithCancel(context.Background())++ defer cancel()++ var mu sync.Mutex++ var seen []string++ done := make(chan struct{})++ go func() {++ _ = a.ds.Run(ctx, func(s video.Sample) {++ mu.Lock()++ seen = append(seen, string(s.Data))++ mu.Unlock()++ })++ close(done)++ }()++ send := func(v string) {++ select {++ case src.out <- v:++ case <-time.After(time.Second):++ t.Fatal("source blocked")++ }++ }++ time.Sleep(20 * time.Millisecond)++ send("RAW1")++ time.Sleep(20 * time.Millisecond)++ a.SetCurtain(true, "manual")++ time.Sleep(20 * time.Millisecond)++ send("RAW2")++ time.Sleep(150 * time.Millisecond)++ mu.Lock()++ got := append([]string(nil), seen...)++ mu.Unlock()++ if !containsStr(got, "RAW1") {++ t.Fatalf("no frame before the curtain: %v", got)++ }++ if containsStr(got, "RAW2") {++ t.Fatalf("a captured frame leaked through the curtain: %v", got)++ }++ if !containsStr(got, "CURTAIN") {++ t.Fatalf("the curtain frame was not sent: %v", got)++ }++ a.SetCurtain(false, "manual")++ time.Sleep(30 * time.Millisecond)++ send("RAW3")++ time.Sleep(30 * time.Millisecond)++ mu.Lock()++ got = append([]string(nil), seen...)++ mu.Unlock()++ if !containsStr(got, "RAW3") {++ t.Fatalf("no frame after the curtain came down: %v", got)++ }++ cancel()++ <-done++}++++func TestTheTicketNeverReachesALogOrStatus(t *testing.T) {++ var buf bytes.Buffer++ api := &fakeAPI{}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Log: slog.New(slog.NewTextHandler(&buf, nil)),++ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})++ ctx, cancel := context.WithCancel(context.Background())++ done := make(chan struct{})++ go func() { a.Run(ctx); close(done) }()++ time.Sleep(40 * time.Millisecond)++ cancel()++ <-done++ if strings.Contains(buf.String(), "wpt_") || strings.Contains(buf.String(), testTicket) {++ t.Fatalf("a ticket reached the log: %s", buf.String())++ }++ st := a.Status()++ b, _ := json.Marshal(st)++ if strings.Contains(string(b), "wpt_") {++ t.Fatalf("a ticket reached status: %s", b)++ }++ // The Secret type is enough on its own: %v, %+v and JSON all redact it.++ pub := Publish{WHIPURL: "https://whip.example/x", Ticket: credentials.NewSecret(testTicket)}++ for _, s := range []string{fmt.Sprintf("%v", pub.Ticket), fmt.Sprintf("%+v", pub), fmt.Sprintf("%#v", pub.Ticket)} {++ if strings.Contains(s, testTicket) {++ t.Fatalf("a Secret printed its value: %s", s)++ }++ }++}++++func TestRepeatedRetriesDownshiftTo720p(t *testing.T) {++ api := &fakeAPI{}++ var widths []int++ var mu sync.Mutex++ pub := &fakePub{}++ pub.state.Store(video.StateRetrying)++ a, _ := newTestAgent(t, api, AgentOptions{Client: api, StateEvery: 5 * time.Millisecond,++ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}, CurtainJPEG: blackJPEG(16, 9),++ NewPublisher: func(video.Options) publisher { return pub },++ NewSource: func(_ Capture, e EncodeOptions) video.Source {++ mu.Lock()++ widths = append(widths, e.Width)++ mu.Unlock()++ return &fakeSource{out: make(chan string, 1)}++ }})++ ctx, cancel := context.WithCancel(context.Background())++ go a.Run(ctx)++ time.Sleep(200 * time.Millisecond)++ cancel()++ mu.Lock()++ got := append([]int(nil), widths...)++ mu.Unlock()++ seen720 := false++ for _, w := range got {++ if w == 1280 {++ seen720 = true++ }++ }++ if !seen720 {++ t.Fatalf("no 720p restart: %v", got)++ }++}++++type testClock struct{ t time.Time }++++func (c *testClock) now() time.Time { return c.t }++func (c *testClock) add(d time.Duration) { c.t = c.t.Add(d) }++++func waitCurtains(t *testing.T, api *fakeAPI, want int) []string {++ t.Helper()++ deadline := time.Now().Add(2 * time.Second)++ for {++ api.mu.Lock()++ got := append([]string(nil), api.curtains...)++ api.mu.Unlock()++ if len(got) >= want {++ return got++ }++ if time.Now().After(deadline) {++ t.Fatalf("only %d curtain reports: %v", len(got), got)++ }++ time.Sleep(5 * time.Millisecond)++ }++}++++func TestPluginSilentForThreeSecondsBlanksAndMutes(t *testing.T) {++ clock := &testClock{t: time.Unix(1_800_000_000, 0)}++ api := &fakeAPI{}++ pub := &fakePub{}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",++ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,++ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})++ a.pub = pub++ clock.add(3 * time.Second)++ a.watchCurtain()++ if !a.curtainNow2() {++ t.Fatal("a silent plugin did not raise the curtain")++ }++ if !pub.muted.Load() {++ t.Fatal("a silent plugin did not mute the audio")++ }++ if !a.ds.isUp() {++ t.Fatal("the curtain gate is not up")++ }++ got := waitCurtains(t, api, 1)++ if got[0] != "detector" {++ t.Fatalf("reported reason %q, want detector", got[0])++ }++}++++func TestPluginExitRaisesAtOnceAndRefusesADown(t *testing.T) {++ clock := &testClock{t: time.Unix(1_800_000_000, 0)}++ api := &fakeAPI{}++ pub := &fakePub{}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",++ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,++ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})++ a.pub = pub++ a.PluginLost()++ if !a.curtainNow2() || !pub.muted.Load() {++ t.Fatal("PluginLost did not blank and mute at once")++ }++ // The owner's Curtain now must not uncover the desk while the plugin is gone.++ a.SetCurtain(false, "manual")++ if !a.curtainNow2() {++ t.Fatal("a down was accepted while the plugin is lost")++ }++ // Nor may the plugin's own clear heartbeat: the supervisor says it is down.++ clock.add(3 * time.Second)++ a.ObserveCurtain(false, "detector")++ a.watchCurtain()++ if !a.curtainNow2() {++ t.Fatal("a heartbeat cleared the supervisor's verdict")++ }++}++++func TestPluginBackClearsAfterTwoSeconds(t *testing.T) {++ clock := &testClock{t: time.Unix(1_800_000_000, 0)}++ api := &fakeAPI{}++ pub := &fakePub{}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",++ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,++ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})++ a.pub = pub++ a.PluginLost()++ waitCurtains(t, api, 1)++ clock.add(3 * time.Second)++ a.PluginBack()++ a.ObserveCurtain(false, "detector")++ // The plugin keeps beating once a second; each clear report refreshes its liveness without resetting the hold.++ clock.add(time.Second)++ a.ObserveCurtain(false, "detector")++ clock.add(900 * time.Millisecond)++ a.ObserveCurtain(false, "detector")++ a.watchCurtain()++ if !a.curtainNow2() {++ t.Fatal("the curtain came down before two clear seconds")++ }++ clock.add(150 * time.Millisecond)++ a.ObserveCurtain(false, "detector")++ a.watchCurtain()++ if a.curtainNow2() {++ t.Fatal("the curtain did not come down after two clear seconds")++ }++ if pub.muted.Load() {++ t.Fatal("the audio stayed muted after the curtain came down")++ }++ got := waitCurtains(t, api, 2)++ if got[len(got)-1] != "detector" {++ t.Fatalf("last reason %q", got[len(got)-1])++ }++}++++func TestAPluginThatIsBackButReportsUpKeepsTheCurtainUp(t *testing.T) {++ clock := &testClock{t: time.Unix(1_800_000_000, 0)}++ api := &fakeAPI{}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",++ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,++ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})++ a.PluginLost()++ clock.add(5 * time.Second)++ a.PluginBack()++ a.ObserveCurtain(true, "password_field")++ clock.add(5 * time.Second)++ a.watchCurtain()++ if !a.curtainNow2() {++ t.Fatal("a clear hold lowered a curtain the plugin says is up")++ }++}++++func (a *Agent) curtainNow2() bool {++ a.mu.Lock()++ defer a.mu.Unlock()++ return a.curtain++}++++func TestCurtainUpBeforeRunBlanksTheFirstFrames(t *testing.T) {++ api := &fakeAPI{}++ src := &fakeSource{out: make(chan string, 4)}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api,++ NewSource: func(Capture, EncodeOptions) video.Source { return src },++ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true, Duration: 100 * time.Millisecond}}})++ a.SetCurtain(true, "password_field")++ ctx, cancel := context.WithCancel(context.Background())++ defer cancel()++ var mu sync.Mutex++ var seen []string++ go func() {++ _ = a.ds.Run(ctx, func(s video.Sample) {++ mu.Lock()++ seen = append(seen, string(s.Data))++ mu.Unlock()++ })++ }()++ time.Sleep(20 * time.Millisecond)++ select {++ case src.out <- "RAW":++ default:++ }++ time.Sleep(150 * time.Millisecond)++ mu.Lock()++ got := append([]string(nil), seen...)++ mu.Unlock()++ if containsStr(got, "RAW") || !containsStr(got, "CURTAIN") {++ t.Fatalf("frames before the first poll: %v", got)++ }++}++++func TestAgentRevokeIsTheKillSwitch(t *testing.T) {++ api := &fakeAPI{}++ a, _ := newTestAgent(t, api, AgentOptions{Client: api, CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}},++ CurtainJPEG: blackJPEG(16, 9)})++ ctx, cancel := context.WithCancel(context.Background())++ defer cancel()++ done := make(chan struct{})++ go func() { a.Run(ctx); close(done) }()++ time.Sleep(20 * time.Millisecond)++ a.Revoke()++ select {++ case <-done:++ case <-time.After(2 * time.Second):++ t.Fatal("Revoke did not end the agent")++ }++ time.Sleep(20 * time.Millisecond)++ if atomic.LoadInt32(&api.revoked) == 0 {++ t.Fatal("Revoke was not sent to Live")++ }++ if atomic.LoadInt32(&api.ended) != 0 {++ t.Fatal("End was sent after the kill switch")++ }++}++++func containsStr(xs []string, want string) bool {++ for _, x := range xs {++ if strings.Contains(x, want) {++ return true++ }++ }++ return false++}nnew file mode 100644@@@ -0,0 +1,124 @@++package stream++++import (++ "bytes"++ "context"++ "fmt"++ "image"++ "image/jpeg"++ "os/exec"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++// PrivateScreenText is what the desk shows while the curtain is up. Viewers never see the reason, only this.++const PrivateScreenText = "Private screen"++++// MakeCurtain returns the picture and the pre-encoded frames the desk substitutes while its curtain is up. It uses++// ffmpeg when it is installed (the words "Private screen" on a dark screen) and falls back to a black frame, which is++// still a blank: the curtain never fails open.++func MakeCurtain(ctx context.Context, ffmpeg string, w, h int) ([]video.Sample, []byte) {++ w, h = even(w), even(h)++ if w <= 0 || h <= 0 {++ w, h = 1280, 720++ }++ if w > 1920 {++ h = h * 1920 / w++ w = 1920++ }++ if h > 1080 {++ w = w * 1080 / h++ h = 1080++ }++ w, h = even(w), even(h)++ if ffmpeg == "" {++ ffmpeg = "ffmpeg"++ }++ if path, err := exec.LookPath(ffmpeg); err == nil {++ if frames, err := ffmpegCurtainFrames(ctx, path, w, h); err == nil && len(frames) > 0 {++ if jpg, err := ffmpegCurtainJPEG(ctx, path, w, h); err == nil && len(jpg) > 0 {++ return frames, jpg++ }++ return frames, blackJPEG(w, h)++ }++ if jpg, err := ffmpegCurtainJPEG(ctx, path, w, h); err == nil && len(jpg) > 0 {++ return blackFrames(w, h), jpg++ }++ }++ return blackFrames(w, h), blackJPEG(w, h)++}++++func drawtext(w, h int) string {++ size := h / 18++ if size < 14 {++ size = 14++ }++ return fmt.Sprintf("drawtext=text='%s':fontcolor=0xe8e8ee:fontsize=%d:x=(w-text_w)/2:y=(h-text_h)/2",++ PrivateScreenText, size)++}++++func ffmpegCurtainFrames(ctx context.Context, ffmpeg string, w, h int) ([]video.Sample, error) {++ ctx, cancel := context.WithTimeout(ctx, 10*time.Second)++ defer cancel()++ args := []string{"-hide_banner", "-loglevel", "error", "-nostdin",++ "-f", "lavfi", "-i", fmt.Sprintf("color=c=0x101014:s=%dx%d:r=10", w, h), "-t", "1",++ "-vf", drawtext(w, h),++ "-c:v", "libx264", "-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "main",++ "-pix_fmt", "yuv420p", "-bf", "0", "-g", "10", "-f", "h264", "pipe:1"}++ out, err := exec.CommandContext(ctx, ffmpeg, args...).Output()++ if err != nil {++ return nil, err++ }++ var first *video.Sample++ err = video.AccessUnits(bytes.NewReader(out), func(au []byte, key bool) {++ if first == nil && key {++ cp := append([]byte(nil), au...)++ first = &video.Sample{Data: cp, Key: true, Duration: 100 * time.Millisecond}++ }++ })++ if err != nil {++ return nil, err++ }++ if first == nil {++ return nil, fmt.Errorf("no keyframe in the curtain")++ }++ return []video.Sample{*first}, nil++}++++func ffmpegCurtainJPEG(ctx context.Context, ffmpeg string, w, h int) ([]byte, error) {++ ctx, cancel := context.WithTimeout(ctx, 10*time.Second)++ defer cancel()++ args := []string{"-hide_banner", "-loglevel", "error", "-nostdin",++ "-f", "lavfi", "-i", fmt.Sprintf("color=c=0x101014:s=%dx%d", w, h), "-frames:v", "1",++ "-vf", drawtext(w, h), "-c:v", "mjpeg", "-q:v", "5", "-f", "image2pipe", "pipe:1"}++ return exec.CommandContext(ctx, ffmpeg, args...).Output()++}++++func blackFrames(w, h int) []video.Sample {++ enc, err := video.NewPCMEncoder(w, h)++ if err != nil {++ return nil++ }++ f := video.NewFrame(w, h)++ for i := range f.Y {++ f.Y[i] = 16++ }++ for i := range f.Cb {++ f.Cb[i], f.Cr[i] = 128, 128++ }++ return []video.Sample{{Data: enc.Encode(f, true, nil), Key: true, Duration: 100 * time.Millisecond}}++}++++func blackJPEG(w, h int) []byte {++ img := image.NewRGBA(image.Rect(0, 0, w, h))++ // A dark, quiet card; the planner draws the words when ffmpeg is there.++ for i := 0; i < len(img.Pix); i += 4 {++ img.Pix[i], img.Pix[i+1], img.Pix[i+2], img.Pix[i+3] = 16, 16, 20, 255++ }++ var buf bytes.Buffer++ if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 80}); err != nil {++ return nil++ }++ return buf.Bytes()++}nnew file mode 100644@@@ -0,0 +1,42 @@++package stream++++import (++ "errors"++ "image"++ "strconv"++++ "github.com/makiuchi-d/gozxing"++ "github.com/makiuchi-d/gozxing/qrcode"++++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++// DecodeProbe reads a probe frame's QR payload: the millisecond clock the desk drew it with. It is the same decode the++// headless viewer does (ZXing there; gozxing here), so the probe can be checked without a browser.++func DecodeProbe(f *video.Frame) (int64, error) {++ return decodeQRImage(frameGray(f))++}++++func frameGray(f *video.Frame) image.Image {++ img := image.NewGray(image.Rect(0, 0, f.W, f.H))++ for y := 0; y < f.H; y++ {++ copy(img.Pix[y*img.Stride:y*img.Stride+f.W], f.Y[y*f.W:(y+1)*f.W])++ }++ return img++}++++func decodeQRImage(img image.Image) (int64, error) {++ bmp, err := gozxing.NewBinaryBitmapFromImage(img)++ if err != nil {++ return 0, err++ }++ res, err := qrcode.NewQRCodeReader().Decode(bmp, nil)++ if err != nil {++ return 0, err++ }++ ms, err := strconv.ParseInt(res.GetText(), 10, 64)++ if err != nil {++ return 0, errors.New("stream: the probe QR does not hold a millisecond clock")++ }++ return ms, nil++}nnew file mode 100644@@@ -0,0 +1,285 @@++package stream++++import (++ "context"++ "errors"++ "fmt"++ "log/slog"++ "os/exec"++ "strconv"++ "strings"++ "sync/atomic"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++// Capture kinds, one per platform the design names plus the browser desk's own surface and a synthetic source for the++// probe and tests.++const (++ CaptureX11 = "x11grab"++ CapturePipeWire = "pipewire"++ CaptureDDAGrab = "ddagrab"++ CaptureAVFoundation = "avfoundation"++ CaptureBrowser = "browser"++ CaptureTest = "test"++)++++// Capture is how the core grabs the desk's picture. The computer plugin builds it (it knows the desk's display and++// whether it is a browser desk) and the core runs ffmpeg with it.++type Capture struct {++ Kind string `json:"kind"`++ Display string `json:"display,omitempty"` // x11grab/pipewire: :99++ Pipe string `json:"pipe,omitempty"` // browser: the FIFO the page's screencast writes JPEGs into++ Device string `json:"device,omitempty"` // avfoundation: "1:none"++ PipeWire string `json:"pipewire,omitempty"` // pipewire: the node id, when the plugin has one++ Width int `json:"width,omitempty"`++ Height int `json:"height,omitempty"`++}++++// EncodeOptions is the design's encoder recipe. The zero value is 1080p30, CBR 4 Mb/s, a keyframe every two seconds,++// 0 B-frames, zerolatency and stillimage, and VFR 5–30 with duplicate frames dropped.++type EncodeOptions struct {++ Width, Height int++ FPS int++ BitrateKbps int++ Keyframe time.Duration++ // NoDecimate keeps every captured frame (probe and tests); the desk otherwise drops duplicates, which is what makes++ // the stream variable-rate.++ NoDecimate bool++ // Still keeps x264's stillimage tuning (mostly static screens). Off for the probe, where the QR moves.++ Still bool++}++++// DefaultEncode is the recipe measured in the design.++func DefaultEncode() EncodeOptions {++ return EncodeOptions{Width: 1920, Height: 1080, FPS: 30, BitrateKbps: 4000, Keyframe: 2 * time.Second, Still: true}++}++++// LowBandwidth is the 720p rung the desk falls to on a low bandwidth estimate.++func (o EncodeOptions) LowBandwidth() EncodeOptions {++ o.Width, o.Height = 1280, 720++ return o++}++++func (o EncodeOptions) withDefaults() EncodeOptions {++ if o.FPS <= 0 {++ o.FPS = 30++ }++ if o.Keyframe <= 0 {++ o.Keyframe = 2 * time.Second++ }++ if o.BitrateKbps <= 0 {++ o.BitrateKbps = 4000++ }++ if o.Width <= 0 || o.Height <= 0 {++ o.Width, o.Height = 1920, 1080++ }++ o.Width, o.Height = even(o.Width), even(o.Height)++ return o++}++++func even(n int) int {++ if n%2 != 0 {++ return n - 1++ }++ return n++}++++// Args builds the ffmpeg command line for a capture and the design's recipe. It is pure so the recipe can be checked++// without a display or a camera.++func Args(ffmpeg string, c Capture, o EncodeOptions) ([]string, error) {++ if ffmpeg == "" {++ ffmpeg = "ffmpeg"++ }++ o = o.withDefaults()++ if c.Width > 0 && c.Height > 0 { // the desk's own size, unless the caller asked for a ladder rung++ if o.Width == 1920 && o.Height == 1080 {++ o.Width, o.Height = even(c.Width), even(c.Height)++ }++ }++ args := []string{"-hide_banner", "-loglevel", "error", "-nostdin"}++++ var filter []string++ switch c.Kind {++ case CaptureX11:++ display := c.Display++ if display == "" {++ return nil, errors.New("stream: x11grab needs a display (:N)")++ }++ if !strings.Contains(display, ".") {++ display += ".0" // the plugin may already name the screen (":90.0")++ }++ args = append(args, "-f", "x11grab", "-draw_mouse", "1", "-framerate", strconv.Itoa(o.FPS),++ "-video_size", fmt.Sprintf("%dx%d", o.Width, o.Height), "-i", display)++ case CapturePipeWire:++ // ffmpeg 7.1's pipewire input; the plugin only chooses it when ffmpeg reports the device.++ node := c.PipeWire++ if node == "" {++ return nil, errors.New("stream: pipewire needs a node id")++ }++ args = append(args, "-f", "pipewire", "-framerate", strconv.Itoa(o.FPS), "-i", node)++ case CaptureDDAGrab:++ args = append(args, "-f", "lavfi", "-i",++ fmt.Sprintf("ddagrab=output_idx=0:framerate=%d:video_size=%dx%d", o.FPS, o.Width, o.Height))++ filter = append(filter, "hwdownload", "format=bgra")++ case CaptureAVFoundation:++ device := c.Device++ if device == "" {++ device = "1:none"++ }++ args = append(args, "-f", "avfoundation", "-capture_cursor", "1", "-framerate", strconv.Itoa(o.FPS), "-i", device)++ case CaptureBrowser:++ if c.Pipe == "" {++ return nil, errors.New("stream: a browser desk needs its screencast pipe")++ }++ args = append(args, "-f", "image2pipe", "-framerate", strconv.Itoa(o.FPS), "-i", c.Pipe)++ case CaptureTest:++ args = append(args, "-f", "lavfi", "-i", fmt.Sprintf("testsrc2=size=%dx%d:rate=%d", o.Width, o.Height, o.FPS))++ default:++ return nil, fmt.Errorf("stream: unknown capture %q", c.Kind)++ }++++ // Variable rate: drop frames that did not change. -fps_mode vfr keeps the capture's timestamps, so a still screen++ // costs about 5 frames a second and a moving one up to 30. The source's timing is the only clock; nothing is++ // duplicated to a fixed rate.++ if !o.NoDecimate {++ filter = append(filter, "mpdecimate=hi=64*12:lo=64*5:frac=0.33")++ }++ if len(filter) > 0 {++ args = append(args, "-vf", strings.Join(filter, ","))++ }++ // The cap, not a constant rate: -fps_mode vfr keeps the capture's own timestamps and -r would contradict it.++ args = append(args, "-fps_mode", "vfr")++++ // H.264 main, no B-frames, zerolatency, keyframe every two seconds of wall clock even when frames were dropped.++ args = append(args,++ "-c:v", "libx264",++ "-preset", "ultrafast",++ "-tune", tune(o.Still),++ "-profile:v", "main",++ "-pix_fmt", "yuv420p",++ "-bf", "0",++ "-g", strconv.Itoa(o.FPS*int(o.Keyframe.Seconds())),++ "-keyint_min", strconv.Itoa(o.FPS*int(o.Keyframe.Seconds())),++ "-sc_threshold", "0",++ "-force_key_frames", fmt.Sprintf("expr:gte(t,n_forced*%g)", o.Keyframe.Seconds()),++ "-b:v", fmt.Sprintf("%dk", o.BitrateKbps),++ "-maxrate", fmt.Sprintf("%dk", o.BitrateKbps),++ "-bufsize", fmt.Sprintf("%dk", o.BitrateKbps),++ "-x264-params", "nal-hrd=cbr",++ "-f", "h264", "pipe:1",++ )++ return args, nil++}++++func tune(still bool) string {++ if still {++ return "zerolatency,stillimage"++ }++ return "zerolatency"++}++++// FFmpegSource runs one ffmpeg capture+encode pipeline and hands its Annex B access units to the publisher. It restarts++// the child with backoff when it exits, like a camera source does.++type FFmpegSource struct {++ FFmpeg string++ Capture Capture++ Options EncodeOptions++ Log *slog.Logger++ keyReq atomic.Bool++}++++func (s *FFmpegSource) Name() string { return "ffmpeg:" + s.Capture.Kind }++func (s *FFmpegSource) RequestKeyframe() { s.keyReq.Store(true) }++++// Run restarts ffmpeg until ctx ends. A keyframe request cannot reach an already-running ffmpeg; the two-second++// force_key_frames expression covers a new viewer, and a restart covers the rest.++func (s *FFmpegSource) Run(ctx context.Context, sink func(video.Sample)) error {++ backoff := time.Second++ for ctx.Err() == nil {++ start := time.Now()++ err := s.runOnce(ctx, sink)++ if ctx.Err() != nil {++ return nil++ }++ if time.Since(start) > time.Minute {++ backoff = time.Second++ }++ if s.Log != nil {++ s.Log.Warn("stream capture exited; restarting", "capture", s.Capture.Kind, "err", err, "in", backoff)++ }++ select {++ case <-ctx.Done():++ return nil++ case <-time.After(backoff):++ }++ if backoff < 30*time.Second {++ backoff *= 2++ }++ }++ return nil++}++++func (s *FFmpegSource) runOnce(ctx context.Context, sink func(video.Sample)) error {++ ff := s.FFmpeg++ if ff == "" {++ ff = "ffmpeg"++ }++ args, err := Args(ff, s.Capture, s.Options)++ if err != nil {++ return err++ }++ cmd := exec.CommandContext(ctx, ff, args...)++ var stderr strings.Builder++ cmd.Stderr = &limitedWriter{w: &stderr, n: 4096}++ out, err := cmd.StdoutPipe()++ if err != nil {++ return err++ }++ if err := cmd.Start(); err != nil {++ return err++ }++ last := time.Now()++ perr := video.AccessUnits(out, func(au []byte, key bool) {++ now := time.Now()++ d := now.Sub(last)++ if d < time.Millisecond {++ d = time.Millisecond++ }++ if d > time.Second {++ d = time.Second++ }++ last = now++ sink(video.Sample{Data: au, Key: key, Duration: d})++ })++ werr := cmd.Wait()++ if perr != nil {++ return perr++ }++ if werr != nil {++ return fmt.Errorf("%w: %s", werr, strings.TrimSpace(stderr.String()))++ }++ return errors.New("ffmpeg exited")++}++++type limitedWriter struct {++ w interface{ Write([]byte) (int, error) }++ n int++}++++func (l *limitedWriter) Write(p []byte) (int, error) {++ if l.n <= 0 {++ return len(p), nil++ }++ q := p++ if len(q) > l.n {++ q = q[:l.n]++ }++ l.n -= len(q)++ _, _ = l.w.Write(q)++ return len(p), nil++}++++// probeCapture is the source the latency probe draws its QR clock on.++func probeCapture(w, h int) Capture { return Capture{Kind: CaptureTest, Width: w, Height: h} }nnew file mode 100644@@@ -0,0 +1,163 @@++package stream++++import (++ "context"++ "os/exec"++ "strings"++ "testing"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++func hasArg(args []string, want string) bool {++ for i := 0; i < len(args); i++ {++ if args[i] == want {++ return true++ }++ }++ return false++}++++func argAfter(args []string, flag string) string {++ out := ""++ for i := 0; i+1 < len(args); i++ {++ if args[i] == flag {++ out = args[i+1]++ }++ }++ return out++}++++func TestEncoderRecipe(t *testing.T) {++ args, err := Args("ffmpeg", Capture{Kind: CaptureX11, Display: ":99", Width: 1600, Height: 900}, DefaultEncode())++ if err != nil {++ t.Fatal(err)++ }++ checks := map[string]string{++ "-bf": "0",++ "-profile:v": "main",++ "-preset": "ultrafast",++ "-tune": "zerolatency,stillimage",++ "-g": "60",++ "-keyint_min": "60",++ "-force_key_frames": "expr:gte(t,n_forced*2)",++ "-b:v": "4000k",++ "-maxrate": "4000k",++ "-bufsize": "4000k",++ "-fps_mode": "vfr",++ "-x264-params": "nal-hrd=cbr",++ "-f": "h264",++ }++ for flag, want := range checks {++ if got := argAfter(args, flag); got != want {++ t.Errorf("%s = %q, want %q (args %v)", flag, got, want, args)++ }++ }++ if !strings.Contains(argAfter(args, "-vf"), "mpdecimate") {++ t.Errorf("no duplicate-frame dropping: %v", args)++ }++ if got := argAfter(args, "-video_size"); got != "1600x900" {++ t.Errorf("capture size = %q", got)++ }++ if !hasArg(args, "pipe:1") {++ t.Errorf("no Annex B output: %v", args)++ }++}++++func TestEncoderLowBandwidthRung(t *testing.T) {++ args, err := Args("ffmpeg", Capture{Kind: CaptureX11, Display: ":99", Width: 1920, Height: 1080},++ DefaultEncode().LowBandwidth())++ if err != nil {++ t.Fatal(err)++ }++ if got := argAfter(args, "-video_size"); got != "1280x720" {++ t.Errorf("720p rung = %q", got)++ }++}++++func TestEachCaptureKind(t *testing.T) {++ cases := []Capture{++ {Kind: CaptureX11, Display: ":97", Width: 800, Height: 600},++ {Kind: CaptureX11, Display: ":97.0", Width: 800, Height: 600},++ {Kind: CapturePipeWire, PipeWire: "42", Width: 800, Height: 600},++ {Kind: CaptureDDAGrab, Width: 800, Height: 600},++ {Kind: CaptureAVFoundation, Device: "2:none"},++ {Kind: CaptureBrowser, Pipe: "/run/ov/page.fifo"},++ {Kind: CaptureTest, Width: 800, Height: 600},++ }++ for _, c := range cases {++ args, err := Args("ffmpeg", c, EncodeOptions{Width: 800, Height: 600, FPS: 30})++ if err != nil {++ t.Errorf("%s: %v", c.Kind, err)++ continue++ }++ joined := strings.Join(args, " ")++ switch c.Kind {++ case CaptureX11:++ if !strings.Contains(joined, "-f x11grab") || !strings.Contains(joined, ":97.0") || strings.Contains(joined, ":97.0.0") {++ t.Errorf("x11grab args: %s", joined)++ }++ case CapturePipeWire:++ if !strings.Contains(joined, "-f pipewire") || !strings.Contains(joined, " 42 ") {++ t.Errorf("pipewire args: %s", joined)++ }++ case CaptureDDAGrab:++ if !strings.Contains(joined, "ddagrab=output_idx=0") || !strings.Contains(argAfter(args, "-vf"), "hwdownload") {++ t.Errorf("ddagrab args: %s", joined)++ }++ case CaptureAVFoundation:++ if !strings.Contains(joined, "-f avfoundation") || !strings.Contains(joined, "2:none") {++ t.Errorf("avfoundation args: %s", joined)++ }++ case CaptureBrowser:++ if !strings.Contains(joined, "-f image2pipe") || !strings.Contains(joined, "/run/ov/page.fifo") {++ t.Errorf("browser args: %s", joined)++ }++ case CaptureTest:++ if !strings.Contains(joined, "testsrc2=") {++ t.Errorf("test args: %s", joined)++ }++ }++ }++}++++func TestCaptureNeedsItsInput(t *testing.T) {++ if _, err := Args("ffmpeg", Capture{Kind: CaptureX11}, DefaultEncode()); err == nil {++ t.Error("x11grab without a display was accepted")++ }++ if _, err := Args("ffmpeg", Capture{Kind: CaptureBrowser}, DefaultEncode()); err == nil {++ t.Error("a browser desk without a pipe was accepted")++ }++ if _, err := Args("ffmpeg", Capture{Kind: "nope"}, DefaultEncode()); err == nil {++ t.Error("an unknown capture was accepted")++ }++}++++func TestFFmpegSourcePublishesAccessUnits(t *testing.T) {++ ff, err := exec.LookPath("ffmpeg")++ if err != nil {++ t.Skip("no ffmpeg on this machine")++ }++ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)++ defer cancel()++ src := &FFmpegSource{FFmpeg: ff, Capture: Capture{Kind: CaptureTest, Width: 320, Height: 240},++ Options: EncodeOptions{Width: 320, Height: 240, FPS: 15, Keyframe: time.Second}}++ samples := make(chan video.Sample, 8)++ go func() {++ _ = src.Run(ctx, func(s video.Sample) {++ select {++ case samples <- s:++ default:++ }++ })++ }()++ select {++ case s := <-samples:++ if len(s.Data) == 0 || s.Duration <= 0 {++ t.Fatalf("empty sample: %+v", s)++ }++ case <-time.After(10 * time.Second):++ t.Fatal("no access units from ffmpeg")++ }++}nnew file mode 100644@@@ -0,0 +1,403 @@++// Package stream is Node's part of Agent Streams S1: the desk stream. It publishes a desk's screen to OpenVibe.Live++// over WHIP with a publish ticket (wpt_…), polls the stream's state, sends snapshots while nobody watches, renews the++// ticket at three hours, blanks the picture behind the curtain, and measures glass-to-glass latency with a QR probe.++//++// The design is plan-state-2026-10-22/AGENT-STREAMS-S1-DESIGN-2026-11-06.md. Tickets and tokens are credentials.Secret:++// they go on the wire and nowhere else, never into a log line, an error or `openvibe-node status`.++package stream++++import (++ "bytes"++ "context"++ "encoding/json"++ "errors"++ "fmt"++ "io"++ "net"++ "net/http"++ "net/url"++ "regexp"++ "strings"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++)++++// DefaultLiveURL is where the agent-stream API lives.++const DefaultLiveURL = "https://openvibe.live"++++// ErrTicketInvalid is Live refusing a wpt_ (401 invalid_ticket): it is wrong, expired, revoked or names another stream.++var ErrTicketInvalid = errors.New("the publish ticket was refused (invalid_ticket)")++++// ErrAlreadyPublishing is a second WHIP session for a stream that already has one (409 already_publishing).++var ErrAlreadyPublishing = errors.New("this stream already has a WHIP session (already_publishing)")++++var (++ idRe = regexp.MustCompile(`^ags_[0-9A-HJKMNP-TV-Z]{26}$`)++ ticketRe = regexp.MustCompile(`wpt_[A-Za-z0-9_-]{43}`)++ viewTickRe = regexp.MustCompile(`wvt_[A-Za-z0-9_-]{43}`)++ bearerAnyRe = regexp.MustCompile(`(?i)bearer\s+[A-Za-z0-9._~+/-]{8,}`)++)++++// Task is what the agent is doing and where that task lives (live.agent-stream@1 task).++type Task struct {++ Source string `json:"source"` // codes, actor, desk or node++ Ref string `json:"ref"`++ Title string `json:"title"`++ URL string `json:"url,omitempty"`++}++++// CreateRequest is the body of POST /api/streams/agent.++type CreateRequest struct {++ Title string `json:"title"`++ Task Task `json:"task"`++ Visibility string `json:"visibility"`++ AllowSites []string `json:"allow_sites,omitempty"`++}++++// Stream is live.agent-stream@1, the fields Node needs.++type Stream struct {++ ID string `json:"id"`++ Owner string `json:"owner"`++ Channel string `json:"channel"`++ Title string `json:"title"`++ Task Task `json:"task"`++ Visibility string `json:"visibility"`++ Mode string `json:"mode"`++ State string `json:"state"`++ Viewers int `json:"viewers"`++ WorkedSeconds int64 `json:"worked_seconds"`++ PageURL string `json:"page_url"`++ Label string `json:"label"`++ StartedAt *string `json:"started_at"`++ EndedAt *string `json:"ended_at"`++}++++// Publish is live.agent-stream-publish@1: where the desk publishes and with what.++type Publish struct {++ WHIPURL string `json:"whip_url"`++ Ticket credentials.Secret `json:"-"`++ ExpiresAt time.Time `json:"-"`++}++++// publishWire is Publish on the wire; the ticket is read once and wrapped in a Secret.++type publishWire struct {++ WHIPURL string `json:"whip_url"`++ Ticket string `json:"ticket"`++ ExpiresAt string `json:"expires_at"`++}++++// Curtain is the desk's privacy screen as Live records it.++type Curtain struct {++ Up bool `json:"up"`++ Since *time.Time `json:"since,omitempty"`++ Reason string `json:"reason,omitempty"`++}++++// State is live.agent-stream-state@1, what the desk reads every ten seconds.++type State struct {++ Viewers int `json:"viewers"`++ Encode bool `json:"encode"`++ Curtain Curtain `json:"curtain"`++ EndsAt *time.Time `json:"ends_at"`++}++++// CreateResult is the 201 answer of POST /api/streams/agent.++type CreateResult struct {++ Stream Stream `json:"stream"`++ Publish Publish `json:"publish"`++}++++// Client talks to Live's agent-stream API. Token buys the node token that POST /api/streams/agent authenticates with;++// every other call uses the stream's publish ticket.++type Client struct {++ base string++ token func(context.Context) (credentials.Secret, error)++ http *http.Client++ ua string++}++++// ClientOptions configure a Client.++type ClientOptions struct {++ BaseURL string++ Token func(context.Context) (credentials.Secret, error)++ HTTPClient *http.Client++ UserAgent string++}++++// NewClient returns a client for base (DefaultLiveURL when empty). Plain http is accepted only for a loopback host,++// which is what tests use.++func NewClient(o ClientOptions) (*Client, error) {++ base := strings.TrimSuffix(o.BaseURL, "/")++ if base == "" {++ base = DefaultLiveURL++ }++ u, err := url.Parse(base)++ if err != nil || u.Host == "" || u.User != nil {++ return nil, errors.New("stream: the Live URL is not an origin")++ }++ if u.Scheme != "https" && !(u.Scheme == "http" && isLoopback(u.Hostname())) {++ return nil, errors.New("stream: the Live URL must be https")++ }++ if o.Token == nil {++ return nil, errors.New("stream: no node token source (pair this machine with OpenVibe.Network)")++ }++ c := &Client{base: base, token: o.Token, http: o.HTTPClient, ua: o.UserAgent}++ if c.http == nil {++ c.http = &http.Client{Timeout: 20 * time.Second}++ }++ if c.ua == "" {++ c.ua = "openvibe-node"++ }++ return c, nil++}++++func isLoopback(host string) bool {++ if host == "localhost" {++ return true++ }++ ip := net.ParseIP(host)++ return ip != nil && ip.IsLoopback()++}++++// Create calls POST /api/streams/agent with the machine's node token (audience openvibe.live). It is the only call++// that carries a token; the answer's ticket is stored in a Secret and never logged.++func (c *Client) Create(ctx context.Context, req CreateRequest) (CreateResult, error) {++ var out CreateResult++ if err := validateCreate(req); err != nil {++ return out, err++ }++ tok, err := c.token(ctx)++ if err != nil {++ return out, err++ }++ var wire struct {++ Stream Stream `json:"stream"`++ Publish publishWire `json:"publish"`++ }++ if err := c.do(ctx, http.MethodPost, "/api/streams/agent", tok, req, &wire); err != nil {++ return out, err++ }++ pub, err := parsePublish(wire.Publish)++ if err != nil {++ return out, err++ }++ out.Stream, out.Publish = wire.Stream, pub++ return out, nil++}++++// State calls GET /api/streams/agent/:id/state with the ticket.++func (c *Client) State(ctx context.Context, id string, ticket credentials.Secret) (State, error) {++ var out State++ if err := checkID(id); err != nil {++ return out, err++ }++ err := c.do(ctx, http.MethodGet, "/api/streams/agent/"+id+"/state", ticket, nil, &out)++ return out, err++}++++// Renew calls POST /api/streams/agent/:id/ticket with the ticket and returns the new one. The desk renews at 3 h.++func (c *Client) Renew(ctx context.Context, id string, ticket credentials.Secret) (Publish, error) {++ var out Publish++ if err := checkID(id); err != nil {++ return out, err++ }++ var wire publishWire++ if err := c.do(ctx, http.MethodPost, "/api/streams/agent/"+id+"/ticket", ticket, nil, &wire); err != nil {++ return out, err++ }++ return parsePublish(wire)++}++++// Revoke calls DELETE /api/streams/agent/:id/ticket: the kill switch. Live ends the WHIP session at once.++func (c *Client) Revoke(ctx context.Context, id string, ticket credentials.Secret) error {++ if err := checkID(id); err != nil {++ return err++ }++ return c.do(ctx, http.MethodDelete, "/api/streams/agent/"+id+"/ticket", ticket, nil, nil)++}++++// Curtain reports the desk's privacy screen: POST /api/streams/agent/:id/curtain {up, reason}.++func (c *Client) Curtain(ctx context.Context, id string, ticket credentials.Secret, up bool, reason string) error {++ if err := checkID(id); err != nil {++ return err++ }++ if !validReason(reason) {++ return fmt.Errorf("stream: curtain reason %q is not one of password_field, secret_file, env_print, manual, site, detector", reason)++ }++ return c.do(ctx, http.MethodPost, "/api/streams/agent/"+id+"/curtain", ticket,++ map[string]any{"up": up, "reason": reason}, nil)++}++++// PostFrame sends a snapshot (JPEG or WebP, at most 200 KB): PUT /api/streams/agent/:id/frame.++func (c *Client) PostFrame(ctx context.Context, id string, ticket credentials.Secret, frame []byte) error {++ if err := checkID(id); err != nil {++ return err++ }++ if len(frame) == 0 || len(frame) > MaxFrameBytes {++ return fmt.Errorf("stream: a snapshot is 1 to %d bytes (got %d)", MaxFrameBytes, len(frame))++ }++ typ := http.DetectContentType(frame)++ if typ != "image/jpeg" && typ != "image/webp" {++ return fmt.Errorf("stream: a snapshot must be JPEG or WebP (got %s)", typ)++ }++ return c.doRaw(ctx, http.MethodPut, "/api/streams/agent/"+id+"/frame", ticket, typ, frame, nil)++}++++// End calls POST /api/streams/agent/:id/end.++func (c *Client) End(ctx context.Context, id string, ticket credentials.Secret) error {++ if err := checkID(id); err != nil {++ return err++ }++ return c.do(ctx, http.MethodPost, "/api/streams/agent/"+id+"/end", ticket, nil, nil)++}++++// MaxFrameBytes is the snapshot cap in the design.++const MaxFrameBytes = 200 * 1024++++func validateCreate(r CreateRequest) error {++ if strings.TrimSpace(r.Title) == "" || len(r.Title) > 200 {++ return errors.New("stream: title is 1 to 200 characters")++ }++ switch r.Task.Source {++ case "codes", "actor", "desk", "node":++ default:++ return errors.New("stream: task.source is codes, actor, desk or node")++ }++ if r.Task.Ref == "" || len(r.Task.Ref) > 200 || r.Task.Title == "" || len(r.Task.Title) > 200 {++ return errors.New("stream: task.ref and task.title are required (at most 200 characters)")++ }++ if r.Task.URL != "" {++ if u, err := url.Parse(r.Task.URL); err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {++ return errors.New("stream: task.url must be an http(s) URL")++ }++ }++ switch r.Visibility {++ case "private", "unlisted", "public":++ default:++ return errors.New("stream: visibility is private, unlisted or public")++ }++ if len(r.AllowSites) > 50 {++ return errors.New("stream: allow_sites carries at most 50 hosts")++ }++ for _, s := range r.AllowSites {++ if u, err := url.Parse(s); err != nil || u.Host == "" {++ return fmt.Errorf("stream: allow_sites entry %q is not a URL", s)++ }++ }++ return nil++}++++func checkID(id string) error {++ if !idRe.MatchString(id) {++ return fmt.Errorf("stream: %q is not an agent-stream id (ags_…)", id)++ }++ return nil++}++++func validReason(r string) bool {++ switch r {++ case "password_field", "secret_file", "env_print", "manual", "site", "detector":++ return true++ }++ return false++}++++func parsePublish(w publishWire) (Publish, error) {++ var out Publish++ if w.WHIPURL == "" || w.Ticket == "" || w.ExpiresAt == "" {++ return out, errors.New("stream: the publish part is missing whip_url, ticket or expires_at")++ }++ u, err := url.Parse(w.WHIPURL)++ if err != nil || u.Host == "" || u.User != nil {++ return out, errors.New("stream: the publish whip_url is not a URL")++ }++ if !ticketRe.MatchString(w.Ticket) || !strings.HasPrefix(w.Ticket, "wpt_") {++ return out, errors.New("stream: the publish ticket is not a wpt_…")++ }++ exp, err := time.Parse(time.RFC3339, w.ExpiresAt)++ if err != nil {++ return out, errors.New("stream: expires_at is not RFC 3339")++ }++ out.WHIPURL, out.Ticket, out.ExpiresAt = w.WHIPURL, credentials.NewSecret(w.Ticket), exp++ return out, nil++}++++func (c *Client) do(ctx context.Context, method, path string, auth credentials.Secret, body any, out any) error {++ var raw []byte++ typ := ""++ if body != nil {++ b, err := json.Marshal(body)++ if err != nil {++ return err++ }++ raw, typ = b, "application/json"++ }++ return c.doRaw(ctx, method, path, auth, typ, raw, out)++}++++func (c *Client) doRaw(ctx context.Context, method, path string, auth credentials.Secret, typ string, body []byte, out any) error {++ var rdr io.Reader++ if body != nil {++ rdr = bytes.NewReader(body)++ }++ req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr)++ if err != nil {++ return err++ }++ if typ != "" {++ req.Header.Set("Content-Type", typ)++ }++ // A ticket is sent only to the configured Live origin, never after a redirect to another host.++ req.Header.Set("User-Agent", c.ua)++ if !auth.IsZero() {++ req.Header.Set("Authorization", "Bearer "+auth.Reveal())++ }++ resp, err := c.http.Do(req)++ if err != nil {++ return errors.New(redact(err.Error()))++ }++ defer resp.Body.Close()++ raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))++ if resp.StatusCode/100 != 2 {++ var pe struct {++ Code string `json:"code"`++ Error string `json:"error"`++ Detail string `json:"detail"`++ }++ _ = json.Unmarshal(raw, &pe)++ code := pe.Code++ if code == "" {++ code = pe.Error++ }++ switch {++ case resp.StatusCode == http.StatusUnauthorized && (code == "invalid_ticket" || code == ""):++ return ErrTicketInvalid++ case resp.StatusCode == http.StatusConflict && code == "already_publishing":++ return ErrAlreadyPublishing++ }++ msg := strings.TrimSpace(code + " " + pe.Detail)++ if msg == "" {++ msg = strings.TrimSpace(string(raw))++ if len(msg) > 200 {++ msg = msg[:200]++ }++ }++ return fmt.Errorf("stream: Live answered HTTP %d %s", resp.StatusCode, redact(msg))++ }++ if out != nil {++ if err := json.Unmarshal(raw, out); err != nil {++ return errors.New("stream: Live's answer is not valid JSON")++ }++ }++ return nil++}++++// redact removes anything that looks like a publish or view ticket, and any bearer value, from a message that may be++// logged or returned in an error.++func redact(s string) string {++ s = ticketRe.ReplaceAllString(s, "[ticket]")++ s = viewTickRe.ReplaceAllString(s, "[view-ticket]")++ s = bearerAnyRe.ReplaceAllString(s, "Bearer [redacted]")++ return s++}nnew file mode 100644@@@ -0,0 +1,193 @@++package stream++++import (++ "context"++ "encoding/json"++ "errors"++ "io"++ "net/http"++ "net/http/httptest"++ "strings"++ "sync"++ "testing"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++)++++const (++ testTicket = "wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG"++ testID = "ags_01J8Z4M2Q0R7T9YV3K6N8P1W2X"++)++++type recorded struct {++ method, path, auth, ctype string++ body []byte++}++++func liveFake(t *testing.T, h func(w http.ResponseWriter, r *http.Request, rec recorded)) (*Client, *[]recorded) {++ t.Helper()++ var mu sync.Mutex++ var got []recorded++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {++ b, _ := io.ReadAll(r.Body)++ rec := recorded{method: r.Method, path: r.URL.Path, auth: r.Header.Get("Authorization"), ctype: r.Header.Get("Content-Type"), body: b}++ mu.Lock()++ got = append(got, rec)++ mu.Unlock()++ h(w, r, rec)++ }))++ t.Cleanup(srv.Close)++ c, err := NewClient(ClientOptions{BaseURL: srv.URL, Token: func(context.Context) (credentials.Secret, error) {++ return credentials.NewSecret("node-token"), nil++ }, UserAgent: "test"})++ if err != nil {++ t.Fatal(err)++ }++ return c, &got++}++++func TestCreateParsesPublishAndNeverLogsTheTicket(t *testing.T) {++ c, got := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {++ if rec.method != "POST" || rec.path != "/api/streams/agent" {++ t.Errorf("request %s %s", rec.method, rec.path)++ }++ if rec.auth != "Bearer node-token" {++ t.Errorf("create auth = %q", rec.auth)++ }++ var req CreateRequest++ if err := json.Unmarshal(rec.body, &req); err != nil {++ t.Error(err)++ }++ if req.Title != "fix the tests" || req.Task.Source != "codes" || req.Visibility != "private" {++ t.Errorf("create body = %+v", req)++ }++ w.WriteHeader(http.StatusCreated)++ io.WriteString(w, `{"stream":{"id":"`+testID+`","title":"fix the tests","state":"live"},++ "publish":{"whip_url":"https://whip.openvibe.live/whip/agent/`+testID+`","ticket":"`+testTicket+`","expires_at":"2026-11-06T16:00:00Z"}}`)++ })++ res, err := c.Create(context.Background(), CreateRequest{Title: "fix the tests", Visibility: "private",++ Task: Task{Source: "codes", Ref: "job_1", Title: "fix"}})++ if err != nil {++ t.Fatal(err)++ }++ if res.Stream.ID != testID || res.Publish.WHIPURL == "" {++ t.Fatalf("create = %+v", res)++ }++ if res.Publish.Ticket.Reveal() != testTicket {++ t.Fatalf("ticket = %v", res.Publish.Ticket)++ }++ if s := res.Publish.Ticket.String(); strings.Contains(s, "wpt_") {++ t.Fatalf("Secret prints its value: %q", s)++ }++ // The ticket appears only in the wire answer; nothing this package hands out in an error or status does.++ b, _ := json.Marshal(res)++ if strings.Contains(string(b), "wpt_") {++ t.Fatalf("the ticket reached JSON: %s", b)++ }++ _ = got++}++++func TestTicketCallsUseTheTicketAsBearer(t *testing.T) {++ c, got := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {++ switch {++ case strings.HasSuffix(rec.path, "/state"):++ io.WriteString(w, `{"viewers":2,"encode":true,"curtain":{"up":false},"ends_at":"2026-11-06T16:00:00Z"}`)++ case rec.method == "POST" && strings.HasSuffix(rec.path, "/ticket"):++ io.WriteString(w, `{"whip_url":"https://whip.openvibe.live/whip/agent/`+testID+`","ticket":"`+testTicket+`","expires_at":"2026-11-06T17:00:00Z"}`)++ case rec.method == "DELETE":++ w.WriteHeader(http.StatusNoContent)++ case strings.HasSuffix(rec.path, "/curtain"):++ if string(rec.body) != `{"reason":"manual","up":true}` {++ t.Errorf("curtain body = %s", rec.body)++ }++ w.WriteHeader(http.StatusNoContent)++ case strings.HasSuffix(rec.path, "/frame"):++ if rec.ctype != "image/jpeg" || len(rec.body) < 4 {++ t.Errorf("frame = %q %d bytes", rec.ctype, len(rec.body))++ }++ w.WriteHeader(http.StatusNoContent)++ case strings.HasSuffix(rec.path, "/end"):++ w.WriteHeader(http.StatusNoContent)++ }++ })++ ctx := context.Background()++ tick := credentials.NewSecret(testTicket)++ if st, err := c.State(ctx, testID, tick); err != nil || st.Viewers != 2 || !st.Encode {++ t.Fatalf("state = %+v %v", st, err)++ }++ if p, err := c.Renew(ctx, testID, tick); err != nil || p.Ticket.Reveal() != testTicket {++ t.Fatalf("renew = %+v %v", p, err)++ }++ if err := c.Revoke(ctx, testID, tick); err != nil {++ t.Fatal(err)++ }++ if err := c.Curtain(ctx, testID, tick, true, "manual"); err != nil {++ t.Fatal(err)++ }++ jpeg := []byte{0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0}++ if err := c.PostFrame(ctx, testID, tick, jpeg); err != nil {++ t.Fatal(err)++ }++ if err := c.End(ctx, testID, tick); err != nil {++ t.Fatal(err)++ }++ for _, r := range *got {++ if r.path == "/api/streams/agent" {++ continue++ }++ if r.auth != "Bearer "+testTicket {++ t.Errorf("%s %s auth = %q", r.method, r.path, r.auth)++ }++ }++}++++func TestTicketErrors(t *testing.T) {++ c, _ := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {++ switch {++ case strings.HasSuffix(rec.path, "/state"):++ w.WriteHeader(http.StatusUnauthorized)++ io.WriteString(w, `{"code":"invalid_ticket"}`)++ case strings.HasSuffix(rec.path, "/end"):++ w.WriteHeader(http.StatusConflict)++ io.WriteString(w, `{"code":"already_publishing"}`)++ case strings.HasSuffix(rec.path, "/frame"):++ w.WriteHeader(http.StatusBadRequest)++ io.WriteString(w, `{"code":"invalid_frame","detail":"ticket `+testTicket+`"}`)++ }++ })++ ctx := context.Background()++ tick := credentials.NewSecret(testTicket)++ if _, err := c.State(ctx, testID, tick); !errors.Is(err, ErrTicketInvalid) {++ t.Fatalf("state: %v", err)++ }++ if err := c.End(ctx, testID, tick); !errors.Is(err, ErrAlreadyPublishing) {++ t.Fatalf("end: %v", err)++ }++ err := c.PostFrame(ctx, testID, tick, []byte{0xff, 0xd8, 0xff, 0xe0})++ if err == nil || strings.Contains(err.Error(), "wpt_") {++ t.Fatalf("frame error leaks a ticket: %v", err)++ }++}++++func TestCreateValidation(t *testing.T) {++ c, _ := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {++ t.Error("invalid request reached Live")++ })++ bad := []CreateRequest{++ {Title: "", Task: Task{Source: "codes", Ref: "r", Title: "t"}, Visibility: "private"},++ {Title: "t", Task: Task{Source: "other", Ref: "r", Title: "t"}, Visibility: "private"},++ {Title: "t", Task: Task{Source: "codes", Ref: "r", Title: "t"}, Visibility: "friends"},++ {Title: "t", Task: Task{Source: "codes", Ref: "r", Title: "t"}, Visibility: "private", AllowSites: []string{"not a url"}},++ }++ for i, r := range bad {++ if _, err := c.Create(context.Background(), r); err == nil {++ t.Errorf("case %d accepted", i)++ }++ }++ if err := c.Curtain(context.Background(), testID, credentials.NewSecret(testTicket), true, "because"); err == nil {++ t.Error("an unknown curtain reason was accepted")++ }++ if err := c.PostFrame(context.Background(), testID, credentials.NewSecret(testTicket), make([]byte, MaxFrameBytes+1)); err == nil {++ t.Error("an oversized frame was accepted")++ }++}nnew file mode 100644@@@ -0,0 +1,219 @@++package stream++++import (++ "context"++ "errors"++ "fmt"++ "log/slog"++ "net/url"++ "strings"++ "sync"++ "time"++++ qrcode "github.com/skip2/go-qrcode"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++// Probe QR: each frame carries the millisecond clock the desk drew it. The headless viewer decodes it and subtracts++// the instant it was decoded, which is the glass-to-glass latency.++const (++ ProbeDefaultSeconds = 60++ ProbeDefaultWidth = 640++ ProbeDefaultHeight = 360++ ProbeDefaultFPS = 10++)++++// ProbeOptions configure `openvibe-node stream --probe`.++type ProbeOptions struct {++ WHIPURL string++ Ticket credentials.Secret++ Seconds int++ Width int++ Height int++ FPS int++ Loopback bool++ Log *slog.Logger++ Now func() time.Time++ // NewPublisher is for tests.++ NewPublisher func(video.Options) publisher++}++++// ProbeSource draws a QR of the current millisecond clock in the top-left corner of every frame, on the built-in PCM++// encoder (no ffmpeg, no display). It is the clock the viewer reads.++type ProbeSource struct {++ W, H, FPS int++ Now func() time.Time++ mu sync.Mutex++ lastMS int64++ keyReq bool++}++++func (p *ProbeSource) Name() string { return "probe" }++++func (p *ProbeSource) RequestKeyframe() {++ p.mu.Lock()++ p.keyReq = true++ p.mu.Unlock()++}++++// LastMS is the clock value of the last frame drawn.++func (p *ProbeSource) LastMS() int64 {++ p.mu.Lock()++ defer p.mu.Unlock()++ return p.lastMS++}++++func (p *ProbeSource) Run(ctx context.Context, sink func(video.Sample)) error {++ if p.W <= 0 || p.H <= 0 {++ p.W, p.H = ProbeDefaultWidth, ProbeDefaultHeight++ }++ if p.FPS <= 0 {++ p.FPS = ProbeDefaultFPS++ }++ if p.Now == nil {++ p.Now = time.Now++ }++ w, h := mbSize(p.W), mbSize(p.H)++ enc, err := video.NewPCMEncoder(w, h)++ if err != nil {++ return err++ }++ frame := time.Second / time.Duration(p.FPS)++ tick := time.NewTicker(frame)++ defer tick.Stop()++ var prev *video.Frame++ for n := 0; ; n++ {++ now := p.Now()++ f := video.NewFrame(w, h)++ drawProbe(f, now.UnixMilli())++ p.mu.Lock()++ p.lastMS = now.UnixMilli()++ key := prev == nil || n%10 == 0 || p.keyReq++ p.keyReq = false++ p.mu.Unlock()++ var coded []int++ if !key {++ coded = video.ChangedMBs(prev, f, 4)++ }++ sink(video.Sample{Data: enc.Encode(f, key, coded), Key: key, Duration: frame})++ prev = f++ select {++ case <-ctx.Done():++ return nil++ case <-tick.C:++ }++ }++}++++// mbSize rounds a picture to the PCM encoder's macroblock grid.++func mbSize(n int) int {++ if n < 16 {++ return 16++ }++ return n / 16 * 16++}++++// drawProbe paints the QR on white with a quiet zone, plus a moving marker so the encoder sees motion.++func drawProbe(f *video.Frame, ms int64) {++ for i := range f.Y {++ f.Y[i] = 235++ }++ for i := range f.Cb {++ f.Cb[i], f.Cr[i] = 128, 128++ }++ q, err := qrcode.New(fmt.Sprintf("%d", ms), qrcode.Medium)++ if err != nil {++ return++ }++ bmp := q.Bitmap()++ // Four modules of quiet zone, then whole-pixel modules. At 640×360 the code is about 200 px square.++ scale := min(f.W, f.H) / (len(bmp) + 8)++ if scale < 2 {++ scale = 2++ }++ size := len(bmp) * scale++ x0, y0 := 8, 8++ for y := 0; y < size && y0+y < f.H; y++ {++ for x := 0; x < size && x0+x < f.W; x++ {++ if bmp[y/scale][x/scale] {++ f.Y[(y0+y)*f.W+(x0+x)] = 16++ }++ }++ }++ // A marker sweeping along the bottom shows the stream is moving (and keeps VFR from dropping every frame).++ pos := int(ms/100) % (f.W - 32)++ for y := f.H - 24; y < f.H-8; y++ {++ for x := pos; x < pos+32; x++ {++ f.Y[y*f.W+x] = 16++ }++ }++}++++// Probe publishes the QR clock until seconds elapse (or ctx ends) and reports the frames written.++func Probe(ctx context.Context, o ProbeOptions) (int64, error) {++ if o.Seconds <= 0 {++ o.Seconds = ProbeDefaultSeconds++ }++ if o.Width <= 0 {++ o.Width = ProbeDefaultWidth++ }++ if o.Height <= 0 {++ o.Height = ProbeDefaultHeight++ }++ if o.FPS <= 0 {++ o.FPS = ProbeDefaultFPS++ }++ if o.Now == nil {++ o.Now = time.Now++ }++ if o.Log == nil {++ o.Log = slog.Default()++ }++ if err := checkProbeURL(o.WHIPURL); err != nil {++ return 0, err++ }++ if o.Ticket.IsZero() || !strings.HasPrefix(o.Ticket.Reveal(), "wpt_") {++ return 0, errors.New("stream: --probe needs the stream's publish ticket (--ticket wpt_…)")++ }++ opts := video.Options{WHIPURL: o.WHIPURL, PublishKey: o.Ticket, Audio: true,++ UserAgent: "openvibe-node/stream-probe", IncludeLoopback: o.Loopback, Log: o.Log}++ src := &ProbeSource{W: o.Width, H: o.Height, FPS: o.FPS, Now: o.Now}++ ctx, cancel := context.WithTimeout(ctx, time.Duration(o.Seconds)*time.Second)++ defer cancel()++ var pub publisher++ if o.NewPublisher != nil {++ pub = o.NewPublisher(opts)++ } else {++ pub = video.NewPublisher(opts)++ }++ done := make(chan struct{})++ go func() { pub.Run(ctx, src); close(done) }()++ start := time.Now()++ tick := time.NewTicker(5 * time.Second)++ defer tick.Stop()++ for {++ select {++ case <-ctx.Done():++ <-done++ return pub.Frames(), nil++ case <-tick.C:++ o.Log.Info("stream probe publishing", "elapsed", time.Since(start).Round(time.Second).String())++ }++ }++}++++func checkProbeURL(raw string) error {++ if raw == "" {++ return errors.New("stream: --probe needs a WHIP URL (--whip-url)")++ }++ u, err := url.Parse(raw)++ if err != nil || u.Host == "" || u.User != nil {++ return errors.New("stream: --whip-url is not a URL")++ }++ if u.Scheme != "https" && !(u.Scheme == "http" && isLoopback(u.Hostname())) {++ return errors.New("stream: --whip-url must be https")++ }++ return nil++}nnew file mode 100644@@@ -0,0 +1,97 @@++package stream++++import (++ "context"++ "testing"++ "time"++++ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"++ "github.com/OpenVibers/OpenVibe.Node/internal/video"++)++++func TestProbeSourceQRRoundTrips(t *testing.T) {++ want := int64(1_800_000_000_123)++ src := &ProbeSource{W: 640, H: 360, FPS: 10, Now: func() time.Time { return time.UnixMilli(want) }}++ got := make(chan video.Sample, 2)++ ctx, cancel := context.WithCancel(context.Background())++ defer cancel()++ go func() { _ = src.Run(ctx, func(s video.Sample) { got <- s }) }()++ select {++ case s := <-got:++ f := video.NewFrame(640, 360)++ // The PCM encoder wrote an IDR of the frame; decode the source's own drawing straight from a fresh frame.++ _ = s++ drawProbe(f, want)++ ms, err := DecodeProbe(f)++ if err != nil {++ t.Fatal(err)++ }++ if ms != want {++ t.Fatalf("decoded %d, want %d", ms, want)++ }++ case <-time.After(2 * time.Second):++ t.Fatal("no probe frame")++ }++}++++func TestProbeSourceClockAdvances(t *testing.T) {++ base := time.UnixMilli(1_800_000_000_000)++ i := 0++ src := &ProbeSource{W: 640, H: 360, FPS: 20, Now: func() time.Time {++ v := base.Add(time.Duration(i) * 50 * time.Millisecond)++ i++++ return v++ }}++ samples := make(chan video.Sample, 4)++ ctx, cancel := context.WithCancel(context.Background())++ defer cancel()++ go func() { _ = src.Run(ctx, func(s video.Sample) { samples <- s }) }()++ for n := 0; n < 3; n++ {++ select {++ case <-samples:++ case <-time.After(2 * time.Second):++ t.Fatal("no probe frame")++ }++ }++ if src.LastMS() < base.UnixMilli()+50 {++ t.Fatalf("the clock did not advance: %d", src.LastMS())++ }++}++++type countingPub struct{ frames int64 }++++func (c *countingPub) Run(ctx context.Context, s video.Source) {++ _ = s.Run(ctx, func(video.Sample) { c.frames++ })++}++func (c *countingPub) SetPaused(bool) {}++func (c *countingPub) SetMuted(bool) {}++func (c *countingPub) SetPublishKey(credentials.Secret) {}++func (c *countingPub) State() string { return video.StateLive }++func (c *countingPub) Frames() int64 { return c.frames }++func (c *countingPub) LastError() string { return "" }++++func TestProbePublishesUntilTimeIsUp(t *testing.T) {++ pub := &countingPub{}++ n, err := Probe(context.Background(), ProbeOptions{++ WHIPURL: "http://127.0.0.1:1/whip/agent/" + testID,++ Ticket: credentials.NewSecret(testTicket),++ Seconds: 1, Width: 320, Height: 180, FPS: 20,++ NewPublisher: func(video.Options) publisher { return pub },++ })++ if err != nil {++ t.Fatal(err)++ }++ if n < 5 {++ t.Fatalf("probe published %d frames in a second", n)++ }++}++++func TestProbeNeedsAURLAndATicket(t *testing.T) {++ if _, err := Probe(context.Background(), ProbeOptions{Seconds: 1, NewPublisher: func(video.Options) publisher { return &countingPub{} }}); err == nil {++ t.Error("no URL accepted")++ }++ if _, err := Probe(context.Background(), ProbeOptions{WHIPURL: "https://whip.openvibe.live/whip/agent/" + testID,++ Seconds: 1, NewPublisher: func(video.Options) publisher { return &countingPub{} }}); err == nil {++ t.Error("no ticket accepted")++ }++}@@@ -37,6 +37,11 @@ type Options struct {// IncludeLoopback offers loopback ICE candidates (tests on one machine).IncludeLoopback boolConnectTimeout time.Duration++ // Audio adds a silent Opus track (Agent Streams S1): S4's voice then needs no renegotiation. The packets are the++ // 20 ms silence frame from pion's own e2e test, sent whether or not video is flowing.++ Audio bool++ // UserAgent names the Node on the WHIP request.++ UserAgent string}// States reported by Publisher.State.@@@ -59,6 +64,9 @@ type Publisher struct {track *webrtc.TrackLocalStaticSampleneedK boolsrc Source++ pause atomic.Bool++ mute atomic.Bool++ key atomic.Value // credentials.Secret: the publish ticket, replaceable by renewal}func NewPublisher(opt Options) *Publisher {@@@ -74,17 +82,35 @@ func NewPublisher(opt Options) *Publisher {p := &Publisher{opt: opt, log: opt.Log.With("component", "video")}p.state.Store(StateOff)p.lastEr.Store("")++ p.key.Store(opt.PublishKey)return p}// State is off, connecting, live or retrying.func (p *Publisher) State() string { return p.state.Load().(string) }--// Frames counts samples written to a live session.++// Frames counts video samples written to a live session.func (p *Publisher) Frames() int64 { return p.frames.Load() }func (p *Publisher) LastError() string { return p.lastEr.Load().(string) }++// SetPaused stops sending video samples while keeping the WHIP session and its audio track up. Snapshot mode uses it:++// Live asked for no encoding, but the stream must not look dropped.++func (p *Publisher) SetPaused(paused bool) { p.pause.Store(paused) }++++// SetMuted stops the audio track's packets. The curtain calls it: a desk whose screen is blanked must not carry the++// desk's sound either (S1's track is silent, but S4's voice uses the same switch).++func (p *Publisher) SetMuted(muted bool) { p.mute.Store(muted) }++++// SetPublishKey replaces the ticket a renewed stream reconnects with. It is never logged.++func (p *Publisher) SetPublishKey(k credentials.Secret) {++ if !k.IsZero() {++ p.key.Store(k)++ }++}++++func (p *Publisher) publishKey() credentials.Secret { return p.key.Load().(credentials.Secret) }++// Run publishes src until ctx ends, reconnecting with backoff.func (p *Publisher) Run(ctx context.Context, src Source) {p.mu.Lock()@@@ -107,7 +133,7 @@ func (p *Publisher) Run(ctx context.Context, src Source) {}msg := "ended"if err != nil {-- msg = scrub(err.Error(), p.opt.PublishKey)++ msg = scrub(err.Error(), p.publishKey())}p.lastEr.Store(msg)p.state.Store(StateRetrying)@@@ -135,7 +161,7 @@ func (p *Publisher) write(s Sample) {needK = false}p.mu.Unlock()-- if track == nil || needK {++ if track == nil || needK || p.pause.Load() {return}if err := track.WriteSample(media.Sample{Data: s.Data, Duration: s.Duration}); err == nil {@@@ -143,6 +169,25 @@ func (p *Publisher) write(s Sample) {}}++// OpusSilence is one 20 ms silence frame (pion/webrtc e2e_test.go's silentOpusFrame: 8 kHz mono, 20 ms).++var OpusSilence = []byte{0xf8, 0xff, 0xfe}++++func (p *Publisher) sendSilence(ctx context.Context, track *webrtc.TrackLocalStaticSample) {++ tick := time.NewTicker(20 * time.Millisecond)++ defer tick.Stop()++ for {++ select {++ case <-ctx.Done():++ return++ case <-tick.C:++ if p.mute.Load() {++ continue++ }++ _ = track.WriteSample(media.Sample{Data: OpusSilence, Duration: 20 * time.Millisecond})++ }++ }++}++func (p *Publisher) requestKeyframe() {p.mu.Lock()src := p.src@@@ -162,6 +207,15 @@ func (p *Publisher) newAPI() (*webrtc.API, error) {}, webrtc.RTPCodecTypeVideo); err != nil {return nil, err}++ if p.opt.Audio {++ if err := m.RegisterCodec(webrtc.RTPCodecParameters{++ RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2,++ SDPFmtpLine: "minptime=10;useinbandfec=1"},++ PayloadType: 111,++ }, webrtc.RTPCodecTypeAudio); err != nil {++ return nil, err++ }++ }ir := &interceptor.Registry{}if err := webrtc.RegisterDefaultInterceptors(m, ir); err != nil {return nil, err@@@ -198,6 +252,18 @@ func (p *Publisher) session(ctx context.Context) error {if err != nil {return err}++ var audio *webrtc.TrackLocalStaticSample++ var silenceOnce sync.Once++ if p.opt.Audio {++ audio, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus,++ ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"}, "audio", "openvibe-node")++ if err != nil {++ return err++ }++ if _, err := pc.AddTransceiverFromTrack(audio, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}); err != nil {++ return err++ }++ }go func() { // RTCP: a viewer's PLI or FIR asks for a keyframe.for {pkts, _, err := tr.Sender().ReadRTCP()@@@ -267,8 +333,11 @@ func (p *Publisher) session(ctx context.Context) error {p.track, p.needK = track, truep.mu.Unlock()p.requestKeyframe()++ if audio != nil {++ silenceOnce.Do(func() { go p.sendSilence(ctx, audio) })++ }p.state.Store(StateLive)-- p.log.Info("video live", "whip", scrub(redactURL(p.opt.WHIPURL), p.opt.PublishKey))++ p.log.Info("video live", "whip", scrub(redactURL(p.opt.WHIPURL), p.publishKey()))case webrtc.PeerConnectionStateDisconnected:disconnectedAt = time.Now()case webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed:@@@ -288,17 +357,21 @@ func (p *Publisher) post(ctx context.Context, sdp string) (answer, resource strireturn "", "", err}req.Header.Set("Content-Type", "application/sdp")-- if !p.opt.PublishKey.IsZero() {-- req.Header.Set("Authorization", "Bearer "+p.opt.PublishKey.Reveal())++ key := p.publishKey()++ if !key.IsZero() {++ req.Header.Set("Authorization", "Bearer "+key.Reveal())++ }++ if p.opt.UserAgent != "" {++ req.Header.Set("User-Agent", p.opt.UserAgent)}resp, err := p.opt.HTTPClient.Do(req)if err != nil {-- return "", "", fmt.Errorf("whip: %s", scrub(err.Error(), p.opt.PublishKey))++ return "", "", fmt.Errorf("whip: %s", scrub(err.Error(), key))}defer resp.Body.Close()body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {-- return "", "", fmt.Errorf("whip: HTTP %d %s", resp.StatusCode, scrub(string(bytes.TrimSpace(body[:min(len(body), 200)])), p.opt.PublishKey))++ return "", "", fmt.Errorf("whip: HTTP %d %s", resp.StatusCode, scrub(string(bytes.TrimSpace(body[:min(len(body), 200)])), key))}loc := resp.Header.Get("Location")if loc != "" {@@@ -321,9 +394,10 @@ func (p *Publisher) delete(resource string) {if err != nil {return}-- // The publish key goes only to the WHIP endpoint's own origin, whatever Location the server returned.-- if !p.opt.PublishKey.IsZero() && sameOrigin(resource, p.opt.WHIPURL) {-- req.Header.Set("Authorization", "Bearer "+p.opt.PublishKey.Reveal())++ // The ticket goes only to the WHIP endpoint's own origin, whatever Location the server returned.++ key := p.publishKey()++ if !key.IsZero() && sameOrigin(resource, p.opt.WHIPURL) {++ req.Header.Set("Authorization", "Bearer "+key.Reveal())}if resp, err := p.opt.HTTPClient.Do(req); err == nil {resp.Body.Close()@@@ -23,7 +23,7 @@ func TestPublishTestPatternOverWHIP(t *testing.T) {ctx, cancel := context.WithCancel(context.Background())done := make(chan struct{})go func() { p.Run(ctx, &TestPattern{W: 160, H: 96, FPS: 20}); close(done) }()-- deadline := time.Now().Add(15 * time.Second)++ deadline := time.Now().Add(40 * time.Second)for rx.Packets.Load() < 20 {if time.Now().After(deadline) {t.Fatalf("no media: state %s frames %d packets %d err %q", p.State(), p.Frames(), rx.Packets.Load(), p.LastError())@@@ -43,6 +43,52 @@ func TestPublishTestPatternOverWHIP(t *testing.T) {}}++func TestPublishWithSilentAudio(t *testing.T) {++ rx := &WHIPReceiver{PublishKey: "pk_audio"}++ srv := httptest.NewServer(rx)++ defer srv.Close()++ defer rx.Close()++ p := NewPublisher(Options{WHIPURL: srv.URL + "/whip/agent/ags_x", PublishKey: credentials.NewSecret("pk_audio"),++ Audio: true, IncludeLoopback: true, Log: slog.New(slog.NewTextHandler(io.Discard, nil))})++ ctx, cancel := context.WithCancel(context.Background())++ done := make(chan struct{})++ go func() { p.Run(ctx, &TestPattern{W: 160, H: 96, FPS: 20}); close(done) }()++ deadline := time.Now().Add(40 * time.Second)++ for rx.AudioPackets.Load() < 5 || p.Frames() == 0 {++ if time.Now().After(deadline) {++ t.Fatalf("no audio+video: state %s frames %d audio %d err %q", p.State(), p.Frames(), rx.AudioPackets.Load(), p.LastError())++ }++ time.Sleep(20 * time.Millisecond)++ }++ video := p.Frames()++ p.SetPaused(true)++ time.Sleep(500 * time.Millisecond)++ if grew := p.Frames() - video; grew > 2 {++ t.Fatalf("paused video kept flowing: %d frames", grew)++ }++ if rx.AudioPackets.Load() < 5 {++ t.Fatal("audio stopped")++ }++ // The curtain mutes the track: no audio packets while muted, and they resume when it comes down.++ p.SetMuted(true)++ time.Sleep(300 * time.Millisecond)++ muted := rx.AudioPackets.Load()++ time.Sleep(500 * time.Millisecond)++ if grew := rx.AudioPackets.Load() - muted; grew > 2 {++ t.Fatalf("muted audio kept flowing: %d packets", grew)++ }++ p.SetMuted(false)++ deadline = time.Now().Add(3 * time.Second)++ for rx.AudioPackets.Load() <= muted+2 {++ if time.Now().After(deadline) {++ t.Fatal("audio did not resume after unmute")++ }++ time.Sleep(20 * time.Millisecond)++ }++ cancel()++ <-done++}++func TestPublishRetriesOnBadKey(t *testing.T) {rx := &WHIPReceiver{PublishKey: "right"}srv := httptest.NewServer(rx)@@@ -59,11 +105,18 @@ func TestPublishRetriesOnBadKey(t *testing.T) {}time.Sleep(10 * time.Millisecond)}++ // A renewed ticket replaces the key the next session reconnects with.++ rx.SetPublishKey("right")++ p.SetPublishKey(credentials.NewSecret("right"))++ deadline = time.Now().Add(10 * time.Second)++ for p.State() != StateLive {++ if time.Now().After(deadline) {++ t.Fatalf("renewed key never published: state %s err %q", p.State(), p.LastError())++ }++ time.Sleep(20 * time.Millisecond)++ }cancel()<-done-- if e := p.LastError(); e == "" || contains(e, "wrong-key-value") {-- t.Fatalf("last error %q", e)-- }}// The receiver's key can change while it serves (a fake Bot rotating the publish key): under -race this is the check@@@ -17,9 +17,10 @@ import (type WHIPReceiver struct {PublishKey string-- Packets atomic.Int64-- Deletes atomic.Int64-- BadAuth atomic.Int64++ Packets atomic.Int64++ AudioPackets atomic.Int64++ Deletes atomic.Int64++ BadAuth atomic.Int64mu sync.Mutexpcs map[string]*webrtc.PeerConnection@@@ -78,11 +79,15 @@ func (w *WHIPReceiver) post(rw http.ResponseWriter, r *http.Request) {return}pc.OnTrack(func(t *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {++ audio := t.Kind() == webrtc.RTPCodecTypeAudiofor {if _, _, err := t.ReadRTP(); err != nil {return}w.Packets.Add(1)++ if audio {++ w.AudioPackets.Add(1)++ }}})if err := pc.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: string(offer)}); err != nil {@@@ -69,6 +69,14 @@ Command kinds and their answers (the ack's ``data``):against ``max_desks``. ``describe`` lists the kinds this machine runs (``desk.kinds``). Starting and stopping act(refused at observe, approved at ask), ``status`` runs at every level, and the kill switch stops the desk. Whole desksneed Linux with Xvfb, browser desks a Chromium-family browser; config ``allow_desk: false`` turns both off.++* ``desk.stream`` ``{"action": "start"|"stop"|"status"|"curtain", "desk"?, "allow_sites"?, "up"?, "reason"?}``++ (Agent Streams S1, ``stream.py`` and ``curtain.py``): the agent's own stream on OpenVibe.Live. ``start`` answers++ ``{streaming, sid, capture, curtain, detectors}`` with the capture source the core's ffmpeg should use (x11grab,++ PipeWire, ddagrab, AVFoundation, or a browser desk's page through a FIFO); the core mints the stream and the ``wpt_``++ ticket on Live, so neither reaches this plugin. The privacy curtain's detectors (password field, secret-named++ window/file, an environment print, the hotkey, the browser's allow-list) run here and report ``curtain`` events;++ a detector that fails or is unsure puts the curtain up and never down. Config ``allow_desk_stream: false`` turns it++ off. The core publishes over WHIP and snapshots while nobody watches.* **A desk host for other people** (``desk_isolation: "sandbox"``, Node v0.32, Desks plan D5): every desk is a browserdesk whose browser runs under ``openvibe-node desk-sandbox`` (the jobs' sandbox: its own namespaces and cgroup as``worker.run_as``, ``desk_memory_mb``, ``desk_cpu_millis``, ``desk_disk_mb`` and ``desk_pids`` each, only the public@@@ -108,10 +116,16 @@ import urllib.parsefrom openvibe_plugin import Deferred, Fault, Pluginfrom .browser import KEY_CODES, Browser, BrowserError, find_binary++from .curtain import (Curtain, EnvPrintDetector, Hotkey, PasswordFieldDetector, SecretFileDetector, SiteDetector,++ _active_window, install_sigusr1)from .desk import MAX_DESKS_CAP, BrowserDesk, Desks, PrivateDesk, default_max_desksfrom .egress import EgressProxy, not_public++from .stream import PageScreencast, capture_kinds, desktop_capturefrom .files import Files, Handles++# The desk stream's curtain heartbeat: the core declares the curtain lost if it hears nothing for two seconds, so this++# must stay well under that (and the test patches it to run fast).++DESK_STREAM_HEARTBEAT_S = 1.0LEVELS = ("observe", "ask", "trusted", "full")RANK = {name: i for i, name in enumerate(LEVELS)}READ_OPS = frozenset({"list", "stat", "read"})@@@ -366,6 +380,14 @@ class Computer(Plugin):self.allow_screen_stream = Trueself.stream = None # the live screen: {"sid", "until", "fps", "tool", "dir", "stop": Event}self.stream_mu = threading.Lock()++ # The agent's desk stream (Agent Streams S1): capture, the curtain's detectors and the hotkey.++ self.allow_desk_stream = True++ self.desk_stream = None++ self.desk_stream_mu = threading.Lock()++ self.desk_stream_allow_sites = []++ self.desk_stream_hotkey = "ctrl+alt+p"++ self.desk_stream_detectors = None++ self._sigusr1_old = Noneself.browser_mu = threading.Lock()self.browser_op_mu = threading.Lock()self.browser_factory = browser_factory or self._new_browser@@@ -428,6 +450,18 @@ class Computer(Plugin):self.notify = config.get("notify", True) is not Falseself.allow_browser = config.get("allow_browser", True) is not Falseself.allow_screen_stream = config.get("allow_screen_stream", True) is not False++ self.allow_desk_stream = config.get("allow_desk_stream", True) is not False++ self.desk_stream_hotkey = str(config.get("desk_stream_hotkey") or "ctrl+alt+p")++ sites = config.get("desk_stream_allow_sites") or []++ if not isinstance(sites, list) or not all(isinstance(u, str) and u for u in sites) or len(sites) > 50:++ raise Fault("bad_value", "desk_stream_allow_sites is a list of up to 50 URLs")++ self.desk_stream_allow_sites = list(sites)++ dets = config.get("desk_stream_detectors")++ if dets is not None:++ known = ("password_field", "secret_file", "env_print", "manual", "site")++ if not isinstance(dets, list) or not all(d in known for d in dets):++ raise Fault("bad_value", "desk_stream_detectors is a list of %s" % ", ".join(known))++ self.desk_stream_detectors = list(dets)self.allow_desk = config.get("allow_desk", True) is not Falseself.desks.max_desks = _int(config.get("max_desks"), "max_desks", 1, MAX_DESKS_CAP, default_max_desks())self.desk_idle_s = _int(config.get("desk_idle_s"), "desk_idle_s", 60, 86400, 900)@@@ -492,6 +526,10 @@ class Computer(Plugin):caps["browser"] = {"level": self.level, "browser": os.path.basename(binary), "headless": self.browser_headless,"egress": self.browser_egress}kinds = self._desk_kinds()++ if self.allow_desk_stream and (kinds or tool):++ caps["desk.stream"] = {"capture": capture_kinds(self.platform), "browser": "browser" in kinds,++ "curtain": ["password_field", "secret_file", "env_print", "manual", "site", "detector"],++ "hotkey": self.desk_stream_hotkey, "allow_sites": True}if kinds:# A browser desk's screen and input are its page's own (the browser's pipe); a whole desk's need X tools.whole, page = "desktop" in kinds, "browser" in kinds@@@ -941,6 +979,8 @@ class Computer(Plugin):return self._start(cmd, lambda: self._screen(tool, width, env=desk_env))if cmd.kind == "screen_stream":return self._screen_stream(v, env=desk_env, desk=desk_name, page=page)++ if cmd.kind == "desk.stream":++ return self._desk_stream_op(cmd, v, level)if cmd.kind == "fs":op = v.get("op")if op in READ_OPS:@@@ -988,7 +1028,7 @@ class Computer(Plugin):returnif kind in SHARED_DESK_KINDS and v.get("desk", "main") != "main":return-- if kind == "screen_stream":++ if kind in ("screen_stream", "desk.stream"):raise Fault("not_allowed", "this desk host shows a desk as pictures (screen), not live: its one stream is its operator's")raise Fault("not_allowed", "this machine is a desk host for other people: only its browser desks take commands")@@@ -1646,7 +1686,10 @@ class Computer(Plugin):left = deadline - self._deadline_clock()() - 1.0if left < 1:raise Fault("expired", "no time was left to run it")-- return self._exec(argv, cwd, stdin, min(timeout, left), env=env)++ out = self._exec(argv, cwd, stdin, min(timeout, left), env=env)++ # The desk stream's environment-print detector sees the output before it leaves the machine.++ self._desk_stream_saw_output(argv, out)++ return outreturn run, self._exec_summary(argv, cwd.path if cwd is not None else None, stdin, desk)@@@ -1852,6 +1895,7 @@ class Computer(Plugin):offset = _int(v.get("offset"), "offset", 0, 2 ** 53, 0)length = _int(v.get("length"), "length", 0, self.max_read, self.max_read)data, size = target.read(offset, length)++ self._desk_stream_saw_path(target.path)return {"path": target.path, "size": size, "offset": offset, "length": len(data),"truncated": offset + len(data) < size, "content_b64": base64.b64encode(data).decode("ascii")}@@@ -1990,18 +2034,22 @@ class Computer(Plugin):try:with lock(): # one action at a time in each browser; two desks' browsers act in parallelif action == "open":-- return b.open(url)-- if action == "read":-- return b.read()-- if action == "screenshot":-- return b.screenshot(width)-- if action == "click":-- return b.click(selector=selector, text=None if selector else text)-- if action == "fill":-- return b.fill(selector, text)-- if action == "press":-- return b.press(key)-- return b.back()++ out = b.open(url)++ elif action == "read":++ out = b.read()++ elif action == "screenshot":++ out = b.screenshot(width)++ elif action == "click":++ out = b.click(selector=selector, text=None if selector else text)++ elif action == "fill":++ out = b.fill(selector, text)++ elif action == "press":++ out = b.press(key)++ else:++ out = b.back()++ if isinstance(out, dict):++ self._desk_stream_saw_url(out.get("url"))++ return outexcept BrowserError as e:raise Fault(e.code, e.message)@@@ -2273,6 +2321,206 @@ class Computer(Plugin):def _stream_capable(self, tool):return self._have_pillow() or self._native_jpeg(tool, "x.jpg") is not None++ # ---- the agent's desk stream (Agent Streams S1) ---------------------------------------------------------------++ def _desk_stream_op(self, cmd, v, level):++ """desk.stream: start/stop/status/curtain one agent stream. The core mints the wpt_ ticket and publishes; the++ plugin says how to capture and runs the curtain's detectors (fail closed)."""++ action = v.get("action")++ if action == "status":++ return self._desk_stream_status()++ if action == "stop":++ return {"streaming": False, "stopped": self._stop_desk_stream("stopped by its owner")}++ if action == "curtain":++ with self.desk_stream_mu:++ st = self.desk_stream++ if st is None:++ raise Fault("not_ready", "no desk stream is running")++ up = v.get("up") is True++ reason = v.get("reason") if v.get("reason") in ("manual", "site", "detector") else "manual"++ if up:++ st["curtain"].raise_(reason)++ else:++ st["curtain"].lower()++ return self._desk_stream_status()++ if action != "start":++ raise Fault("bad_value", "action is start, stop, status or curtain")++ if not self.allow_desk_stream:++ raise Fault("unsupported", "this machine does not stream its desks (allow_desk_stream is off)")++ if self._shared():++ raise Fault("not_allowed", "a desk host for other people shows a desk as pictures, never live")++ desk = v.get("desk") or "main"++ if desk != "main":++ self.desks.check(desk)++ if not self.desks.status(desk).get("running"):++ raise Fault("not_ready", "start the desk first (desk start, desk %s)" % desk)++ allow_sites = v.get("allow_sites")++ if not isinstance(allow_sites, list):++ allow_sites = self.desk_stream_allow_sites++ allow_sites = [u for u in allow_sites if isinstance(u, str) and u][:50]++ self._stop_desk_stream("replaced")++ browser_desk = desk != "main" and self.desks.kind(desk) == "browser"++ screencast = None++ if browser_desk:++ status = self.desks.status(desk)++ width, height = status.get("width") or 1280, status.get("height") or 800++ browser = self._desk_browser(desk)++ try:++ screencast = PageScreencast(browser, width=width, height=height).start()++ except ValueError as e:++ raise Fault("unsupported", str(e))++ capture = {"kind": "browser", "pipe": screencast.fifo, "width": width, "height": height}++ else:++ if desk == "main":++ if not self.allow_screen_stream:++ raise Fault("unsupported", "this machine does not show its screen (allow_screen_stream is off)")++ if not self._screen_tool():++ raise Fault("unsupported", "no screen capture tool on this machine")++ env = self.env++ width, height = self._display_size(env)++ else:++ status = self.desks.status(desk)++ env = self.desks.env(desk)++ width, height = status.get("width") or 1280, status.get("height") or 800++ try:++ capture = desktop_capture(self.platform, env, width, height,++ ffmpeg=self.which("ffmpeg") or "ffmpeg")++ except ValueError as e:++ raise Fault("unsupported", str(e))++ current_url = (lambda: browser.last_navigation) if browser_desk else None++ detectors = self._desk_stream_detectors(allow_sites, env if not browser_desk else None, current_url)++ curtain = Curtain(detectors=list(detectors.values()), on_change=self._desk_stream_curtain_changed)++ # Fail closed: the picture starts blank and only clears after every detector has been clear for two seconds.++ curtain.raise_("detector")++ curtain.start()++ hotkey = Hotkey(self._desk_stream_toggle, self.desk_stream_hotkey)++ hotkey.start()++ self._sigusr1_old = install_sigusr1(self._desk_stream_toggle)++ st = {"sid": os.urandom(8).hex(), "desk": desk, "capture": capture, "screencast": screencast,++ "curtain": curtain, "hotkey": hotkey, "detectors": detectors, "allow_sites": allow_sites,++ "since": self.clock()}++ stop = threading.Event()++ st["heartbeat"], st["heartbeat_thread"] = stop, threading.Thread(++ target=self._desk_stream_heartbeat, args=(st, stop), name="computer-curtain-beat", daemon=True)++ with self.desk_stream_mu:++ self.desk_stream = st++ st["heartbeat_thread"].start()++ return {"streaming": True, "sid": st["sid"], "capture": capture, "curtain": curtain.snapshot(),++ "detectors": sorted(detectors), "hotkey": self.desk_stream_hotkey}++++ def _desk_stream_heartbeat(self, st, stop):++ """Tell the core the curtain state every second. Its watchdog blanks the desk if this goes quiet for two++ seconds, so a hung plugin cannot leave a picture up; a clear state still has to last two seconds there."""++ while not stop.is_set():++ snap = st["curtain"].snapshot()++ try:++ self.emit_event("curtain", up=bool(snap["up"]), reason=snap["reason"] or "detector", heartbeat=True)++ except Exception:++ pass++ stop.wait(DESK_STREAM_HEARTBEAT_S)++++ def _desk_stream_status(self):++ with self.desk_stream_mu:++ st = self.desk_stream++ if st is None:++ return {"streaming": False}++ out = {"streaming": True, "sid": st["sid"], "desk": st["desk"], "capture": st["capture"],++ "curtain": st["curtain"].snapshot(), "detectors": sorted(st["detectors"]), "allow_sites": st["allow_sites"]}++ cast = st.get("screencast")++ if cast is not None:++ out["page"] = cast.status()++ return out++++ def _stop_desk_stream(self, reason):++ with self.desk_stream_mu:++ st, self.desk_stream = self.desk_stream, None++ if st is None:++ return False++ cast = st.get("screencast")++ if cast is not None:++ cast.stop()++ beat = st.get("heartbeat")++ if beat is not None:++ beat.set()++ thread = st.get("heartbeat_thread")++ if thread is not None and thread is not threading.current_thread() and thread.is_alive():++ thread.join(1.0)++ st["hotkey"].stop()++ st["curtain"].stop()++ if self._sigusr1_old is not None:++ try:++ signal.signal(signal.SIGUSR1, self._sigusr1_old)++ except (ValueError, OSError):++ pass++ self._sigusr1_old = None++ self.log("desk stream ended (%s)", reason)++ return True++++ def _desk_stream_detectors(self, allow_sites, env=None, current_url=None):++ wanted = self.desk_stream_detectors or ("password_field", "secret_file", "env_print", "site")++ out = {}++ if "password_field" in wanted:++ out["password_field"] = PasswordFieldDetector()++ if "secret_file" in wanted:++ out["secret_file"] = SecretFileDetector(window=(lambda: _active_window(env)) if env else None)++ if "env_print" in wanted:++ out["env_print"] = EnvPrintDetector()++ if "site" in wanted and allow_sites:++ out["site"] = SiteDetector(allow_sites, current_url=current_url)++ return out++++ def _desk_stream_curtain_changed(self, up, reason):++ self.emit_event("curtain", up=bool(up), reason=reason)++ self.log("curtain %s (%s)", "up" if up else "down", reason)++++ def _desk_stream_toggle(self, _reason="manual"):++ with self.desk_stream_mu:++ st = self.desk_stream++ if st is None:++ return++ if st["curtain"].snapshot()["up"]:++ st["curtain"].lower()++ else:++ st["curtain"].raise_("manual")++++ def _desk_stream_saw_path(self, path):++ with self.desk_stream_mu:++ st = self.desk_stream++ if st is not None and "secret_file" in st["detectors"]:++ st["detectors"]["secret_file"].saw_path(path)++++ def _desk_stream_saw_output(self, argv, out):++ with self.desk_stream_mu:++ st = self.desk_stream++ if st is None or "env_print" not in st["detectors"]:++ return++ text = ""++ if isinstance(out, dict):++ text = (out.get("stdout") or "") + "\n" + (out.get("stderr") or "")++ st["detectors"]["env_print"].saw_output(argv, text)++ if "secret_file" in st["detectors"]:++ for a in argv:++ st["detectors"]["secret_file"].saw_path(a)++++ def _desk_stream_saw_url(self, url):++ with self.desk_stream_mu:++ st = self.desk_stream++ if st is not None and "site" in st["detectors"]:++ st["detectors"]["site"].saw_url(url)++++ def _display_size(self, env):++ """The display's pixel size for x11grab, or 1920×1080 where there is no xdotool."""++ if not (env or {}).get("DISPLAY"):++ return 1920, 1080++ try:++ r = subprocess.run(["xdotool", "getdisplaygeometry"], capture_output=True, text=True, timeout=3,++ env=self._child_env(env) if env is not None else None)++ w, h = [int(x) for x in r.stdout.split()]++ if w >= 320 and h >= 180:++ return w, h++ except (OSError, ValueError, subprocess.SubprocessError):++ pass++ return 1920, 1080++def _screen_stream(self, v, env=None, desk=None, page=False):action = v.get("action")if action == "stop":@@@ -227,6 +227,7 @@ class Browser:self.behind = [] # (target, session) of the pages it was opened from, newest lastself.tab_events = [] # Target.targetCreated / targetDestroyed seen since the last _followself.tab_change = None # "opened" or "closed" since the last _mark: said in the action's receipt++ self.last_navigation = None # the URL of the last main-frame navigation (a desk stream's allow-list reads it)self.follow_mu = threading.Lock()self.after_start = None # called once the browser runs (a kept profile's session cookies go back in)@@@ -398,6 +399,10 @@ class Browser:waiting[1] = {"error": {"message": "Session closed: its page closed"}}waiting[0].set()self.events.append(msg["method"])++ if msg["method"] == "Page.frameNavigated":++ frame = (msg.get("params") or {}).get("frame") or {}++ if not frame.get("parentId") and frame.get("url"):++ self.last_navigation = frame["url"]if msg["method"] in ("Target.targetCreated", "Target.targetDestroyed"):self.tab_events = (self.tab_events + [(msg["method"], msg.get("params") or {})])[-50:]if len(self.events) > 500:nnew file mode 100644@@@ -0,0 +1,494 @@++"""The desk stream's privacy curtain (Agent Streams S1).++++The curtain is a state, not a button: while it is up, no captured frame reaches the publisher and the desk reports++``{"up": true, "reason": …}`` to OpenVibe.Live, which pauses every viewer's consumer as well. A detector answers with a++reason when it sees something private, with ``None`` when it is clear, and may raise when it cannot tell; **a detector++that raises, or that is unsure, puts the curtain up and never down** (the design's fail-closed rule). The curtain comes++down only when every detector has been clear for two seconds, or when the owner lowers it.++++Detectors:++ * ``password_field`` — the focused accessible element is a password field (AT-SPI on Linux, UI Automation on Windows,++ the AX API on macOS; the probe is injected so each platform, and the tests, supply their own);++ * ``secret_file`` — the focused window's title or class looks secret, or a secret-named file was just read;++ * ``env_print`` — an ``exec`` printed environment values or ran ``env``/``printenv``;++ * ``manual`` — the person pressed the hotkey or chose Curtain now;++ * ``site`` — a browser desk navigated outside its allow-list;++ * ``detector`` — a detector failed or could not decide.++++The hotkey is Ctrl+Alt+P by default on X11 (XGrabKey through ctypes, no extra dependency). ``SIGUSR1`` toggles the++curtain too, which is what a service can be given without a display. Neither the reason nor a detector's error text is++allowed to carry anything a viewer sees: viewers only ever see "Private screen".++"""++from __future__ import annotations++++import ctypes++import os++import re++import signal++import subprocess++import sys++import threading++import time++from urllib.parse import urlparse++++CURTAIN_CLEAR_HOLD_S = 2.0++CURTAIN_POLL_S = 0.25++# How long a one-off signal (a secret file read, an environment print) keeps the curtain up once it is seen.++CURTAIN_EVENT_HOLD_S = 10.0++++# A path or window title that looks secret. `.env`, `*credentials*`, a private key, PEMs and password managers.++SECRET_NAME = re.compile(++ r"(^|[/\\])(\.env(\.[A-Za-z0-9_-]+)?|[^/\\]*credentials[^/\\]*|id_rsa|id_dsa|id_ecdsa|id_ed25519|"++ r"[^/\\]*\.(pem|key|p12|pfx|kdbx|ovpn)|[^/\\]*secret[^/\\]*)$",++ re.IGNORECASE)++SECRET_WINDOW = re.compile(++ r"(keepass|1password|bitwarden|lastpass|dashlane|nordpass|password|credential|secret|\benv\b)", re.IGNORECASE)++++REASONS = ("password_field", "secret_file", "env_print", "manual", "site", "detector")++++# A line from `env`/`printenv`, or an exported assignment. Three or more in one output is an environment print.++ENV_LINE = re.compile(r"^[A-Z_][A-Z0-9_]{2,}=.*$")++++++def looks_secret_name(name):++ return bool(name) and bool(SECRET_NAME.search(name))++++++def looks_secret_window(title, klass=""):++ return bool(title and SECRET_WINDOW.search(title)) or bool(klass and SECRET_WINDOW.search(klass))++++++class Detector:++ """A curtain detector. poll() returns None when clear, a reason when it sees something private, and may raise."""++ reason = "detector"++++ def poll(self):++ return None++++++class PasswordFieldDetector(Detector):++ reason = "password_field"++++ def __init__(self, probe=None):++ self.probe = probe or _password_field_focused++++ def poll(self):++ v = self.probe()++ if v is None:++ # Unsure is not clear.++ return self.reason++ return self.reason if v else None++++++class SecretFileDetector(Detector):++ reason = "secret_file"++++ def __init__(self, window=None, clock=time.monotonic):++ self.window = window or _active_window++ self.clock = clock++ self.until = 0.0++++ def saw_path(self, name):++ if looks_secret_name(name):++ self.until = max(self.until, self.clock() + CURTAIN_EVENT_HOLD_S)++++ def poll(self):++ title, klass = self.window()++ if looks_secret_window(title, klass):++ return self.reason++ if self.clock() < self.until:++ return self.reason++ return None++++++class EnvPrintDetector(Detector):++ reason = "env_print"++++ def __init__(self, clock=time.monotonic):++ self.clock = clock++ self.until = 0.0++++ def saw_output(self, argv, output):++ """The exec wrapper calls this with what the command printed, before the output leaves the machine."""++ if not output:++ return++ cmd = os.path.basename(argv[0]) if argv else ""++ if cmd in ("env", "printenv") or any(a in ("-0", "--null") for a in argv):++ self.until = max(self.until, self.clock() + CURTAIN_EVENT_HOLD_S)++ return++ hits = 0++ for line in output.splitlines():++ if ENV_LINE.match(line):++ hits += 1++ if hits >= 3:++ self.until = max(self.until, self.clock() + CURTAIN_EVENT_HOLD_S)++ return++++ def poll(self):++ return self.reason if self.clock() < self.until else None++++++class SiteDetector(Detector):++ reason = "site"++++ def __init__(self, allow_sites=None, current_url=None):++ self.allow = [u for u in (allow_sites or []) if u]++ self.current_url = current_url++ self.last = None++++ def allows(self, url):++ if not self.allow:++ return True++ host = urlparse(url).hostname or ""++ for entry in self.allow:++ want = urlparse(entry).hostname if "://" in entry else entry++ if not want:++ continue++ if host == want or host.endswith("." + want):++ return True++ return False++++ def saw_url(self, url):++ self.last = url++ if url and not self.allows(url):++ return self.reason++ return None++++ def poll(self):++ if self.current_url is not None:++ try:++ url = self.current_url()++ except Exception:++ return self.reason++ else:++ url = self.last++ return self.saw_url(url)++++++class ManualDetector(Detector):++ reason = "manual"++++ def __init__(self):++ self.raised = False++ self.lock = threading.Lock()++++ def raise_(self):++ with self.lock:++ self.raised = True++++ def lower(self):++ with self.lock:++ self.raised = False++++ def poll(self):++ with self.lock:++ return self.reason if self.raised else None++++++class Curtain:++ """The fail-closed curtain. evaluate() is pure enough to test with an injected clock and detectors."""++++ def __init__(self, detectors=None, clock=time.monotonic, clear_hold=CURTAIN_CLEAR_HOLD_S, on_change=None):++ self.detectors = list(detectors or [])++ self.clock = clock++ self.clear_hold = clear_hold++ self.on_change = on_change++ self.lock = threading.RLock()++ self.up = False++ self.reason = None++ self.since = None++ self.clear_since = None++ self.error = None++ self._stop = threading.Event()++ self._thread = None++ self.manual = ManualDetector()++ self.detectors.append(self.manual)++++ def add(self, detector):++ with self.lock:++ self.detectors.append(detector)++ return detector++++ def evaluate(self):++ """One pass: any raised or unsure detector raises the curtain; all clear for two seconds lowers it."""++ with self.lock:++ reason = None++ for det in self.detectors:++ try:++ got = det.poll()++ except Exception as e: # a detector that fails is never clear++ got = "detector"++ self.error = "%s: %s" % (getattr(det, "reason", "detector"), e)++ if got:++ reason = got if got in REASONS else "detector"++ break++ now = self.clock()++ if reason:++ self.clear_since = None++ return self._set(True, reason, now)++ if not self.up:++ self.clear_since = None++ return False++ if self.clear_since is None:++ self.clear_since = now++ if now - self.clear_since >= self.clear_hold:++ return self._set(False, None, now)++ return True++++ def _set(self, up, reason, now):++ if up == self.up:++ if up and reason and reason != self.reason:++ self.reason = reason++ if self.on_change:++ try:++ self.on_change(up, reason)++ except Exception:++ pass++ return self.up++ self.up = up++ self.reason = reason++ self.since = now if up else None++ self.clear_since = None++ if self.on_change:++ try:++ self.on_change(up, reason or "detector")++ except Exception:++ pass++ return self.up++++ def raise_(self, reason="manual"):++ if reason == "manual":++ self.manual.raise_()++ with self.lock:++ self._set(True, reason if reason in REASONS else "manual", self.clock())++++ def lower(self):++ """The owner's own button: lower now (the detectors may raise it again on the next pass)."""++ self.manual.lower()++ with self.lock:++ self.clear_since = self.clock()++ return self._set(False, None, self.clock())++++ def snapshot(self):++ with self.lock:++ return {"up": self.up, "reason": self.reason if self.up else None,++ "since": None if self.since is None else int(self.since)}++++ def start(self):++ if self._thread is not None:++ return++ self._stop.clear()++ self._thread = threading.Thread(target=self._loop, name="computer-curtain", daemon=True)++ self._thread.start()++++ def _loop(self):++ while not self._stop.is_set():++ try:++ self.evaluate()++ except Exception:++ self.raise_("detector")++ self._stop.wait(CURTAIN_POLL_S)++++ def stop(self):++ self._stop.set()++ thread, self._thread = self._thread, None++ if thread is not None and thread is not threading.current_thread() and thread.is_alive():++ thread.join(1.0)++++++# ---- platform probes -----------------------------------------------------------------------------------------------++++def _active_window(env=None):++ """(title, WM_CLASS) of the focused X11 window, or ("", "") when there is none (no display, no xdotool)."""++ run_env = None++ if env is not None:++ run_env = dict(os.environ)++ run_env.update({k: v for k, v in env.items() if isinstance(v, str)})++ if not (run_env or os.environ).get("DISPLAY"):++ return "", ""++ try:++ win = subprocess.run(["xdotool", "getactivewindow"], capture_output=True, text=True, timeout=2, env=run_env)++ except (OSError, subprocess.SubprocessError):++ return "", ""++ wid = win.stdout.strip()++ if not wid:++ return "", ""++ try:++ title = subprocess.run(["xdotool", "getwindowname", wid], capture_output=True, text=True, timeout=2,++ env=run_env).stdout.strip()++ except (OSError, subprocess.SubprocessError):++ title = ""++ try:++ klass = subprocess.run(["xprop", "-id", wid, "WM_CLASS"], capture_output=True, text=True, timeout=2,++ env=run_env).stdout.strip()++ except (OSError, subprocess.SubprocessError):++ klass = ""++ return title, klass++++++def _password_field_focused():++ """True when the focused AT-SPI element is a password field, False when it is not, None when it cannot tell.++++ On Windows and macOS the probe is the platform's accessibility API; this module's default is the Linux one. A++ missing accessibility stack is None (unsure), which raises the curtain: the stream is never shown blind.++ """++ if sys.platform == "win32":++ return _password_field_windows()++ if sys.platform == "darwin":++ return _password_field_macos()++ try:++ import pyatspi++ except Exception:++ return None++ try:++ desktop = pyatspi.Registry.getDesktop(0)++ focused = _focused_accessible(desktop, pyatspi, 0)++ if focused is None:++ return False++ return focused.getRole() == pyatspi.ROLE_PASSWORD_TEXT++ except Exception:++ return None++++++def _focused_accessible(node, pyatspi, depth):++ """The accessible with STATE_FOCUSED, walking at most a dozen levels (accessibility trees can be huge)."""++ if depth > 12:++ return None++ try:++ state = node.getState()++ if state is not None and state.contains(pyatspi.STATE_FOCUSED):++ return node++ except Exception:++ pass++ try:++ children = node++ except Exception:++ return None++ try:++ for child in children:++ got = _focused_accessible(child, pyatspi, depth + 1)++ if got is not None:++ return got++ except Exception:++ return None++ return None++++++def _password_field_windows():++ # UI Automation through PowerShell: the focused element's ControlType is Edit and IsPassword is true.++ try:++ script = ("Add-Type -AssemblyName UIAutomationClient;"++ "$e=[System.Windows.Automation.AutomationElement]::FocusedElement;"++ "if($e -and $e.Current.IsPassword){'yes'}else{'no'}")++ r = subprocess.run(["powershell", "-NoProfile", "-Command", script], capture_output=True, text=True, timeout=3)++ except (OSError, subprocess.SubprocessError):++ return None++ if "yes" in r.stdout:++ return True++ if "no" in r.stdout:++ return False++ return None++++++def _password_field_macos():++ # The AX API is only reachable from a signed/trusted helper; without it the answer is unsure.++ return None++++++# ---- the hotkey ---------------------------------------------------------------------------------------------------++++class Hotkey:++ """A global curtain hotkey. X11: Ctrl+Alt+P through XGrabKey (ctypes, no dependency). Elsewhere: unavailable, and++ the owner uses Curtain now on Live or `openvibe-node stream` (SIGUSR1) instead."""++++ def __init__(self, callback, combo="ctrl+alt+p"):++ self.callback = callback++ self.combo = combo++ self._stop = threading.Event()++ self._thread = None++ self.error = None++++ def available(self):++ return sys.platform.startswith("linux") and bool(os.environ.get("DISPLAY"))++++ def start(self):++ if not self.available() or self._thread is not None:++ return False++ self._thread = threading.Thread(target=self._run, name="computer-curtain-hotkey", daemon=True)++ self._thread.start()++ return True++++ def stop(self):++ self._stop.set()++ thread, self._thread = self._thread, None++ if thread is not None and thread.is_alive():++ thread.join(1.0)++++ def _run(self):++ try:++ x11 = ctypes.CDLL("libX11.so.6")++ except OSError as e:++ self.error = str(e)++ return++ # Declare the few calls used; keycodes and masks are constants in X11/X.h.++ x11.XOpenDisplay.restype = ctypes.c_void_p++ x11.XOpenDisplay.argtypes = [ctypes.c_char_p]++ x11.XDefaultRootWindow.restype = ctypes.c_ulong++ x11.XDefaultRootWindow.argtypes = [ctypes.c_void_p]++ x11.XStringToKeysym.restype = ctypes.c_ulong++ x11.XStringToKeysym.argtypes = [ctypes.c_char_p]++ x11.XKeysymToKeycode.restype = ctypes.c_ubyte++ x11.XKeysymToKeycode.argtypes = [ctypes.c_void_p, ctypes.c_ulong]++ x11.XGrabKey.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_uint, ctypes.c_ulong, ctypes.c_int,++ ctypes.c_int, ctypes.c_int]++ x11.XPending.argtypes = [ctypes.c_void_p]++ x11.XNextEvent.argtypes = [ctypes.c_void_p, ctypes.c_void_p]++ x11.XCloseDisplay.argtypes = [ctypes.c_void_p]++ x11.XCloseDisplay.restype = ctypes.c_int++ dpy = x11.XOpenDisplay(None)++ if not dpy:++ self.error = "no X display"++ return++ try:++ key = self.combo.split("+")[-1]++ mods = 0++ parts = [p.strip().lower() for p in self.combo.split("+")[:-1]]++ if "ctrl" in parts or "control" in parts:++ mods |= 4 # ControlMask++ if "alt" in parts or "mod1" in parts:++ mods |= 8 # Mod1Mask++ if "shift" in parts:++ mods |= 1 # ShiftMask++ keysym = x11.XStringToKeysym(key.encode("ascii"))++ keycode = x11.XKeysymToKeycode(dpy, keysym)++ root = x11.XDefaultRootWindow(dpy)++ # Grab the four CapsLock/NumLock variants, else the hotkey stops working when either is on.++ for extra in (0, 2, 16, 18): # LockMask, Mod2Mask++ x11.XGrabKey(dpy, keycode, mods | extra, root, 1, 1, 1)++ event = ctypes.create_string_buffer(192) # XEvent is 24 longs; this is plenty for KeyPress.++ while not self._stop.is_set():++ while x11.XPending(dpy):++ x11.XNextEvent(dpy, event)++ try:++ self.callback("manual")++ except Exception:++ pass++ self._stop.wait(0.05)++ finally:++ x11.XCloseDisplay(dpy)++++++def install_sigusr1(toggle):++ """A service's local curtain hotkey: SIGUSR1 toggles it (kill -USR1 <pid>). Returns the previous handler."""++ try:++ old = signal.getsignal(signal.SIGUSR1)++++ def handler(signum, frame):++ try:++ toggle()++ except Exception:++ pass++ signal.signal(signal.SIGUSR1, handler)++ return old++ except (ValueError, AttributeError, OSError):++ return Nonennew file mode 100644@@@ -0,0 +1,178 @@++"""How the desk stream captures a picture (Agent Streams S1).++++For a whole desk (an Xvfb display) and the machine's own screen the core runs ffmpeg itself, with the display and size++this module reports: x11grab or PipeWire on Linux, ddagrab on Windows, AVFoundation on macOS. A browser desk has no++display to grab, so its own page is captured here over the DevTools pipe at about five frames a second (the floor of++the design's 5–30 VFR) and written as JPEGs into a small FIFO that the core's ffmpeg reads with `-f image2pipe`.++++The FIFO is 0600 in a 0700 folder and is removed with the stream. Nothing here calls the owner's screen: a browser desk++uses its own headless browser, and a named desk's display is its Xvfb one.++"""++from __future__ import annotations++++import errno++import os++import subprocess++import sys++import tempfile++import threading++import time++++PAGE_STREAM_FPS = 5++PAGE_STREAM_WIDTH = 1280++PAGE_STREAM_QUALITY = 70++FIFO_NAME = "frames.fifo"++++++def ffmpeg_has_pipewire(ffmpeg):++ """Whether this ffmpeg has the pipewire input device (7.1 and later)."""++ try:++ out = subprocess.run([ffmpeg or "ffmpeg", "-hide_banner", "-devices"],++ capture_output=True, text=True, timeout=5).stdout++ except (OSError, subprocess.SubprocessError):++ return False++ return "pipewire" in out++++++def capture_kinds(platform=None):++ platform = platform or sys.platform++ if platform == "win32":++ return ["ddagrab"]++ if platform == "darwin":++ return ["avfoundation"]++ return ["x11grab", "pipewire"]++++++def desktop_capture(platform, env, width, height, ffmpeg="", pipewire_node=None):++ """The capture spec for a display (a desk's Xvfb or the owner's screen), or a Fault-worthy message."""++ width, height = _size(width, height)++ platform = platform or sys.platform++ if platform == "win32":++ return {"kind": "ddagrab", "width": width, "height": height}++ if platform == "darwin":++ return {"kind": "avfoundation", "device": os.environ.get("OPENVIBE_AVFOUNDATION_SCREEN", "1:none"),++ "width": width, "height": height}++ display = (env or {}).get("DISPLAY")++ if display:++ # x11grab takes `:N.0`; the desk already carries its own display number.++ if not display.endswith(".0"):++ display += ".0"++ return {"kind": "x11grab", "display": display, "width": width, "height": height}++ if (env or {}).get("WAYLAND_DISPLAY"):++ node = pipewire_node or os.environ.get("OPENVIBE_PIPEWIRE_NODE")++ if not node:++ raise ValueError("a Wayland desk needs OPENVIBE_PIPEWIRE_NODE set to the screen-cast node id")++ if not ffmpeg_has_pipewire(ffmpeg):++ raise ValueError("this ffmpeg has no pipewire input; install ffmpeg 7.1+ or set a display")++ return {"kind": "pipewire", "pipewire": str(node), "width": width, "height": height}++ raise ValueError("no display to capture on this machine")++++++def _size(width, height):++ try:++ width, height = int(width), int(height)++ except (TypeError, ValueError):++ width, height = 1280, 800++ width = max(320, min(3840, width)) & ~15++ height = max(180, min(2160, height)) & ~15++ return width, height++++++class PageScreencast:++ """Write a browser desk's page into a FIFO as JPEGs for the core's ffmpeg (image2pipe)."""++++ def __init__(self, browser, width=PAGE_STREAM_WIDTH, height=800, fps=PAGE_STREAM_FPS, clock=time.monotonic):++ if not hasattr(os, "mkfifo"):++ raise ValueError("a browser desk's stream needs a named pipe (Linux or macOS), which this system has not")++ self.browser = browser++ self.width = width++ self.height = height++ self.fps = max(1, min(10, fps))++ self.clock = clock++ self.stop_event = threading.Event()++ self.thread = None++ self.dir = tempfile.mkdtemp(prefix="openvibe-page-stream-")++ self.fifo = os.path.join(self.dir, FIFO_NAME)++ os.mkfifo(self.fifo, 0o600)++ self.error = None++ self.frames = 0++++ def start(self):++ self.thread = threading.Thread(target=self._loop, name="computer-page-stream", daemon=True)++ self.thread.start()++ return self++++ def _loop(self):++ fd = None++ try:++ # OPENVIBE's ffmpeg opens the read end; wait for it without blocking the plugin's stop.++ while not self.stop_event.is_set():++ try:++ fd = os.open(self.fifo, os.O_WRONLY | os.O_NONBLOCK)++ break++ except OSError as e:++ if e.errno not in (errno.ENXIO, errno.ENOENT):++ self.error = str(e)++ return++ self.stop_event.wait(0.1)++ if fd is None:++ return++ os.set_blocking(fd, False)++ frame = os.path.join(self.dir, "frame-%d.jpg" % os.getpid())++ period = 1.0 / self.fps++ failures = 0++ while not self.stop_event.is_set():++ started = self.clock()++ try:++ self.browser.frame(frame, self.width, PAGE_STREAM_QUALITY)++ with open(frame, "rb") as f:++ data = f.read()++ view = memoryview(data)++ while view:++ try:++ n = os.write(fd, view)++ except BlockingIOError:++ # ffmpeg is behind: drop this frame rather than grow a queue.++ n = len(view)++ view = view[n:]++ self.frames += 1++ failures = 0++ except OSError as e:++ if e.errno == errno.EPIPE:++ return++ self.error = str(e)++ return++ except Exception as e: # a page that is navigating, a browser that restarted: try again briefly++ failures += 1++ if failures >= 10:++ self.error = str(e)++ return++ self.stop_event.wait(max(0.0, period - (self.clock() - started)))++ finally:++ if fd is not None:++ try:++ os.close(fd)++ except OSError:++ pass++ self._cleanup()++++ def _cleanup(self):++ try:++ os.unlink(self.fifo)++ except OSError:++ pass++ try:++ os.rmdir(self.dir)++ except OSError:++ pass++++ def status(self):++ return {"fifo": self.fifo, "width": self.width, "height": self.height, "fps": self.fps,++ "frames": self.frames, "error": self.error}++++ def stop(self):++ self.stop_event.set()++ thread, self.thread = self.thread, None++ if thread is not None and thread is not threading.current_thread() and thread.is_alive():++ thread.join(2.0)++ self._cleanup()nnew file mode 100644@@@ -0,0 +1,292 @@++"""The agent desk stream (Agent Streams S1): the curtain's triggers and its fail-closed rule, the capture each platform++chooses, a browser desk's screencast into its FIFO, and the plugin's desk.stream start/stop/status."""++import io++import os++import shutil++import sys++import tempfile++import threading++import time++import unittest++from types import SimpleNamespace++++HERE = os.path.dirname(os.path.abspath(__file__))++sys.path.insert(0, HERE)++sys.path.insert(0, os.path.dirname(HERE))++sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(HERE)), "sdk"))++++import openvibe_computer # noqa: E402++from openvibe_computer import Computer # noqa: E402++from openvibe_computer.curtain import (Curtain, EnvPrintDetector, PasswordFieldDetector, SecretFileDetector,++ SiteDetector, looks_secret_name, looks_secret_window) # noqa: E402++from openvibe_computer.stream import PageScreencast, desktop_capture # noqa: E402++from openvibe_plugin import Fault # noqa: E402++++++class Clock:++ def __init__(self):++ self.t = 0.0++++ def __call__(self):++ return self.t++++++class Answer:++ def __init__(self, value):++ self.value = value++++ def poll(self):++ return self.value++++++class Boom:++ reason = "password_field"++++ def poll(self):++ raise RuntimeError("the accessibility bus went away")++++++class FakeDesk:++ kind = "desktop"++++ def __init__(self, display=":90"):++ self.display = display++++ def available(self):++ return True++++ def window_manager(self):++ return "metacity"++++ def running(self):++ return True++++ def status(self):++ return {"running": True, "display": self.display, "width": 800, "height": 600, "kind": "desktop"}++++ def env(self):++ return {"DISPLAY": self.display, "DBUS_SESSION_BUS_ADDRESS": "unix:path=/tmp/ov-test"}++++++def curtain(detectors=(), clock=None):++ clock = clock or Clock()++ return Curtain(list(detectors), clock=clock, clear_hold=2.0), clock++++++class CurtainTest(unittest.TestCase):++ def test_site_detector_follows_a_navigation_it_did_not_see(self):++ nav = {"url": "https://example.com/a"}++ c, clock = curtain([SiteDetector(["https://example.com"], current_url=lambda: nav["url"])])++ self.assertFalse(c.evaluate())++ nav["url"] = "https://bank.example/login"++ self.assertTrue(c.evaluate())++ nav["url"] = "https://docs.example.com/"++ clock.t = 10.0++ self.assertTrue(c.evaluate())++ clock.t = 12.1++ self.assertFalse(c.evaluate())++++ def test_each_trigger_raises(self):++ cases = [++ ("password", PasswordFieldDetector(probe=lambda: True), None),++ ("secret window", SecretFileDetector(window=lambda: ("~/.env", "")), None),++ ("secret class", SecretFileDetector(window=lambda: ("Documents", "WM_CLASS(STRING) = keepassxc")), None),++ ("env", EnvPrintDetector(), ("env", "HOME=/root\nPATH=/bin\n")),++ ("site", SiteDetector(["https://example.com"]), "https://evil.example/x"),++ ]++ for name, det, action in cases:++ with self.subTest(name=name):++ c, clock = curtain([det])++ if isinstance(det, EnvPrintDetector):++ det.saw_output(list(action), "HOME=/root\nPATH=/bin\n")++ elif isinstance(det, SiteDetector):++ det.saw_url(action)++ self.assertTrue(c.evaluate(), name)++ self.assertTrue(c.snapshot()["up"], name)++ c, _ = curtain([])++ c.raise_("manual")++ self.assertTrue(c.evaluate())++ self.assertEqual(c.snapshot()["reason"], "manual")++++ def test_secret_names(self):++ for name in (".env", "/home/a/.env.local", "/home/a/credentials.json", "id_rsa", "/k/server.pem", "/k/x.key"):++ self.assertTrue(looks_secret_name(name), name)++ for name in ("notes.txt", "/home/a/environment.md", "keyboard.png"):++ self.assertFalse(looks_secret_name(name), name)++ self.assertTrue(looks_secret_window("Passwords — Bitwarden"))++ self.assertFalse(looks_secret_window("OpenVibe — Dashboard"))++++ def test_detector_exception_is_fail_closed(self):++ c, _ = curtain([Boom()])++ self.assertTrue(c.evaluate())++ self.assertTrue(c.snapshot()["up"])++ self.assertEqual(c.snapshot()["reason"], "detector")++++ def test_unsure_is_not_clear(self):++ c, _ = curtain([PasswordFieldDetector(probe=lambda: None)])++ self.assertTrue(c.evaluate())++ self.assertEqual(c.snapshot()["reason"], "password_field")++++ def test_clear_for_two_seconds_before_down(self):++ state = {"pw": True}++ c, clock = curtain([PasswordFieldDetector(probe=lambda: state["pw"])])++ self.assertTrue(c.evaluate())++ state["pw"] = False++ clock.t = 10.0++ self.assertTrue(c.evaluate(), "the hold starts now")++ clock.t = 11.9++ self.assertTrue(c.evaluate(), "still inside the two seconds")++ clock.t = 12.1++ self.assertFalse(c.evaluate(), "two clear seconds lower it")++ self.assertFalse(c.snapshot()["up"])++++ def test_owner_lowers_now(self):++ c, _ = curtain([])++ c.raise_("manual")++ self.assertTrue(c.snapshot()["up"])++ c.lower()++ self.assertFalse(c.snapshot()["up"])++ # But the next pass still has to see every detector clear; a raised detector raises it again.++ c.add(Answer("site"))++ self.assertTrue(c.evaluate())++++ def test_change_callback_reports_the_reason_never_a_detector_error(self):++ seen = []++ c = Curtain([Boom()], clock=Clock(), on_change=lambda up, why: seen.append((up, why)))++ c.evaluate()++ self.assertEqual(seen, [(True, "detector")])++ self.assertNotIn("accessibility", seen[0][1])++++++class CaptureTest(unittest.TestCase):++ def test_each_platform(self):++ self.assertEqual(desktop_capture("linux", {"DISPLAY": ":99"}, 1600, 900),++ {"kind": "x11grab", "display": ":99.0", "width": 1600, "height": 896})++ self.assertEqual(desktop_capture("win32", {}, 1600, 900)["kind"], "ddagrab")++ self.assertEqual(desktop_capture("darwin", {}, 1600, 900)["kind"], "avfoundation")++++ def test_no_display_is_refused(self):++ with self.assertRaises(ValueError):++ desktop_capture("linux", {}, 1280, 800)++++ def test_browser_screencast_writes_jpeg_frames(self):++ if not hasattr(os, "mkfifo"):++ self.skipTest("a named pipe is Linux/macOS only")++ try:++ from PIL import Image++ except ImportError:++ self.skipTest("pillow not installed")++ tmp = tempfile.mkdtemp(prefix="ov-page-")++ self.addCleanup(lambda: shutil.rmtree(tmp, ignore_errors=True))++ jpeg = os.path.join(tmp, "one.jpg")++ Image.new("RGB", (64, 48), (10, 20, 30)).save(jpeg, "JPEG")++++ class FakeBrowser:++ def frame(self, path, width, quality):++ shutil.copy(jpeg, path)++++ cast = PageScreencast(FakeBrowser(), width=64, height=48, fps=10)++ got = []++++ def read_one():++ with open(cast.fifo, "rb") as f:++ got.append(f.read(4))++++ reader = threading.Thread(target=read_one)++ reader.start()++ cast.start()++ deadline = time.time() + 5++ while not got and time.time() < deadline:++ time.sleep(0.02)++ cast.stop()++ reader.join(2)++ self.assertTrue(got and got[0][:2] == b"\xff\xd8", "a JPEG frame reached the FIFO")++++ def test_capture_kinds(self):++ from openvibe_computer.stream import capture_kinds++ self.assertEqual(capture_kinds("win32"), ["ddagrab"])++ self.assertEqual(capture_kinds("darwin"), ["avfoundation"])++ self.assertIn("x11grab", capture_kinds("linux"))++++++class PluginTest(unittest.TestCase):++ def make(self, **config):++ self.dir = tempfile.mkdtemp(prefix="ov-stream-")++ self.addCleanup(lambda: shutil.rmtree(self.dir, ignore_errors=True))++ p = Computer(which=lambda n: "/usr/bin/" + n if n in ("Xvfb", "xdotool", "import", "xprop") else None,++ env={"DISPLAY": ":0"}, platform="linux")++ p.setup({"level": "trusted", "roots": [self.dir], **config})++ p.desks.desks["private"] = FakeDesk()++ # The detectors need a real accessibility stack and display; the plugin's start path is what this tests.++ p._desk_stream_detectors = lambda allow_sites, env=None, current_url=None: {}++ return p++++ def cmd(self, value):++ return SimpleNamespace(kind="desk.stream", value=value, id="c1", deadline=time.monotonic() + 30)++++ def test_describe_offers_desk_stream(self):++ p = self.make()++ caps = p.describe({})++ self.assertIn("desk.stream", caps)++ self.assertIn("x11grab", caps["desk.stream"]["capture"])++ off = Computer(which=lambda n: "/usr/bin/" + n if n in ("Xvfb",) else None, env={"DISPLAY": ":0"}, platform="linux")++ off.setup({"level": "trusted", "roots": [self.dir], "allow_desk_stream": False})++ self.assertNotIn("desk.stream", off.describe({"allow_desk_stream": False, "roots": [self.dir]}))++++ def test_start_status_stop(self):++ p = self.make()++ started = p.handle(self.cmd({"action": "start", "desk": "private", "title": "t"}))++ self.assertTrue(started["streaming"])++ self.assertEqual(started["capture"]["kind"], "x11grab")++ self.assertEqual(started["capture"]["display"], ":90.0")++ self.assertTrue(started["curtain"]["up"], "the curtain starts up (fail closed)")++ self.assertTrue(p.handle(self.cmd({"action": "status"}))["streaming"])++ p.handle(self.cmd({"action": "curtain", "up": False}))++ self.assertFalse(p.handle(self.cmd({"action": "status"}))["curtain"]["up"])++ self.assertTrue(p.handle(self.cmd({"action": "stop"}))["stopped"])++ self.assertFalse(p.handle(self.cmd({"action": "status"}))["streaming"])++++ def test_start_is_refused_when_off(self):++ p = self.make(allow_desk_stream=False)++ with self.assertRaises(Fault) as e:++ p.handle(self.cmd({"action": "start", "desk": "private"}))++ self.assertEqual(e.exception.code, "unsupported")++++ def test_curtain_requires_a_stream(self):++ p = self.make()++ with self.assertRaises(Fault) as e:++ p.handle(self.cmd({"action": "curtain", "up": True}))++ self.assertEqual(e.exception.code, "not_ready")++++ def test_a_running_stream_beats_its_curtain_state(self):++ old = openvibe_computer.DESK_STREAM_HEARTBEAT_S++ openvibe_computer.DESK_STREAM_HEARTBEAT_S = 0.02++ self.addCleanup(lambda: setattr(openvibe_computer, "DESK_STREAM_HEARTBEAT_S", old))++ p = self.make()++ events = []++ p.emit_event = lambda name, **fields: events.append((name, fields))++ p.handle(self.cmd({"action": "start", "desk": "private"}))++ deadline = time.time() + 2++ while sum(1 for n, f in events if n == "curtain" and f.get("heartbeat")) < 2 and time.time() < deadline:++ time.sleep(0.01)++ beats = [f for n, f in events if n == "curtain" and f.get("heartbeat")]++ self.assertGreaterEqual(len(beats), 2, events)++ self.assertTrue(beats[0]["up"], "the first beat reports the fail-closed curtain up")++ p.handle(self.cmd({"action": "stop"}))++ n = len(beats)++ time.sleep(0.1)++ self.assertEqual(len([f for x, f in events if x == "curtain" and f.get("heartbeat")]), n,++ "the heartbeat stops with the stream")++++ def test_secret_read_raises_the_curtain(self):++ p = self.make()++ p.handle(self.cmd({"action": "start", "desk": "private"}))++ p._desk_stream_detectors = lambda allow_sites, env=None, current_url=None: {}++ p._desk_stream_saw_path("/home/a/.env")++ # No secret detector in this injected set: nothing changes, which is why the real set always includes one.++ self.assertTrue(p.handle(self.cmd({"action": "status"}))["streaming"])++ p.handle(self.cmd({"action": "stop"}))++++++if __name__ == "__main__":++ unittest.main()nnew file mode 100755@@@ -0,0 +1,168 @@++#!/usr/bin/env python3++"""Headless viewer for the Agent Streams S1 latency probe.++++It joins a WebRTC stream over WHEP (the design's ``openvibe-node stream --probe`` publishes a QR of the millisecond++clock it drew each frame), decodes that QR from every video frame, and reports the glass-to-glass latency:++``now_ms - clock_in_the_QR``. The pass line is p95 under 1 s over 60 s.++++ scripts/stream-viewer.sh --whep https://openvibe.live/whep/agent/ags_… --seconds 60++++The view ticket is a secret: it comes from OPENVIBE_VIEW_TICKET (or ``--ticket-stdin``), never from argv. A WHEP++endpoint is what this needs; Live's S1 player speaks mediasoup over /ws/broadcast, so until Live exposes WHEP the++number is measured through a relay that does (the design allows a later WHEP endpoint for third-party players).++++Requires aiortc and OpenCV (scripts/stream-viewer.sh puts them in a cache venv). ``--self-test`` checks the statistics++without a network.++"""++import argparse++import asyncio++import json++import os++import sys++import time++++PASS_P95_MS = 1000.0++++++class LatencyReport:++ def __init__(self):++ self.samples = []++++ def add(self, sent_ms, now_ms=None):++ now_ms = time.time() * 1000.0 if now_ms is None else now_ms++ delta = now_ms - sent_ms++ if 0 <= delta <= 60_000:++ self.samples.append(delta)++ return delta++ return None++++ def summary(self):++ if not self.samples:++ return {"frames": 0, "p50_ms": None, "p95_ms": None, "max_ms": None}++ s = sorted(self.samples)++ def pct(p):++ return s[min(len(s) - 1, int(round(p / 100.0 * (len(s) - 1))))]++ return {"frames": len(s), "p50_ms": round(pct(50), 1), "p95_ms": round(pct(95), 1),++ "max_ms": round(s[-1], 1)}++++++def self_test():++ r = LatencyReport()++ for ms in range(1, 101):++ r.add(0, ms)++ got = r.summary()++ assert got["frames"] == 100 and got["p95_ms"] == 95.0, got++ assert r.add(10_000, 1_000) is None, "a QR from the future is not a sample"++ try:++ import cv2 # noqa: F401++ import numpy # noqa: F401++ except ImportError as e:++ print("self-test: statistics ok, QR decoder missing: %s" % e)++ return 1++ print("self-test: statistics and QR decoder ok")++ return 0++++++async def run_whep(args):++ try:++ import cv2++ import numpy as np++ from aiortc import RTCPeerConnection, RTCSessionDescription++ except ImportError as e:++ print("missing a viewer dependency (%s): run scripts/stream-viewer.sh" % e, file=sys.stderr)++ return 2++ import aiohttp++++ ticket = args.ticket++ headers = {"Content-Type": "application/sdp", "Accept": "application/sdp"}++ if ticket:++ headers["Authorization"] = "Bearer " + ticket++++ pc = RTCPeerConnection()++ report = LatencyReport()++ decoder = cv2.QRCodeDetector()++ seen = {"frames": 0, "qr": 0}++++ @pc.on("track")++ def on_track(track):++ if track.kind != "video":++ return++++ async def consume():++ while True:++ try:++ frame = await track.recv()++ except Exception:++ return++ seen["frames"] += 1++ try:++ img = frame.to_ndarray(format="bgr24")++ except Exception:++ continue++ ok, decoded, *_ = decoder.detectAndDecode(img)++ if not ok or not decoded:++ continue++ try:++ sent = int(decoded.strip())++ except ValueError:++ continue++ seen["qr"] += 1++ report.add(sent)++ asyncio.ensure_future(consume())++++ pc.addTransceiver("video", direction="recvonly")++ offer = await pc.createOffer()++ await pc.setLocalDescription(offer)++ # Wait for the ICE gathering to finish (a few seconds), then send the complete offer.++ deadline = time.monotonic() + 10++ while pc.iceGatheringState != "complete" and time.monotonic() < deadline:++ await asyncio.sleep(0.1)++ async with aiohttp.ClientSession() as session:++ async with session.post(args.whep, data=pc.localDescription.sdp, headers=headers) as resp:++ body = await resp.text()++ if resp.status not in (200, 201):++ print("WHEP answered HTTP %d: %s" % (resp.status, body[:200]), file=sys.stderr)++ await pc.close()++ return 3++ await pc.setRemoteDescription(RTCSessionDescription(body, "answer"))++ print("viewer: joined %s; measuring for %ds" % (args.whep, args.seconds), file=sys.stderr)++ try:++ await asyncio.sleep(args.seconds)++ finally:++ await pc.close()++ out = report.summary()++ out.update({"seconds": args.seconds, "video_frames": seen["frames"], "qr_frames": seen["qr"],++ "p95_under_1s": out["p95_ms"] is not None and out["p95_ms"] < args.max_p95_ms})++ if args.json:++ print(json.dumps(out))++ else:++ if out["frames"] == 0:++ print("no QR frame decoded in %ds: is --probe publishing to this stream?" % args.seconds, file=sys.stderr)++ else:++ print("glass-to-glass latency over %ds: p50 %.1f ms, p95 %.1f ms, max %.1f ms (%d frames)" %++ (args.seconds, out["p50_ms"], out["p95_ms"], out["max_ms"], out["frames"]))++ print("S1 pass line: p95 under %.0f ms → %s" % (args.max_p95_ms, "PASS" if out["p95_under_1s"] else "FAIL"))++ return 0 if out["p95_under_1s"] else 1++++++def main():++ ap = argparse.ArgumentParser()++ ap.add_argument("--whep", help="the WHEP endpoint of the stream")++ ap.add_argument("--seconds", type=int, default=60)++ ap.add_argument("--json", action="store_true")++ ap.add_argument("--self-test", action="store_true")++ ap.add_argument("--max-p95-ms", type=float, default=PASS_P95_MS)++ ap.add_argument("--ticket-stdin", action="store_true", help="read the view ticket from one line of stdin")++ args = ap.parse_args()++ if args.self_test:++ return self_test()++ if not args.whep:++ ap.error("--whep is required (or --self-test)")++ args.ticket = os.environ.get("OPENVIBE_VIEW_TICKET", "")++ if not args.ticket and args.ticket_stdin:++ args.ticket = sys.stdin.readline().strip()++ return asyncio.run(run_whep(args))++++++if __name__ == "__main__":++ sys.exit(main())nnew file mode 100755@@@ -0,0 +1,26 @@++#!/bin/sh++# The latency viewer's dependencies in a cache venv, then the viewer. See scripts/stream-viewer.py.++set -eu++++here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)++cache=${XDG_CACHE_HOME:-$HOME/.cache}/openvibe-stream-viewer++venv=$cache/venv++++if [ ! -x "$venv/bin/python" ]; then++ command -v python3 >/dev/null || { echo "stream-viewer: python3 is required" >&2; exit 2; }++ mkdir -p "$cache"++ python3 -m venv "$venv" || {++ echo "stream-viewer: could not create a venv under $cache; install python3-venv" >&2++ exit 2++ }++fi++++if ! "$venv/bin/python" -c 'import aiortc, cv2, numpy' >/dev/null 2>&1; then++ echo "stream-viewer: installing aiortc and OpenCV once into $venv" >&2++ "$venv/bin/python" -m pip install --quiet --disable-pip-version-check aiortc opencv-python-headless numpy || {++ echo "stream-viewer: pip install failed (no network?)" >&2++ exit 2++ }++fi++++exec "$venv/bin/python" "$here/stream-viewer.py" "$@"
@@ -84,6 +84,9 @@ Usage:
@@ -84,6 +84,9 @@ Usage:
openvibe-node memory scan [--tools a,b] [--project DIR …] [--json]
openvibe-node memory scan [--tools a,b] [--project DIR …] [--json]
read the memory files of the coding tools on this machine
read the memory files of the coding tools on this machine
(read-only, local, secrets removed)
(read-only, local, secrets removed)
+ openvibe-node stream --probe --whip-url URL [--seconds N] [--size WxH] [--json]
+ publish a QR of the millisecond clock for the headless latency viewer (the ticket comes from
+ OPENVIBE_STREAM_TICKET, never argv)
openvibe-node share --key ovc_… --port N put this machine's port N on the web (OpenVibe Connect),
openvibe-node share --key ovc_… --port N put this machine's port N on the web (OpenVibe Connect),
behind OpenVibe sign-in, until Ctrl-C
behind OpenVibe sign-in, until Ctrl-C
openvibe-node share add|list|remove … the shares the Node service keeps up (openvibe-node share)
openvibe-node share add|list|remove … the shares the Node service keeps up (openvibe-node share)
@@ -148,6 +151,9 @@ func runWithStdin(args []string, stdin io.Reader, stdout, stderr io.Writer) int
@@ -148,6 +151,9 @@ func runWithStdin(args []string, stdin io.Reader, stdout, stderr io.Writer) int
if cmd == "desk-sandbox" { // its own flags, up to "--": a browser's flags after it are never the Node's
if cmd == "desk-sandbox" { // its own flags, up to "--": a browser's flags after it are never the Node's
return cmdDeskSandbox(rest, stderr)
return cmdDeskSandbox(rest, stderr)
}
}
+ if cmd == "stream" { // its own flags: --probe and the latency probe's
+ return cmdStream(rest, stdin, stdout, stderr)
+ }
fs := flag.NewFlagSet(cmd, flag.ContinueOnError)
fs := flag.NewFlagSet(cmd, flag.ContinueOnError)
fs.SetOutput(stderr)
fs.SetOutput(stderr)
g := &globals{}
g := &globals{}
new file mode 100644
new file mode 100644
@@ -0,0 +1,97 @@
@@ -0,0 +1,97 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+ "github.com/OpenVibers/OpenVibe.Node/internal/stream"
+)
+
+// cmdStream is `openvibe-node stream --probe`: it publishes a QR of the millisecond clock over WHIP, so a headless
+// viewer can report the glass-to-glass latency. The ticket comes from the environment or stdin, never from argv.
+func cmdStream(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ fs := flag.NewFlagSet("stream", flag.ContinueOnError)
+ fs.SetOutput(stderr)
+ probe := fs.Bool("probe", false, "publish the QR millisecond clock (the latency probe)")
+ whip := fs.String("whip-url", "", "the stream's WHIP URL (https://whip.openvibe.live/whip/agent/ags_…)")
+ seconds := fs.Int("seconds", stream.ProbeDefaultSeconds, "how long to publish (seconds)")
+ size := fs.String("size", "640x360", "picture size, WxH")
+ fps := fs.Int("fps", stream.ProbeDefaultFPS, "frames per second")
+ loopback := fs.Bool("loopback", false, "offer loopback ICE candidates (a WHIP endpoint on this machine)")
+ ticketStdin := fs.Bool("ticket-stdin", false, "read the publish ticket from one line on stdin")
+ asJSON := fs.Bool("json", false, "print JSON")
+ if err := fs.Parse(args); err != nil {
+ return 2
+ }
+ if fs.NArg() > 0 {
+ fmt.Fprintln(stderr, "error: stream takes no arguments")
+ return 2
+ }
+ if !*probe {
+ fmt.Fprintln(stderr, "usage: openvibe-node stream --probe --whip-url URL [--seconds N] [--size WxH] [--json]")
+ fmt.Fprintln(stderr, "the publish ticket is read from OPENVIBE_STREAM_TICKET, or with --ticket-stdin from stdin (never argv)")
+ return 2
+ }
+ width, height, err := parseSize(*size)
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return 2
+ }
+ ticket, err := probeTicket(stdin, *ticketStdin)
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return 2
+ }
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+ n, err := stream.Probe(ctx, stream.ProbeOptions{WHIPURL: *whip, Ticket: ticket, Seconds: *seconds,
+ Width: width, Height: height, FPS: *fps, Loopback: *loopback})
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return 1
+ }
+ if *asJSON {
+ b, _ := json.Marshal(map[string]any{"frames": n, "seconds": *seconds, "size": fmt.Sprintf("%dx%d", width, height),
+ "whip_url": *whip})
+ fmt.Fprintln(stdout, string(b))
+ return 0
+ }
+ fmt.Fprintf(stdout, "probe: published %d frames in %ds at %dx%d\n", n, *seconds, width, height)
+ fmt.Fprintln(stdout, "the latency is measured by scripts/stream-viewer.py against the same stream; it reports p50/p95/max")
+ return 0
+}
+
+// probeTicket reads the publish ticket. It is a secret: the environment or stdin, never a flag that `ps` can show.
+func probeTicket(stdin io.Reader, fromStdin bool) (credentials.Secret, error) {
+ t := strings.TrimSpace(os.Getenv("OPENVIBE_STREAM_TICKET"))
+ if t == "" && fromStdin && stdin != nil {
+ // Read at most one line, so a pipe cannot hang the probe.
+ buf := make([]byte, 256)
+ n, _ := stdin.Read(buf)
+ t = strings.TrimSpace(string(buf[:n]))
+ }
+ if t == "" {
+ return credentials.Secret{}, errors.New("set OPENVIBE_STREAM_TICKET to the stream's wpt_… ticket (or pass --ticket-stdin)")
+ }
+ if !strings.HasPrefix(t, "wpt_") || len(t) < 8 {
+ return credentials.Secret{}, errors.New("the publish ticket must be a wpt_… value (OPENVIBE_STREAM_TICKET)")
+ }
+ return credentials.NewSecret(t), nil
+}
+
+func parseSize(s string) (int, int, error) {
+ var w, h int
+ if _, err := fmt.Sscanf(s, "%dx%d", &w, &h); err != nil || w < 160 || h < 90 || w > 3840 || h > 2160 {
+ return 0, 0, fmt.Errorf("size %q is WxH between 160x90 and 3840x2160", s)
+ }
+ return w &^ 15, h &^ 15, nil
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,53 @@
@@ -0,0 +1,53 @@
+package main
+
+import (
+ "bytes"
+ "strings"
+ "testing"
+)
+
+func TestParseSize(t *testing.T) {
+ if w, h, err := parseSize("640x360"); err != nil || w != 640 || h != 352 {
+ t.Fatalf("parseSize = %d,%d,%v", w, h, err)
+ }
+ if _, _, err := parseSize("1x1"); err == nil {
+ t.Fatal("a tiny size was accepted")
+ }
+ if _, _, err := parseSize("nonsense"); err == nil {
+ t.Fatal("nonsense was accepted")
+ }
+}
+
+func TestTheProbeTicketComesFromTheEnvironmentOrStdinNeverArgv(t *testing.T) {
+ t.Setenv("OPENVIBE_STREAM_TICKET", "wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG")
+ tk, err := probeTicket(nil, false)
+ if err != nil || tk.Reveal() == "" {
+ t.Fatalf("env ticket: %v", err)
+ }
+ t.Setenv("OPENVIBE_STREAM_TICKET", "")
+ tk, err = probeTicket(strings.NewReader("wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG\n"), true)
+ if err != nil || tk.Reveal() == "" {
+ t.Fatalf("stdin ticket: %v", err)
+ }
+ if _, err := probeTicket(nil, false); err == nil {
+ t.Fatal("no ticket was accepted")
+ }
+ if _, err := probeTicket(strings.NewReader("not-a-ticket\n"), true); err == nil {
+ t.Fatal("a bad ticket was accepted")
+ }
+}
+
+func TestCmdStreamNeedsAURLAndATicket(t *testing.T) {
+ t.Setenv("OPENVIBE_STREAM_TICKET", "")
+ var out, errb bytes.Buffer
+ if code := cmdStream([]string{"--probe", "--seconds", "1"}, nil, &out, &errb); code != 2 {
+ t.Fatalf("missing URL exit = %d", code)
+ }
+ if strings.Contains(out.String()+errb.String(), "wpt_0123456789") {
+ t.Fatal("a ticket was printed")
+ }
+ if code := cmdStream([]string{"--probe", "--whip-url", "https://whip.openvibe.live/whip/agent/ags_x", "--seconds", "1"},
+ nil, &out, &errb); code != 2 {
+ t.Fatalf("missing ticket exit = %d", code)
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,129 @@
@@ -0,0 +1,129 @@
+# The desk stream (Agent Streams S1, Node's part)
+
+An agent's desk can stream on OpenVibe.Live. The Node captures the desk, encodes it with the screen recipe, publishes
+it over WHIP with a publish ticket, sends a snapshot while nobody watches, blanks the picture behind the privacy
+curtain, and measures the glass-to-glass latency with a QR probe. The design is
+`plan-state-2026-10-22/AGENT-STREAMS-S1-DESIGN-2026-11-06.md`; this document is the Node side of it.
+
+## What runs where
+
+| Piece | Where |
+|---|---|
+| The stream, its ticket, the state poll, snapshots, renewal | the core (`internal/stream`, `internal/node/stream.go`) |
+| Capture selection, the browser desk's page capture, the curtain detectors, the hotkey | the computer plugin (`desk.stream`, `plugins/computer/openvibe_computer/stream.py` and `curtain.py`) |
+| Encoding | ffmpeg (`libx264`), started by the core |
+| Publishing | WHIP with pion (`internal/video`), video + a silent Opus track |
+
+The core mints the stream and its `wpt_…` ticket on Live with this machine's Network node token (audience
+`openvibe.live`, capability `live.agent-stream.publish`). **The plugin never sees the WHIP URL or the ticket**, and
+neither does a log line, an error, `openvibe-node status` or the command's ack. The ticket is a
+`credentials.Secret`.
+
+## The command
+
+`desk.stream` is a computer kind routed to the computer plugin, with the core doing the orchestration:
+
+```json
+{"action": "start", "title": "fix the failing tests",
+ "task": {"source": "codes", "ref": "job_123", "title": "fix the tests", "url": "https://openvibe.codes/…"},
+ "visibility": "private", "allow_sites": ["https://github.com"], "desk": "desk-a"}
+{"action": "stop"}
+{"action": "curtain", "up": true, "reason": "manual"}
+{"action": "status"}
+```
+
+`start` answers `{streaming, id, page_url, visibility, expires_at, capture}`. `desk` may be `main` (the machine's own
+screen), `private`, or `desk-<name>`; a browser desk is captured from its own page. `visibility` defaults to
+`private`. `stream.live_url` in the Node config moves the API off production.
+
+## The encoder recipe
+
+`internal/stream/encode.go` builds the ffmpeg command line from the design:
+
+* H.264 **main**, `-preset ultrafast`, `-tune zerolatency,stillimage`, `-bf 0`;
+* a keyframe every 2 s of wall clock (`-force_key_frames expr:gte(t,n_forced*2)`, `-g 60` at 30 fps);
+* variable rate: `mpdecimate` drops unchanged frames and `-fps_mode vfr` keeps the capture's timestamps (about 5 fps on
+ a still screen, up to 30 when it moves);
+* 1080p30 (or the desk's size), with a 720p rung: after two WHIP retries the core restarts the capture at 720p
+ (`EncodeOptions.LowBandwidth`);
+* CBR capped at 4 Mb/s (`-b:v 4000k -maxrate 4000k -bufsize 4000k -x264-params nal-hrd=cbr`).
+
+Capture is `x11grab` or (ffmpeg 7.1+) `pipewire` on Linux, `ddagrab` on Windows, `avfoundation` on macOS, and
+`-f image2pipe` for a browser desk's page (the plugin writes JPEGs into a 0600 FIFO at about 5 fps, the floor of the
+VFR range). A browser desk has no display to grab.
+
+A silent Opus track is always negotiated (a 20 ms silence frame every 20 ms), so S4's voice needs no renegotiation.
+
+## Live's state and snapshots
+
+`GET /api/streams/agent/:id/state` is polled every 10 s with the ticket. `encode: false` stops the capture and pauses
+video on the WHIP session (the audio keeps it alive); a snapshot is `PUT` every 5 s, JPEG or WebP, at most 200 KB,
+taken through the plugin's own `screen` command. `encode: true` restarts the capture. The ticket is renewed at **3 h**
+(sooner if it would expire first), and a `DELETE /api/streams/agent/:id/ticket` is the kill switch.
+
+## The curtain
+
+The plugin's detectors answer `None` (clear), a reason (raise), or raise (fail closed). Every one of these raises it:
+
+* `password_field` — the focused AT-SPI element is a password field (UI Automation on Windows, the AX API on macOS;
+ a missing accessibility stack is *unsure*, which raises it);
+* `secret_file` — the focused window's title or class looks secret, or a secret-named file was read through `fs` or
+ named in an `exec` (`/.env`, `*credentials*`, `id_rsa`, `*.pem`, `*.key`, a password manager);
+* `env_print` — an `exec` ran `env`/`printenv` or printed three or more environment assignments;
+* `manual` — the hotkey (Ctrl+Alt+P on X11, `SIGUSR1` for a service), Curtain now, or the `curtain` command;
+* `site` — a browser desk navigated outside its `allow_sites` (checked on the agent's own actions and on every
+ main-frame navigation the browser reports);
+* `detector` — any detector failed or returned something it could not decide.
+
+The curtain starts **up** and only comes down after every detector has been clear for 2 s, or when the owner lowers
+it. While it is up the core feeds the publisher a pre-encoded "Private screen" frame instead of the capture, mutes the
+audio track, and reports `POST /api/streams/agent/:id/curtain {up, reason}`; Live pauses every viewer's consumer as
+well. Viewers never see the reason. `desk_stream_detectors` in the plugin config can disable a detector a deployment
+genuinely cannot run (a deliberate choice, not a silent failure).
+
+**A lost plugin is a curtain, not a gap.** The plugin sends its curtain state every second. If the computer plugin
+exits, crashes, is not ready (the supervisor's state), or goes silent for two seconds while a stream is up, the core
+raises the curtain itself at once (reason `detector`), blanks the video and mutes the audio; a "curtain down" command
+is refused while the plugin is lost. It comes down only after the plugin is back **and** has reported clear for two
+seconds, so a restarted plugin's first "clear" cannot uncover the desk.
+
+## The latency probe and the viewer
+
+```
+# Live hands the desk a WHIP URL and a wpt_ ticket (create + state routes).
+OPENVIBE_STREAM_TICKET=wpt_… openvibe-node stream --probe \
+ --whip-url https://whip.openvibe.live/whip/agent/ags_… --seconds 60
+
+# The same ticket's view side, on a WHEP endpoint:
+OPENVIBE_VIEW_TICKET=wvt_… scripts/stream-viewer.sh --whep https://openvibe.live/whep/agent/ags_… --seconds 60
+```
+
+The probe draws a QR of `time.Now().UnixMilli()` in the corner of every frame with the built-in PCM encoder (no ffmpeg,
+no display) and publishes it with the same WHIP path and silent audio. The viewer joins over WHEP, decodes the QR from
+each frame with OpenCV, and reports p50/p95/max; the S1 pass line is **p95 under 1000 ms over 60 s**. `--self-test`
+checks the statistics and the decoder without a network.
+
+**The production number is not measured yet.** Live's S1 routes are being built in
+`live-agent-streams-s1`; the viewer needs a WHEP endpoint (the design allows one for third-party players), and Live's
+S1 player speaks mediasoup over `/ws/broadcast`. Once that endpoint exists, run the two commands above and put the p95
+in this file and in the PR.
+
+## Tests
+
+Go (`go test ./internal/stream ./internal/video`): the recipe's arguments, each capture kind, the publisher's silent
+audio and pause, the state poll turning encoding off into snapshots, renewal at 3 h, the curtain blanking every
+captured frame, the QR probe round-trip, and that a ticket never appears in JSON or an error.
+
+Python (`plugins/computer/tests/test_stream.py`): every curtain trigger, a detector exception and an unsure detector
+raising it, the two-second clear hold, the owner lowering it, the capture each platform chooses, a browser desk's
+JPEG frames reaching the FIFO, and the plugin's start/status/stop/refuse path.
+
+## Platform notes and follow-ups
+
+* Linux desktops use `x11grab`; PipeWire needs ffmpeg 7.1+ and `OPENVIBE_PIPEWIRE_NODE` set to the screen-cast node.
+ The target ffmpeg here is 6.1, so `x11grab` is the default.
+* A browser desk streams its page at 5 fps (an ffmpeg-friendly floor); a real DevTools screencast at up to 30 fps is the
+ next step.
+* The AT-SPI detector sees the accessibility stack of the Node's session; on a private Xvfb desk the desk's own
+ `at-spi2-registryd` must be running for it to see the desk's apps.
+* Simulcast (S2) and the owner's page (Live's part) are the next pieces.
@@ -7,10 +7,12 @@ require (
@@ -7,10 +7,12 @@ require (
github.com/godbus/dbus/v5 v5.1.0
github.com/godbus/dbus/v5 v5.1.0
github.com/gorilla/websocket v1.5.3
github.com/gorilla/websocket v1.5.3
github.com/kardianos/service v1.3.0
github.com/kardianos/service v1.3.0
+ github.com/makiuchi-d/gozxing v0.1.1
github.com/pion/ice/v4 v4.4.4
github.com/pion/ice/v4 v4.4.4
github.com/pion/interceptor v0.1.49
github.com/pion/interceptor v0.1.49
github.com/pion/rtcp v1.2.18
github.com/pion/rtcp v1.2.18
github.com/pion/webrtc/v4 v4.2.22
github.com/pion/webrtc/v4 v4.2.22
+ github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
golang.org/x/sys v0.41.0
golang.org/x/sys v0.41.0
)
)
@@ -31,5 +33,7 @@ require (
@@ -31,5 +33,7 @@ require (
github.com/wlynxg/anet v0.0.5 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/net v0.50.0 // indirect
golang.org/x/net v0.50.0 // indirect
+ golang.org/x/text v0.34.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/time v0.14.0 // indirect
+ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
)
)
@@ -8,6 +8,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
@@ -8,6 +8,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI=
github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI=
github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
+github.com/makiuchi-d/gozxing v0.1.1 h1:xxqijhoedi+/lZlhINteGbywIrewVdVv2wl9r5O9S1I=
+github.com/makiuchi-d/gozxing v0.1.1/go.mod h1:eRIHbOjX7QWxLIDJoQuMLhuXg9LAuw6znsUtRkNw9DU=
github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs=
github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs=
github.com/pion/datachannel v1.6.3/go.mod h1:oRmkXVVXxkUbqa8MatgdNHW29kvtQY3HUdjJQsfv2GE=
github.com/pion/datachannel v1.6.3/go.mod h1:oRmkXVVXxkUbqa8MatgdNHW29kvtQY3HUdjJQsfv2GE=
github.com/pion/dtls/v3 v3.1.9 h1:rpeycmLIkc4krpk1IxP7+39o11QdCXbmV9+FGi9yZJ8=
github.com/pion/dtls/v3 v3.1.9 h1:rpeycmLIkc4krpk1IxP7+39o11QdCXbmV9+FGi9yZJ8=
@@ -40,6 +42,8 @@ github.com/pion/turn/v5 v5.1.2 h1:acEOO+D5txJ52vKfcilmSMItfP23Q9K6pJ1LM5P2VV0=
@@ -40,6 +42,8 @@ github.com/pion/turn/v5 v5.1.2 h1:acEOO+D5txJ52vKfcilmSMItfP23Q9K6pJ1LM5P2VV0=
github.com/pion/turn/v5 v5.1.2/go.mod h1:57Oadc8fHnac15VFyCd6nEaBTVZBxQe2POUMD4lsHRA=
github.com/pion/turn/v5 v5.1.2/go.mod h1:57Oadc8fHnac15VFyCd6nEaBTVZBxQe2POUMD4lsHRA=
github.com/pion/webrtc/v4 v4.2.22 h1:eoRTHzLKW5tI9o5iahw3s4KL4IHPESsfIF06zWi3JAo=
github.com/pion/webrtc/v4 v4.2.22 h1:eoRTHzLKW5tI9o5iahw3s4KL4IHPESsfIF06zWi3JAo=
github.com/pion/webrtc/v4 v4.2.22/go.mod h1:JBu1ba8lUEX4jbYFKofglMQJG7f0rJk7UepKjI3GaLE=
github.com/pion/webrtc/v4 v4.2.22/go.mod h1:JBu1ba8lUEX4jbYFKofglMQJG7f0rJk7UepKjI3GaLE=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
@@ -52,5 +56,9 @@ golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
@@ -52,5 +56,9 @@ golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -110,6 +110,7 @@ type Config struct {
@@ -110,6 +110,7 @@ type Config struct {
DeviceKind string `json:"device_kind,omitempty"`
DeviceKind string `json:"device_kind,omitempty"`
Plugins []PluginConfig `json:"plugins"`
Plugins []PluginConfig `json:"plugins"`
Video VideoConfig `json:"video"`
Video VideoConfig `json:"video"`
+ Stream StreamConfig `json:"stream,omitzero"`
// Limits are the local owner's caps, applied on top of the server's (the stricter value wins).
// Limits are the local owner's caps, applied on top of the server's (the stricter value wins).
Limits LocalLimits `json:"limits"`
Limits LocalLimits `json:"limits"`
// Python is the interpreter for plugins without an explicit command (default: <state>/venv/bin/python, then python3).
// Python is the interpreter for plugins without an explicit command (default: <state>/venv/bin/python, then python3).
@@ -334,6 +335,20 @@ type PluginConfig struct {
@@ -334,6 +335,20 @@ type PluginConfig struct {
StrictTiming bool `json:"strict_timing,omitempty"`
StrictTiming bool `json:"strict_timing,omitempty"`
}
}
+// StreamConfig is the agent desk stream (Agent Streams S1). LiveURL is where POST /api/streams/agent and the state,
+// ticket, curtain and frame calls go; the default is production.
+type StreamConfig struct {
+ LiveURL string `json:"live_url,omitempty"`
+}
+
+// LiveOrigin is the agent-stream API origin, production unless configured.
+func (c StreamConfig) LiveOrigin() string {
+ if c.LiveURL != "" {
+ return c.LiveURL
+ }
+ return "https://openvibe.live"
+}
+
// VideoConfig selects the camera source published over WHIP.
// VideoConfig selects the camera source published over WHIP.
type VideoConfig struct {
type VideoConfig struct {
// Source: "auto" (default: a plugin's h264_command if its program exists, else a plugin's JPEG frames, else the
// Source: "auto" (default: a plugin's h264_command if its program exists, else a plugin's JPEG frames, else the
@@ -450,6 +465,9 @@ func (c *Config) Validate() error {
@@ -450,6 +465,9 @@ func (c *Config) Validate() error {
if c.Video.WHIPURL != "" && !strings.HasPrefix(c.Video.WHIPURL, "https://") && !strings.HasPrefix(c.Video.WHIPURL, "http://") {
if c.Video.WHIPURL != "" && !strings.HasPrefix(c.Video.WHIPURL, "https://") && !strings.HasPrefix(c.Video.WHIPURL, "http://") {
return errors.New("config: video.whip_url must be an https:// URL")
return errors.New("config: video.whip_url must be an https:// URL")
}
}
+ if c.Stream.LiveURL != "" && !strings.HasPrefix(c.Stream.LiveURL, "https://") {
+ return errors.New("config: stream.live_url must be an https:// origin")
+ }
for _, l := range []*float64{c.Limits.MaxSpeed, c.Limits.MaxTurn} {
for _, l := range []*float64{c.Limits.MaxSpeed, c.Limits.MaxTurn} {
if l != nil && (*l < 0 || *l > 1) {
if l != nil && (*l < 0 || *l > 1) {
return errors.New("config: limits must be between 0 and 1")
return errors.New("config: limits must be between 0 and 1")
@@ -35,6 +35,7 @@ import (
@@ -35,6 +35,7 @@ import (
"github.com/OpenVibers/OpenVibe.Node/internal/plugins"
"github.com/OpenVibers/OpenVibe.Node/internal/plugins"
"github.com/OpenVibers/OpenVibe.Node/internal/protocol"
"github.com/OpenVibers/OpenVibe.Node/internal/protocol"
"github.com/OpenVibers/OpenVibe.Node/internal/safety"
"github.com/OpenVibers/OpenVibe.Node/internal/safety"
+ "github.com/OpenVibers/OpenVibe.Node/internal/stream"
"github.com/OpenVibers/OpenVibe.Node/internal/video"
"github.com/OpenVibers/OpenVibe.Node/internal/video"
"github.com/OpenVibers/OpenVibe.Node/internal/worker"
"github.com/OpenVibers/OpenVibe.Node/internal/worker"
)
)
@@ -79,6 +80,11 @@ type Node struct {
@@ -79,6 +80,11 @@ type Node struct {
pub *video.Publisher
pub *video.Publisher
// screen is the owner's live view of the machine's screen (screen.go), apart from a robot's camera.
// screen is the owner's live view of the machine's screen (screen.go), apart from a robot's camera.
screen screenState
screen screenState
+ // deskStream is the agent's desk stream (stream.go, Agent Streams S1); one at a time.
+ deskMu sync.Mutex
+ deskStream *stream.Agent
+ // liveTokens buys node tokens for openvibe.live (Agent Streams S1). Nil unless this machine is Network-paired.
+ liveTokens *link.TokenSource
ctx context.Context
ctx context.Context
// quit ends Run on `openvibe-node quit` (control), and quitting tells Run to answer ErrQuit.
// quit ends Run on `openvibe-node quit` (control), and quitting tells Run to answer ErrQuit.
@@ -228,6 +234,11 @@ func New(opt Options) (*Node, error) {
@@ -228,6 +234,11 @@ func New(opt Options) (*Node, error) {
}
}
}
}
n.link = link.New(lo, n)
n.link = link.New(lo, n)
+ if opt.Creds.NetworkPaired() {
+ // Agent Streams S1: a Node principal acting for its owner calls openvibe.live. Network grants it
+ // live.agent-stream.publish for its owner.
+ n.liveTokens = link.NewTokenSourceFor(opt.Creds, nil, lo.UserAgent, "openvibe.live")
+ }
// The WHIP endpoint is the pairing's whip_url (built by Bot from the publish key) unless the config sets one.
// The WHIP endpoint is the pairing's whip_url (built by Bot from the publish key) unless the config sets one.
whip := opt.Config.Video.WHIPURL
whip := opt.Config.Video.WHIPURL
if whip == "" {
if whip == "" {
@@ -302,6 +313,7 @@ func (n *Node) Run(parent context.Context) error {
@@ -302,6 +313,7 @@ func (n *Node) Run(parent context.Context) error {
}
}
<-ctx.Done()
<-ctx.Done()
n.log.Info("shutting down: stopping every actuator")
n.log.Info("shutting down: stopping every actuator")
+ n.stopDeskStream(false)
n.mgr.StopAll(plugins.StopShutdown)
n.mgr.StopAll(plugins.StopShutdown)
if n.worker != nil {
if n.worker != nil {
n.worker.Close()
n.worker.Close()
@@ -747,6 +759,10 @@ func (n *Node) computer(c protocol.Command, entry *safety.Entry, sentMS int64, r
@@ -747,6 +759,10 @@ func (n *Node) computer(c protocol.Command, entry *safety.Entry, sentMS int64, r
n.screenCommand(c, entry, received, nack)
n.screenCommand(c, entry, received, nack)
return
return
}
}
+ if c.Kind == protocol.KindDeskStream {
+ n.deskStreamCommand(c, entry, received, nack)
+ return
+ }
if protocol.ComputerActs(c.Kind, c.Value) {
if protocol.ComputerActs(c.Kind, c.Value) {
st := n.latch.State()
st := n.latch.State()
if st.Local {
if st.Local {
@@ -928,6 +944,9 @@ func (n *Node) onLatch(st safety.LatchState) {
@@ -928,6 +944,9 @@ func (n *Node) onLatch(st safety.LatchState) {
n.log.Warn("stop cleared; motion allowed again")
n.log.Warn("stop cleared; motion allowed again")
}
}
n.applyPluginLatch(st, true)
n.applyPluginLatch(st, true)
+ if st.Stopped() || st.Computer {
+ n.stopDeskStream(true) // the kill switch: Live ends the WHIP session at once
+ }
n.sendEstopState()
n.sendEstopState()
n.sendStatus()
n.sendStatus()
@@ -1009,6 +1028,12 @@ func (n *Node) output(o plugins.Output) {
@@ -1009,6 +1028,12 @@ func (n *Node) output(o plugins.Output) {
n.tellVoice(n.mgr.Get(o.Plugin))
n.tellVoice(n.mgr.Get(o.Plugin))
return
return
}
}
+ if name == "curtain" {
+ n.deskCurtainEvent(o.Plugin, o.Data)
+ if heartbeat, _ := o.Data["heartbeat"].(bool); heartbeat {
+ return // the stream's one-second liveness beat, not an event for the log or Bot
+ }
+ }
fields := map[string]any{}
fields := map[string]any{}
for k, v := range o.Data {
for k, v := range o.Data {
if k != "name" {
if k != "name" {
@@ -1052,7 +1077,11 @@ func (n *Node) output(o plugins.Output) {
@@ -1052,7 +1077,11 @@ func (n *Node) output(o plugins.Output) {
if _, err := n.latch.SetLocalWith(reason); err != nil {
if _, err := n.latch.SetLocalWith(reason); err != nil {
n.log.Error("a plugin latched the local stop, but the latch could not be saved", "plugin", o.Plugin, "err", err)
n.log.Error("a plugin latched the local stop, but the latch could not be saved", "plugin", o.Plugin, "err", err)
}
}
- case plugins.OutFault, plugins.OutState:
+ case plugins.OutState:
+ n.deskPluginState(o)
+ n.sendStatus()
+ case plugins.OutFault:
+ n.deskPluginFault(o)
n.sendStatus()
n.sendStatus()
}
}
}
}
@@ -1221,21 +1250,23 @@ func strSlice(v any) []string {
@@ -1221,21 +1250,23 @@ func strSlice(v any) []string {
// Status is what `openvibe-node status` prints. It never contains the credential or the publish key.
// Status is what `openvibe-node status` prints. It never contains the credential or the publish key.
type Status struct {
type Status struct {
- Version string `json:"version"`
- DeviceID string `json:"device_id,omitempty"`
- Principal string `json:"principal,omitempty"` // nod_… of a Network-paired machine
- Server string `json:"server,omitempty"`
- Paired bool `json:"paired"`
- Uptime string `json:"uptime"`
- Link LinkStatus `json:"link"`
- Latch safety.LatchState `json:"latch"`
- Limits safety.Limits `json:"limits"`
- Plugins []PluginStatus `json:"plugins"`
- Video VideoStatus `json:"video"`
- Screen ScreenStatus `json:"screen"`
- ConfigDir string `json:"config_dir"`
- StateDir string `json:"state_dir"`
- Telemetry protocol.Telemetry `json:"telemetry"`
+ Version string `json:"version"`
+ DeviceID string `json:"device_id,omitempty"`
+ Principal string `json:"principal,omitempty"` // nod_… of a Network-paired machine
+ Server string `json:"server,omitempty"`
+ Paired bool `json:"paired"`
+ Uptime string `json:"uptime"`
+ Link LinkStatus `json:"link"`
+ Latch safety.LatchState `json:"latch"`
+ Limits safety.Limits `json:"limits"`
+ Plugins []PluginStatus `json:"plugins"`
+ Video VideoStatus `json:"video"`
+ Screen ScreenStatus `json:"screen"`
+ // Desk stream is the agent's own stream, when one is running (Agent Streams S1).
+ DeskStream *stream.Status `json:"desk_stream,omitempty"`
+ ConfigDir string `json:"config_dir"`
+ StateDir string `json:"state_dir"`
+ Telemetry protocol.Telemetry `json:"telemetry"`
// Jobs are the worker's jobs not ended yet; empty when the worker is off.
// Jobs are the worker's jobs not ended yet; empty when the worker is off.
Jobs []worker.JobInfo `json:"jobs"`
Jobs []worker.JobInfo `json:"jobs"`
// Roles are what the machine does for the network, as the owner set them on the hub (plan L34): the ids of the
// Roles are what the machine does for the network, as the owner set them on the hub (plan L34): the ids of the
@@ -1296,6 +1327,7 @@ func (n *Node) Status() Status {
@@ -1296,6 +1327,7 @@ func (n *Node) Status() Status {
s.Jobs = n.worker.Jobs()
s.Jobs = n.worker.Jobs()
}
}
s.Screen = n.screenStatus()
s.Screen = n.screenStatus()
+ s.DeskStream = n.deskStreamStatus()
s.Video.State = video.StateOff
s.Video.State = video.StateOff
if n.pub != nil {
if n.pub != nil {
s.Video.State, s.Video.Frames, s.Video.LastError = n.pub.State(), n.pub.Frames(), n.pub.LastError()
s.Video.State, s.Video.Frames, s.Video.LastError = n.pub.State(), n.pub.Frames(), n.pub.LastError()
new file mode 100644
new file mode 100644
@@ -0,0 +1,363 @@
@@ -0,0 +1,363 @@
+package node
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "image"
+ "image/jpeg"
+ "image/png"
+ "strings"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/mono"
+ "github.com/OpenVibers/OpenVibe.Node/internal/plugins"
+ "github.com/OpenVibers/OpenVibe.Node/internal/protocol"
+ "github.com/OpenVibers/OpenVibe.Node/internal/safety"
+ "github.com/OpenVibers/OpenVibe.Node/internal/stream"
+)
+
+// The agent desk stream (Agent Streams S1, live.agent-stream@1): the core mints the stream and its wpt_ ticket on
+// OpenVibe.Live with this machine's node token, asks the computer plugin for the desk's capture source, then publishes
+// over WHIP. The ticket never goes to the plugin, into a log line or into status.
+//
+// Commands:
+//
+// {"action":"start","title":…,"task":{source,ref,title,url?},"visibility"?,"allow_sites"?,"desk"?}
+// {"action":"stop"}
+// {"action":"curtain","up":true|false,"reason"?:…}
+// {"action":"status"}
+type deskStreamValue struct {
+ Action string `json:"action"`
+ Title string `json:"title"`
+ Task stream.Task `json:"task"`
+ Visibility string `json:"visibility"`
+ AllowSites []string `json:"allow_sites"`
+ Desk string `json:"desk"`
+ Up bool `json:"up"`
+ Reason string `json:"reason"`
+}
+
+// deskStreamCommand answers one desk.stream command. start creates the stream on Live and publishes; stop ends it;
+// curtain blanks it; status reads it. None of them ever carries the ticket.
+func (n *Node) deskStreamCommand(c protocol.Command, entry *safety.Entry, received mono.Instant, nack func(code, msg string)) {
+ var v deskStreamValue
+ if err := json.Unmarshal(c.Value, &v); err != nil {
+ nack(protocol.FaultBadValue, "value must be {action, …}")
+ return
+ }
+ switch v.Action {
+ case "status":
+ var data []byte
+ if st := n.deskStreamStatus(); st != nil {
+ data, _ = json.Marshal(map[string]any{"running": true, "stream": st})
+ } else {
+ data, _ = json.Marshal(map[string]any{"running": false})
+ }
+ r := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds(), Data: data}
+ n.dedup.Complete(entry, r)
+ n.send(r)
+ return
+ case "stop":
+ stopped := n.stopDeskStream(false)
+ p := n.mgr.Route(protocol.KindDeskStream, c.Target, "")
+ if p != nil {
+ // Best effort: the core's answer does not depend on the plugin.
+ wait := p.Begin(n.ctx, c.ID+":stop", protocol.KindDeskStream, json.RawMessage(`{"action":"stop"}`), 10000)
+ go wait()
+ }
+ data, _ := json.Marshal(map[string]any{"stopping": stopped})
+ r := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds(), Data: data}
+ n.dedup.Complete(entry, r)
+ n.send(r)
+ return
+ case "curtain":
+ n.deskMu.Lock()
+ a := n.deskStream
+ n.deskMu.Unlock()
+ if a == nil {
+ nack(protocol.FaultNotReady, "no desk stream is running")
+ return
+ }
+ reason := v.Reason
+ if reason == "" {
+ reason = "manual"
+ }
+ a.SetCurtain(v.Up, reason)
+ p := n.mgr.Route(protocol.KindDeskStream, c.Target, "")
+ if p != nil {
+ body, _ := json.Marshal(map[string]any{"action": "curtain", "up": v.Up, "reason": reason})
+ wait := p.Begin(n.ctx, c.ID, protocol.KindDeskStream, body, 10000)
+ go wait()
+ }
+ r := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds()}
+ n.dedup.Complete(entry, r)
+ n.send(r)
+ return
+ case "start":
+ default:
+ nack(protocol.FaultBadValue, "action is start, stop, curtain or status")
+ return
+ }
+
+ st := n.latch.State()
+ if st.Stopped() || st.Computer {
+ nack(protocol.FaultLocalStop, "stopped on the machine: `openvibe-node resume` on this machine is required")
+ return
+ }
+ if n.liveTokens == nil {
+ nack(protocol.FaultUnsupported, "an agent stream needs this machine paired through OpenVibe.Network (pair again with --network)")
+ return
+ }
+ if p := n.mgr.Route(protocol.KindDeskStream, c.Target, ""); p == nil {
+ nack(protocol.FaultUnsupported, "the computer plugin is off: no desk to stream")
+ return
+ }
+ if v.Visibility == "" {
+ v.Visibility = "private"
+ }
+ if v.Title == "" {
+ v.Title = "AI agent"
+ }
+ if v.Task.Source == "" {
+ v.Task.Source = "desk"
+ }
+ if v.Task.Ref == "" {
+ v.Task.Ref = v.Desk
+ if v.Task.Ref == "" {
+ v.Task.Ref = "desk"
+ }
+ }
+ if v.Task.Title == "" {
+ v.Task.Title = v.Title
+ }
+ ua := "openvibe-node/" + n.opt.Version
+ client, err := stream.NewClient(stream.ClientOptions{BaseURL: n.opt.Config.Stream.LiveOrigin(),
+ Token: n.liveTokens.Token, UserAgent: ua})
+ if err != nil {
+ nack(protocol.FaultBadValue, err.Error())
+ return
+ }
+ created, err := client.Create(n.ctx, stream.CreateRequest{Title: v.Title, Task: v.Task,
+ Visibility: v.Visibility, AllowSites: v.AllowSites})
+ if err != nil {
+ n.log.Warn("desk stream create failed", "err", err)
+ nack(protocol.FaultHardware, err.Error())
+ return
+ }
+ // The plugin is asked after Live minted the ticket: if it cannot start, the stream is ended at once.
+ p := n.mgr.Route(protocol.KindDeskStream, c.Target, "")
+ toPlugin, _ := json.Marshal(map[string]any{"action": "start", "desk": v.Desk, "allow_sites": v.AllowSites,
+ "visibility": v.Visibility, "title": v.Title})
+ go func() {
+ wait := p.Begin(n.ctx, c.ID, protocol.KindDeskStream, toPlugin, 120_000)
+ r := wait()
+ if !r.OK {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ _ = client.End(ctx, created.Stream.ID, created.Publish.Ticket)
+ cancel()
+ nack(r.FaultCode, r.Message)
+ return
+ }
+ var got struct {
+ Capture stream.Capture `json:"capture"`
+ Curtain struct {
+ Up bool `json:"up"`
+ Reason string `json:"reason"`
+ } `json:"curtain"`
+ }
+ _ = json.Unmarshal(r.Data, &got)
+ if got.Capture.Kind == "" {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ _ = client.End(ctx, created.Stream.ID, created.Publish.Ticket)
+ cancel()
+ nack(protocol.FaultHardware, "the desk plugin returned no capture source")
+ return
+ }
+ enc := stream.DefaultEncode()
+ if got.Capture.Width > 0 && got.Capture.Height > 0 {
+ enc.Width, enc.Height = got.Capture.Width, got.Capture.Height
+ }
+ agent, err := stream.NewAgent(stream.AgentOptions{
+ Client: client, ID: created.Stream.ID, Publish: created.Publish, Capture: got.Capture, Encode: enc,
+ FFmpeg: n.opt.Config.Video.FFmpeg, Log: n.log, Plugin: p.Name(),
+ Snapshot: n.deskSnapshotFunc(p, v.Desk),
+ })
+ if err != nil {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ _ = client.End(ctx, created.Stream.ID, created.Publish.Ticket)
+ cancel()
+ nack(protocol.FaultHardware, err.Error())
+ return
+ }
+ // The plugin starts its curtain up (fail closed) before it answers; its change event can arrive before this
+ // agent exists, so take the initial state from the ack and never show the desk unblanked.
+ if got.Curtain.Up {
+ reason := got.Curtain.Reason
+ if reason == "" {
+ reason = "detector"
+ }
+ agent.SetCurtain(true, reason)
+ }
+ n.stopDeskStream(false)
+ n.deskMu.Lock()
+ n.deskStream = agent
+ n.deskMu.Unlock()
+ go func() {
+ if err := agent.Run(n.ctx); err != nil {
+ n.log.Warn("desk stream ended", "stream", created.Stream.ID, "err", err)
+ }
+ }()
+ data, _ := json.Marshal(map[string]any{"streaming": true, "id": created.Stream.ID,
+ "page_url": created.Stream.PageURL, "visibility": created.Stream.Visibility,
+ "expires_at": created.Publish.ExpiresAt.UTC().Format(time.RFC3339), "capture": got.Capture.Kind})
+ r2 := protocol.AckData{ID: c.ID, LatencyMS: mono.Since(received).Milliseconds(), Data: data}
+ n.dedup.Complete(entry, r2)
+ n.send(r2)
+ }()
+}
+
+// stopDeskStream ends the running desk stream, if any → whether one ran. revoke is the kill switch: Live ends the WHIP
+// session at once. Otherwise the stream ends and its card stays.
+func (n *Node) stopDeskStream(revoke bool) bool {
+ n.deskMu.Lock()
+ a := n.deskStream
+ n.deskStream = nil
+ n.deskMu.Unlock()
+ if a == nil {
+ return false
+ }
+ if revoke {
+ a.Revoke()
+ } else {
+ a.Stop()
+ }
+ return true
+}
+
+func (n *Node) deskStreamStatus() *stream.Status {
+ n.deskMu.Lock()
+ a := n.deskStream
+ n.deskMu.Unlock()
+ if a == nil {
+ return nil
+ }
+ st := a.Status()
+ return &st
+}
+
+// deskSnapshotFunc takes a picture for the card while nobody watches: the computer plugin's own screen command, which
+// works the same for a desk's display and a browser desk's page (and never touches the owner's screen).
+func (n *Node) deskSnapshotFunc(p *plugins.Plugin, desk string) func(context.Context) ([]byte, error) {
+ return func(ctx context.Context) ([]byte, error) {
+ value := map[string]any{"max_width": 1280}
+ if desk != "" {
+ value["desk"] = desk
+ }
+ body, _ := json.Marshal(value)
+ id := fmt.Sprintf("desk-stream-frame-%d", time.Now().UnixNano())
+ r := p.Begin(ctx, id, protocol.KindScreen, body, 20_000)()
+ if !r.OK {
+ return nil, fmt.Errorf("the desk plugin could not take a snapshot: %s %s", r.FaultCode, r.Message)
+ }
+ var got struct {
+ Format string `json:"format"`
+ ContentB64 string `json:"content_b64"`
+ }
+ if err := json.Unmarshal(r.Data, &got); err != nil || got.ContentB64 == "" {
+ return nil, errors.New("the desk plugin returned no picture")
+ }
+ raw, err := base64.StdEncoding.DecodeString(got.ContentB64)
+ if err != nil {
+ return nil, errors.New("the desk plugin returned a bad picture")
+ }
+ if strings.EqualFold(got.Format, "png") {
+ return pngToJPEG(raw)
+ }
+ if len(raw) > stream.MaxFrameBytes {
+ return nil, fmt.Errorf("the snapshot is larger than %d bytes", stream.MaxFrameBytes)
+ }
+ return raw, nil
+ }
+}
+
+// pngToJPEG turns a capture tool's PNG into the JPEG the frame endpoint takes.
+func pngToJPEG(raw []byte) ([]byte, error) {
+ img, err := png.Decode(bytes.NewReader(raw))
+ if err != nil {
+ return nil, errors.New("the desk plugin returned a bad PNG")
+ }
+ // Downscale to at most 1280 wide with a simple nearest-neighbour pass; snapshots are a card, not the stream.
+ b := img.Bounds()
+ w, h := b.Dx(), b.Dy()
+ if w > 1280 {
+ h = h * 1280 / w
+ w = 1280
+ }
+ dst := image.NewRGBA(image.Rect(0, 0, w, h))
+ for y := 0; y < h; y++ {
+ for x := 0; x < w; x++ {
+ dst.Set(x, y, img.At(b.Min.X+x*b.Dx()/w, b.Min.Y+y*b.Dy()/h))
+ }
+ }
+ var buf bytes.Buffer
+ if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 80}); err != nil {
+ return nil, err
+ }
+ if buf.Len() > stream.MaxFrameBytes {
+ return nil, fmt.Errorf("the snapshot is larger than %d bytes", stream.MaxFrameBytes)
+ }
+ return buf.Bytes(), nil
+}
+
+// deskCurtainEvent is the computer plugin's own curtain (detectors, hotkey, Curtain now): the desk reports it to Live
+// and blanks the picture. A detector that failed already raised it in the plugin.
+func (n *Node) deskCurtainEvent(plugin string, data map[string]any) {
+ n.deskMu.Lock()
+ a := n.deskStream
+ n.deskMu.Unlock()
+ if a == nil || (a.PluginName() != "" && a.PluginName() != plugin) {
+ return
+ }
+ up, _ := data["up"].(bool)
+ reason, _ := data["reason"].(string)
+ if reason == "" {
+ reason = "detector"
+ }
+ a.ObserveCurtain(up, reason)
+}
+
+// deskPluginState is the supervisor's state for the plugin that owns the desk's detectors: anything but ready is a
+// lost plugin, so the curtain goes up and the audio mutes at once (fail closed).
+func (n *Node) deskPluginState(o plugins.Output) {
+ a := n.currentDeskStream()
+ if a == nil || a.PluginName() == "" || a.PluginName() != o.Plugin {
+ return
+ }
+ state, _ := o.Data["state"].(string)
+ if state == plugins.StateReady {
+ a.PluginBack()
+ return
+ }
+ a.PluginLost()
+}
+
+// deskPluginFault is the plugin going down between state reports (plugin_down): the same fail-closed curtain.
+func (n *Node) deskPluginFault(o plugins.Output) {
+ a := n.currentDeskStream()
+ if a == nil || a.PluginName() != o.Plugin {
+ return
+ }
+ if code, _ := o.Data["fault_code"].(string); code == protocol.FaultPluginDown {
+ a.PluginLost()
+ }
+}
+
+func (n *Node) currentDeskStream() *stream.Agent {
+ n.deskMu.Lock()
+ defer n.deskMu.Unlock()
+ return n.deskStream
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,53 @@
@@ -0,0 +1,53 @@
+package node
+
+import (
+ "bytes"
+ "image"
+ "image/color"
+ "image/jpeg"
+ "image/png"
+ "testing"
+)
+
+// jpegConfig decodes the JPEG the snapshot path produced (it is a normal image, not an encoder detail).
+func jpegConfig(b []byte) (image.Config, error) {
+ return jpeg.DecodeConfig(bytes.NewReader(b))
+}
+
+func TestDeskStreamStatusAndStopAreSafeBeforeAStream(t *testing.T) {
+ n := &Node{}
+ if st := n.deskStreamStatus(); st != nil {
+ t.Fatalf("status before a stream = %+v", st)
+ }
+ if n.stopDeskStream(false) || n.stopDeskStream(true) {
+ t.Fatal("stop reported a stream that never ran")
+ }
+ n.deskCurtainEvent("computer", map[string]any{"up": true, "reason": "manual"})
+}
+
+func TestPNGToJPEG(t *testing.T) {
+ src := image.NewRGBA(image.Rect(0, 0, 2000, 1000))
+ for y := 0; y < 1000; y++ {
+ for x := 0; x < 2000; x++ {
+ src.Set(x, y, color.RGBA{R: uint8(x), G: uint8(y), B: 40, A: 255})
+ }
+ }
+ var buf bytes.Buffer
+ if err := png.Encode(&buf, src); err != nil {
+ t.Fatal(err)
+ }
+ out, err := pngToJPEG(buf.Bytes())
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg, err := jpegConfig(out)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if cfg.Width != 1280 {
+ t.Fatalf("snapshot width = %d, want 1280", cfg.Width)
+ }
+ if len(out) > 200*1024 {
+ t.Fatalf("snapshot is %d bytes", len(out))
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,81 @@
@@ -0,0 +1,81 @@
+package node
+
+import (
+ "context"
+ "io"
+ "log/slog"
+ "testing"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+ "github.com/OpenVibers/OpenVibe.Node/internal/plugins"
+ "github.com/OpenVibers/OpenVibe.Node/internal/protocol"
+ "github.com/OpenVibers/OpenVibe.Node/internal/stream"
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+type watchFakeLive struct{}
+
+func (watchFakeLive) State(context.Context, string, credentials.Secret) (stream.State, error) {
+ return stream.State{Encode: true}, nil
+}
+func (watchFakeLive) Renew(context.Context, string, credentials.Secret) (stream.Publish, error) {
+ return stream.Publish{}, nil
+}
+func (watchFakeLive) Curtain(context.Context, string, credentials.Secret, bool, string) error {
+ return nil
+}
+func (watchFakeLive) PostFrame(context.Context, string, credentials.Secret, []byte) error { return nil }
+func (watchFakeLive) End(context.Context, string, credentials.Secret) error { return nil }
+func (watchFakeLive) Revoke(context.Context, string, credentials.Secret) error { return nil }
+
+func newWatchAgent(t *testing.T) (*Node, *stream.Agent) {
+ t.Helper()
+ a, err := stream.NewAgent(stream.AgentOptions{Client: watchFakeLive{}, ID: "ags_01J8Z4M2Q0R7T9YV3K6N8P1W2X",
+ Publish: stream.Publish{WHIPURL: "https://whip.example/whip/agent/ags_01J8Z4M2Q0R7T9YV3K6N8P1W2X",
+ Ticket: credentials.NewSecret("wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG"), ExpiresAt: time.Now().Add(time.Hour)},
+ Plugin: "computer", Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
+ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ n := &Node{}
+ n.deskStream = a
+ return n, a
+}
+
+func TestAPluginStateThatIsNotReadyRaisesTheCurtain(t *testing.T) {
+ n, a := newWatchAgent(t)
+ if a.Status().Curtain {
+ t.Fatal("the stream started with the curtain up")
+ }
+ n.deskPluginState(plugins.Output{Plugin: "computer", Data: map[string]any{"state": plugins.StateDown}})
+ if !a.Status().Curtain {
+ t.Fatal("the plugin going down did not raise the curtain")
+ }
+ // Another plugin's state is not this stream's business.
+ n.deskPluginState(plugins.Output{Plugin: "dryrun", Data: map[string]any{"state": plugins.StateReady}})
+ if !a.Status().Curtain {
+ t.Fatal("another plugin's state lowered it")
+ }
+ n.deskPluginFault(plugins.Output{Plugin: "computer", Data: map[string]any{"fault_code": protocol.FaultPluginDown}})
+ if !a.Status().Curtain {
+ t.Fatal("plugin_down did not keep it up")
+ }
+}
+
+func TestACurtainHeartbeatFromThePluginIsObserved(t *testing.T) {
+ n, a := newWatchAgent(t)
+ n.deskCurtainEvent("dryrun", map[string]any{"up": true, "reason": "password_field"})
+ if a.Status().Curtain {
+ t.Fatal("another plugin's curtain moved this stream")
+ }
+ n.deskCurtainEvent("computer", map[string]any{"up": true, "reason": "password_field", "heartbeat": true})
+ if !a.Status().Curtain {
+ t.Fatal("a heartbeat saying up did not raise the curtain")
+ }
+ n.deskCurtainEvent("computer", map[string]any{"up": false, "reason": "detector", "heartbeat": true})
+ if !a.Status().Curtain {
+ t.Fatal("one clear heartbeat lowered the curtain without the hold")
+ }
+}
@@ -25,6 +25,7 @@ var capabilityFor = map[string]string{
@@ -25,6 +25,7 @@ var capabilityFor = map[string]string{
protocol.KindInput: "input",
protocol.KindInput: "input",
protocol.KindBrowser: "browser",
protocol.KindBrowser: "browser",
protocol.KindDesk: "desk",
protocol.KindDesk: "desk",
+ protocol.KindDeskStream: "desk.stream",
protocol.KindAI: "ai",
protocol.KindAI: "ai",
}
}
@@ -192,13 +192,19 @@ const (
@@ -192,13 +192,19 @@ const (
// "messages", "max_tokens"?, "temperature"?} → {model, message, usage?}. It reads and changes nothing on the
// "messages", "max_tokens"?, "temperature"?} → {model, message, usage?}. It reads and changes nothing on the
// machine, so it never acts and runs while a stop is latched like a read.
// machine, so it never acts and runs while a stop is latched like a read.
KindAI = "ai"
KindAI = "ai"
+ // KindDeskStream is an agent's own desk stream (Agent Streams S1, live.agent-stream@1): {"action": "start", "title",
+ // "task": {source, ref, title, url?}, "visibility": "private"|"unlisted"|"public", "allow_sites"?}, or
+ // {"action": "stop"|"status"|"curtain", "up"?, "reason"?}. The core mints the stream and its wpt_ ticket on Live,
+ // routes start to the computer plugin for the desk's capture source, then publishes over WHIP. Starting and
+ // stopping act; status does not.
+ KindDeskStream = "desk.stream"
)
)
// Kinds lists every command kind the Node understands.
// Kinds lists every command kind the Node understands.
-var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream, KindDesk, KindAI}
+var Kinds = []string{KindDrive, KindActuator, KindPTZ, KindSay, KindDisplay, KindHalt, KindButton, KindPoint, KindExec, KindFS, KindScreen, KindProbe, KindInput, KindBrowser, KindScreenStream, KindDesk, KindDeskStream, KindAI}
// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.
// ComputerKinds move nothing: they carry long deadlines (DefaultComputerMS, at most MaxComputerMS) and no clamping.
-var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true, KindDesk: true, KindAI: true}
+var ComputerKinds = map[string]bool{KindExec: true, KindFS: true, KindScreen: true, KindProbe: true, KindInput: true, KindBrowser: true, KindScreenStream: true, KindDesk: true, KindDeskStream: true, KindAI: true}
const (
const (
// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps
// DefaultComputerMS is a computer command's reply window when the server sends no deadline; MaxComputerMS caps
@@ -235,6 +241,12 @@ func ComputerActs(kind string, value json.RawMessage) bool {
@@ -235,6 +241,12 @@ func ComputerActs(kind string, value json.RawMessage) bool {
}
}
_ = json.Unmarshal(value, &v)
_ = json.Unmarshal(value, &v)
return v.Action != "status" && v.Action != "list"
return v.Action != "status" && v.Action != "list"
+ case KindDeskStream:
+ var v struct {
+ Action string `json:"action"`
+ }
+ _ = json.Unmarshal(value, &v)
+ return v.Action != "status"
}
}
return false
return false
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,700 @@
@@ -0,0 +1,700 @@
+package stream
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "sync"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+// liveAPI is the part of Client the agent uses; tests substitute it.
+type liveAPI interface {
+ State(context.Context, string, credentials.Secret) (State, error)
+ Renew(context.Context, string, credentials.Secret) (Publish, error)
+ Curtain(context.Context, string, credentials.Secret, bool, string) error
+ PostFrame(context.Context, string, credentials.Secret, []byte) error
+ End(context.Context, string, credentials.Secret) error
+ Revoke(context.Context, string, credentials.Secret) error
+}
+
+// publisher is the part of video.Publisher the agent uses; tests substitute it.
+type publisher interface {
+ Run(context.Context, video.Source)
+ SetPaused(bool)
+ SetMuted(bool)
+ SetPublishKey(credentials.Secret)
+ State() string
+ Frames() int64
+ LastError() string
+}
+
+// AgentOptions configure one desk stream.
+type AgentOptions struct {
+ Client liveAPI
+ ID string
+ Publish Publish
+ Capture Capture
+ Encode EncodeOptions
+ FFmpeg string
+ Log *slog.Logger
+
+ // Snapshot returns a JPEG of the desk for the card while nobody watches. The node wires it to the computer plugin
+ // (a browser desk's page) or to a one-shot ffmpeg capture (a display).
+ Snapshot func(context.Context) ([]byte, error)
+ // CurtainJPEG is the "Private screen" picture posted as the snapshot while the curtain is up.
+ CurtainJPEG []byte
+ // CurtainFrames are pre-encoded H.264 access units of the curtain, looped while it is up.
+ CurtainFrames []video.Sample
+
+ StateEvery time.Duration // default 10 s
+ SnapshotEvery time.Duration // default 5 s
+ RenewAfter time.Duration // default 3 h (the design's renewal point)
+ Now func() time.Time
+ // Plugin is the computer plugin that owns the desk's detectors. Its exit or silence raises the curtain; the
+ // watchdog runs every WatchEvery, a loss is declared after CurtainLostAfter, and clear states must last
+ // CurtainClearHold before the curtain comes down.
+ Plugin string
+ WatchEvery time.Duration
+ CurtainLostAfter time.Duration
+ CurtainClearHold time.Duration
+
+ // NewPublisher and NewSource are for tests; production uses video.NewPublisher and FFmpegSource.
+ NewPublisher func(video.Options) publisher
+ NewSource func(Capture, EncodeOptions) video.Source
+}
+
+// Agent is one desk stream: a WHIP publisher, the capture pipeline, Live's state, the curtain and the ticket.
+type Agent struct {
+ opt AgentOptions
+ log *slog.Logger
+
+ ctx context.Context
+ cancel context.CancelFunc
+ pub publisher
+ ds *deskSource
+
+ mu sync.Mutex
+ ticket credentials.Secret
+ encode bool
+ curtain bool
+ reason string
+ viewers int
+ ended bool
+ revoked bool
+ expiresAt time.Time
+ retries int
+ downshifted bool
+ // The plugin's curtain state: when it was last heard, whether the supervisor says it is down, and since when it
+ // has been reporting clear while the curtain is up.
+ lastPluginAt time.Time
+ pluginDown bool
+ clearSince time.Time
+
+ snapMu sync.Mutex
+}
+
+// NewAgent validates and returns an agent; Run starts it.
+func NewAgent(o AgentOptions) (*Agent, error) {
+ if o.Client == nil || o.ID == "" {
+ return nil, errors.New("stream: an agent needs a client and a stream id")
+ }
+ if err := checkID(o.ID); err != nil {
+ return nil, err
+ }
+ if o.Publish.WHIPURL == "" || o.Publish.Ticket.IsZero() {
+ return nil, errors.New("stream: an agent needs a WHIP URL and a publish ticket")
+ }
+ if o.Publish.ExpiresAt.IsZero() {
+ return nil, errors.New("stream: the publish part has no expiry")
+ }
+ if o.StateEvery <= 0 {
+ o.StateEvery = 10 * time.Second
+ }
+ if o.SnapshotEvery <= 0 {
+ o.SnapshotEvery = 5 * time.Second
+ }
+ if o.RenewAfter <= 0 {
+ o.RenewAfter = 3 * time.Hour
+ }
+ if o.WatchEvery <= 0 {
+ o.WatchEvery = 250 * time.Millisecond
+ }
+ if o.CurtainLostAfter <= 0 {
+ o.CurtainLostAfter = 2 * time.Second
+ }
+ if o.CurtainClearHold <= 0 {
+ o.CurtainClearHold = 2 * time.Second
+ }
+ if o.Now == nil {
+ o.Now = time.Now
+ }
+ if o.Log == nil {
+ o.Log = slog.Default()
+ }
+ if o.NewPublisher == nil {
+ o.NewPublisher = func(vo video.Options) publisher { return video.NewPublisher(vo) }
+ }
+ if o.NewSource == nil {
+ o.NewSource = func(c Capture, e EncodeOptions) video.Source {
+ return &FFmpegSource{FFmpeg: o.FFmpeg, Capture: c, Options: e, Log: o.Log}
+ }
+ }
+ a := &Agent{opt: o, log: o.Log.With("component", "desk-stream"), ticket: o.Publish.Ticket, encode: true,
+ expiresAt: o.Publish.ExpiresAt, lastPluginAt: o.Now()}
+ a.ds = &deskSource{a: a}
+ return a, nil
+}
+
+// Run publishes until ctx ends or the stream ends. It ends the stream at Live (so the card stays), unless Revoke was
+// called, which is the kill switch.
+func (a *Agent) Run(parent context.Context) error {
+ ctx, cancel := context.WithCancel(parent)
+ a.mu.Lock()
+ a.ctx, a.cancel = ctx, cancel
+ revoked := a.revoked
+ a.mu.Unlock()
+ if revoked {
+ cancel()
+ return nil
+ }
+ if len(a.opt.CurtainFrames) == 0 || len(a.opt.CurtainJPEG) == 0 {
+ cctx, ccancel := context.WithTimeout(ctx, 20*time.Second)
+ frames, jpg := MakeCurtain(cctx, a.opt.FFmpeg, a.opt.Capture.Width, a.opt.Capture.Height)
+ ccancel()
+ if len(a.opt.CurtainFrames) == 0 {
+ a.opt.CurtainFrames = frames
+ }
+ if len(a.opt.CurtainJPEG) == 0 {
+ a.opt.CurtainJPEG = jpg
+ }
+ }
+ pub := a.opt.NewPublisher(video.Options{WHIPURL: a.opt.Publish.WHIPURL, PublishKey: a.ticket,
+ Audio: true, UserAgent: "openvibe-node/stream", Log: a.opt.Log})
+ a.mu.Lock()
+ a.pub = pub
+ curtainUp := a.curtain
+ a.mu.Unlock()
+ if curtainUp {
+ pub.SetMuted(true)
+ }
+ defer cancel()
+ go pub.Run(ctx, a.ds)
+
+ state := time.NewTicker(a.opt.StateEvery)
+ defer state.Stop()
+ snap := time.NewTicker(a.opt.SnapshotEvery)
+ defer snap.Stop()
+ renew := time.NewTimer(a.renewDelay())
+ defer renew.Stop()
+ watch := time.NewTicker(a.opt.WatchEvery)
+ defer watch.Stop()
+
+ a.log.Info("desk stream started", "stream", a.opt.ID, "capture", a.opt.Capture.Kind)
+ a.postSnapshot(ctx) // the card has a face before the first state poll
+
+ for {
+ select {
+ case <-ctx.Done():
+ a.finish()
+ return nil
+ case <-renew.C:
+ a.renew(ctx)
+ renew.Reset(a.renewDelay())
+ case <-state.C:
+ if err := a.poll(ctx); err != nil {
+ a.log.Warn("desk stream state poll failed", "stream", a.opt.ID, "err", err)
+ }
+ case <-snap.C:
+ if !a.encodeNow() {
+ a.postSnapshot(ctx)
+ }
+ case <-watch.C:
+ a.watchCurtain()
+ }
+ }
+}
+
+func (a *Agent) renewDelay() time.Duration {
+ // The design renews at three hours; never wait past the ticket's own expiry (with a margin for the round trip).
+ a.mu.Lock()
+ exp := a.expiresAt
+ a.mu.Unlock()
+ if d := a.opt.RenewAfter; d > 0 {
+ if left := exp.Sub(a.opt.Now()) - 15*time.Minute; left > 0 && left < d {
+ return left
+ }
+ return d
+ }
+ return 3 * time.Hour
+}
+
+func (a *Agent) poll(ctx context.Context) error {
+ st, err := a.opt.Client.State(ctx, a.opt.ID, a.ticketNow())
+ if err != nil {
+ return err
+ }
+ a.mu.Lock()
+ a.viewers = st.Viewers
+ curtainUp := a.curtain
+ a.mu.Unlock()
+ if st.Curtain.Up && !curtainUp {
+ // The owner raised it on another surface. Only the desk may bring it down.
+ a.SetCurtain(true, reasonOr(st.Curtain.Reason))
+ }
+ if st.Encode != a.encodeNow() {
+ a.SetEncode(st.Encode)
+ }
+ a.checkBandwidth()
+ if st.EndsAt != nil && !st.EndsAt.After(a.opt.Now()) {
+ a.log.Info("desk stream ended at Live", "stream", a.opt.ID)
+ a.cancel()
+ }
+ return nil
+}
+
+func reasonOr(r string) string {
+ if validReason(r) {
+ return r
+ }
+ return "detector"
+}
+
+func (a *Agent) renew(ctx context.Context) {
+ p, err := a.opt.Client.Renew(ctx, a.opt.ID, a.ticketNow())
+ if err != nil {
+ a.log.Warn("desk stream ticket renewal failed; retrying in a minute", "stream", a.opt.ID, "err", err)
+ a.mu.Lock()
+ runCtx := a.ctx
+ a.mu.Unlock()
+ if runCtx != nil {
+ time.AfterFunc(time.Minute, func() {
+ if runCtx.Err() == nil {
+ a.renew(runCtx)
+ }
+ })
+ }
+ return
+ }
+ a.mu.Lock()
+ a.ticket = p.Ticket
+ if !p.ExpiresAt.IsZero() {
+ a.expiresAt = p.ExpiresAt
+ }
+ a.mu.Unlock()
+ a.pub.SetPublishKey(p.Ticket)
+ a.log.Info("desk stream ticket renewed", "stream", a.opt.ID, "expires_at", p.ExpiresAt.UTC().Format(time.RFC3339))
+}
+
+// checkBandwidth is the design's 720p rung: two WHIP retries close together mean the link is struggling, so the
+// capture restarts at 720p. It never goes back up (S2's simulcast is the real answer).
+func (a *Agent) checkBandwidth() {
+ pub := a.pub
+ if pub == nil {
+ return
+ }
+ if pub.State() != video.StateRetrying {
+ a.mu.Lock()
+ a.retries = 0
+ a.mu.Unlock()
+ return
+ }
+ a.mu.Lock()
+ if a.downshifted {
+ a.mu.Unlock()
+ return
+ }
+ a.retries++
+ down := a.retries >= 2
+ if down {
+ a.downshifted = true
+ a.opt.Encode = a.opt.Encode.LowBandwidth()
+ }
+ a.mu.Unlock()
+ if down {
+ a.log.Info("desk stream downshifted to 720p after repeated WHIP retries", "stream", a.opt.ID)
+ a.ds.restart()
+ }
+}
+
+func (a *Agent) ticketNow() credentials.Secret {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.ticket
+}
+
+func (a *Agent) encodeNow() bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.encode
+}
+
+func (a *Agent) curtainNow() (bool, []byte) {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.curtain, a.opt.CurtainJPEG
+}
+
+// SetEncode is Live's encode flag: false stops the capture and turns the stream into snapshots. The WHIP session and
+// its silent audio stay up, so Live does not see the publisher drop.
+func (a *Agent) SetEncode(on bool) {
+ a.mu.Lock()
+ if a.encode == on {
+ a.mu.Unlock()
+ return
+ }
+ a.encode = on
+ a.mu.Unlock()
+ a.ds.setEncode(on)
+ a.pub.SetPaused(!on)
+ a.log.Info("desk stream encoding changed", "stream", a.opt.ID, "encode", on)
+}
+
+// SetCurtain is the owner's own switch (the curtain command, or the page's Curtain now): it takes effect at once,
+// except that it never lowers a curtain while the plugin that owns the detectors is lost or silent (fail closed).
+func (a *Agent) SetCurtain(up bool, reason string) {
+ if !up && a.pluginLost() {
+ a.log.Warn("desk stream curtain down refused: the computer plugin is lost or silent", "stream", a.opt.ID)
+ return
+ }
+ a.applyCurtain(up, reason)
+}
+
+// ObserveCurtain is one curtain state from the desk plugin: a change, or its one-second heartbeat. A clear state only
+// lowers the curtain after the plugin has kept reporting it for CurtainClearHold, so a plugin that has just come back
+// cannot uncover the desk with its first "clear".
+func (a *Agent) ObserveCurtain(up bool, reason string) {
+ now := a.opt.Now()
+ a.mu.Lock()
+ a.lastPluginAt = now
+ // pluginDown is the supervisor's verdict, not the plugin's own word: a faulted plugin that still beats cannot
+ // clear it. Only PluginBack does.
+ if up {
+ a.clearSince = time.Time{}
+ } else if a.curtain && a.clearSince.IsZero() {
+ a.clearSince = now
+ }
+ a.mu.Unlock()
+ if up {
+ a.applyCurtain(true, reason)
+ }
+}
+
+// PluginLost is the supervisor saying the computer plugin is not ready (it exited, crashed or faulted): the picture
+// blanks and the audio mutes now, without waiting for the heartbeat to go quiet.
+func (a *Agent) PluginLost() {
+ a.mu.Lock()
+ a.pluginDown = true
+ a.clearSince = time.Time{}
+ a.mu.Unlock()
+ a.applyCurtain(true, "detector")
+}
+
+// PluginBack says the plugin is ready again. The curtain stays up until its clear states have lasted the hold.
+func (a *Agent) PluginBack() {
+ a.mu.Lock()
+ a.pluginDown = false
+ a.lastPluginAt = a.opt.Now()
+ a.mu.Unlock()
+}
+
+// PluginName is the computer plugin the stream depends on (the supervisor's state changes are matched on it).
+func (a *Agent) PluginName() string { return a.opt.Plugin }
+
+func (a *Agent) pluginLost() bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.pluginDown || a.opt.Now().Sub(a.lastPluginAt) > a.opt.CurtainLostAfter
+}
+
+// watchCurtain is the fail-closed watchdog: a plugin that is down, or silent for CurtainLostAfter, blanks the desk at
+// once; once it is back, only CurtainClearHold of clear states brings the curtain down.
+func (a *Agent) watchCurtain() {
+ now := a.opt.Now()
+ a.mu.Lock()
+ lost := a.pluginDown || now.Sub(a.lastPluginAt) > a.opt.CurtainLostAfter
+ up := a.curtain
+ clear := up && !a.clearSince.IsZero() && now.Sub(a.clearSince) >= a.opt.CurtainClearHold
+ a.mu.Unlock()
+ if lost {
+ a.mu.Lock()
+ a.clearSince = time.Time{}
+ a.mu.Unlock()
+ a.applyCurtain(true, "detector")
+ return
+ }
+ if clear {
+ a.applyCurtain(false, "detector")
+ }
+}
+
+// applyCurtain changes the picture and the audio and tells Live. It is idempotent, so the watchdog can call it every
+// tick.
+func (a *Agent) applyCurtain(up bool, reason string) {
+ a.mu.Lock()
+ if a.curtain == up {
+ if up && reason != "" && validReason(reason) && reason != a.reason {
+ a.reason = reason
+ }
+ a.mu.Unlock()
+ return
+ }
+ a.curtain = up
+ a.reason = reasonOr(reason)
+ r := a.reason
+ pub := a.pub
+ a.mu.Unlock()
+ a.ds.curtainChanged(up)
+ if pub != nil {
+ pub.SetMuted(up)
+ }
+ go func() {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if err := a.opt.Client.Curtain(ctx, a.opt.ID, a.ticketNow(), up, r); err != nil {
+ a.log.Warn("desk stream curtain report failed", "stream", a.opt.ID, "up", up, "err", err)
+ }
+ }()
+ a.log.Info("desk stream curtain", "stream", a.opt.ID, "up", up, "reason", r)
+}
+
+// Stop ends an ordinary stream: Live keeps the card, with its last frame and worked time.
+func (a *Agent) Stop() {
+ a.mu.Lock()
+ cancel := a.cancel
+ a.mu.Unlock()
+ if cancel != nil {
+ cancel()
+ }
+}
+
+// Revoke is the kill switch: Live ends the WHIP session at once and no End is sent afterwards.
+func (a *Agent) Revoke() {
+ a.mu.Lock()
+ a.revoked = true
+ a.mu.Unlock()
+ // Best effort: the caller is a latch, so it must not wait on the network.
+ go func() {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if err := a.opt.Client.Revoke(ctx, a.opt.ID, a.ticketNow()); err != nil {
+ a.log.Warn("desk stream revoke failed", "stream", a.opt.ID, "err", err)
+ }
+ }()
+ a.mu.Lock()
+ cancel := a.cancel
+ a.mu.Unlock()
+ if cancel != nil {
+ cancel()
+ }
+}
+
+func (a *Agent) finish() {
+ a.mu.Lock()
+ if a.ended || a.revoked {
+ a.mu.Unlock()
+ return
+ }
+ a.ended = true
+ a.mu.Unlock()
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if err := a.opt.Client.End(ctx, a.opt.ID, a.ticketNow()); err != nil {
+ a.log.Warn("desk stream end failed", "stream", a.opt.ID, "err", err)
+ }
+}
+
+func (a *Agent) postSnapshot(ctx context.Context) {
+ if a.opt.Snapshot == nil {
+ return
+ }
+ if !a.snapMu.TryLock() {
+ return // a slow capture must not pile snapshots up
+ }
+ defer a.snapMu.Unlock()
+ var (
+ frame []byte
+ err error
+ )
+ if up, blank := a.curtainNow(); up {
+ frame = blank
+ } else {
+ frame, err = a.opt.Snapshot(ctx)
+ }
+ if err != nil || len(frame) == 0 {
+ if err != nil {
+ a.log.Warn("desk stream snapshot failed", "stream", a.opt.ID, "err", err)
+ }
+ return
+ }
+ if err := a.opt.Client.PostFrame(ctx, a.opt.ID, a.ticketNow(), frame); err != nil {
+ a.log.Warn("desk stream snapshot post failed", "stream", a.opt.ID, "err", err)
+ }
+}
+
+// Status is what `openvibe-node status` shows for one stream.
+type Status struct {
+ ID string `json:"id"`
+ State string `json:"state"`
+ Encode bool `json:"encode"`
+ Curtain bool `json:"curtain"`
+ Viewers int `json:"viewers"`
+ Frames int64 `json:"frames"`
+ LastError string `json:"last_error,omitempty"`
+}
+
+// Status reports the stream without a ticket.
+func (a *Agent) Status() Status {
+ a.mu.Lock()
+ st := Status{ID: a.opt.ID, Encode: a.encode, Curtain: a.curtain, Viewers: a.viewers}
+ pub := a.pub
+ a.mu.Unlock()
+ if pub != nil {
+ st.State = pub.State()
+ st.Frames = pub.Frames()
+ st.LastError = pub.LastError()
+ }
+ return st
+}
+
+// ---- the source the publisher runs ----
+
+// deskSource feeds the publisher. It runs the capture only while Live wants video, and loops the pre-encoded curtain
+// while the curtain is up, so no captured frame can slip through.
+type deskSource struct {
+ a *Agent
+
+ mu sync.Mutex
+ runCtx context.Context
+ sink func(video.Sample)
+ encStop context.CancelFunc
+ encGen int
+ encRun bool
+ up bool
+}
+
+func (d *deskSource) Name() string { return "desk-stream" }
+func (d *deskSource) RequestKeyframe() { d.a.ds.restart() }
+
+func (d *deskSource) Run(ctx context.Context, sink func(video.Sample)) error {
+ d.mu.Lock()
+ d.runCtx, d.sink = ctx, sink
+ up := d.up
+ enc := d.a.encodeNow()
+ d.mu.Unlock()
+ if enc && !up {
+ d.startEncoder()
+ }
+ // While the curtain is up the publisher gets only the curtain loop; the capture keeps running so the picture is
+ // back the instant the detectors clear.
+ tick := time.NewTicker(100 * time.Millisecond)
+ defer tick.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ d.stopEncoder()
+ return nil
+ case <-tick.C:
+ if d.isUp() {
+ d.forwardCurtain()
+ }
+ }
+ }
+}
+
+func (d *deskSource) isUp() bool {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return d.up
+}
+
+func (d *deskSource) forwardCurtain() {
+ d.mu.Lock()
+ sink := d.sink
+ frames := d.a.opt.CurtainFrames
+ d.mu.Unlock()
+ if sink == nil || len(frames) == 0 {
+ return
+ }
+ d.mu.Lock()
+ f := frames[0]
+ d.mu.Unlock()
+ sink(f)
+}
+
+func (d *deskSource) startEncoder() {
+ d.mu.Lock()
+ if d.encRun || d.runCtx == nil || d.up {
+ d.mu.Unlock()
+ return
+ }
+ ctx, cancel := context.WithCancel(d.runCtx)
+ d.encGen++
+ gen := d.encGen
+ d.encRun, d.encStop = true, cancel
+ d.a.mu.Lock()
+ enc := d.a.opt.Encode
+ d.a.mu.Unlock()
+ src := d.a.opt.NewSource(d.a.opt.Capture, enc)
+ d.mu.Unlock()
+ go func() {
+ _ = src.Run(ctx, func(s video.Sample) {
+ d.mu.Lock()
+ up, out := d.up, d.sink
+ d.mu.Unlock()
+ if up || out == nil {
+ return
+ }
+ out(s)
+ })
+ d.mu.Lock()
+ if d.encGen == gen {
+ d.encRun, d.encStop = false, nil
+ }
+ d.mu.Unlock()
+ }()
+}
+
+func (d *deskSource) stopEncoder() {
+ d.mu.Lock()
+ cancel := d.encStop
+ d.encRun, d.encStop = false, nil
+ d.mu.Unlock()
+ if cancel != nil {
+ cancel()
+ }
+}
+
+func (d *deskSource) restart() {
+ d.mu.Lock()
+ was := d.encRun
+ d.mu.Unlock()
+ if !was {
+ return
+ }
+ d.stopEncoder()
+ time.Sleep(10 * time.Millisecond) // let the old ffmpeg release the display
+ d.startEncoder()
+}
+
+func (d *deskSource) setEncode(on bool) {
+ if on {
+ d.startEncoder()
+ } else {
+ d.stopEncoder()
+ }
+}
+
+func (d *deskSource) curtainChanged(up bool) {
+ d.mu.Lock()
+ d.up = up
+ d.mu.Unlock()
+ if up {
+ d.stopEncoder()
+ } else if d.a.encodeNow() {
+ d.startEncoder()
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,528 @@
@@ -0,0 +1,528 @@
+package stream
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+type fakeAPI struct {
+ mu sync.Mutex
+ state func() State
+ renews []credentials.Secret
+ newKey string
+ curtains []string
+ frames int32
+ ended int32
+ revoked int32
+}
+
+func (f *fakeAPI) State(context.Context, string, credentials.Secret) (State, error) {
+ if f.state != nil {
+ return f.state(), nil
+ }
+ return State{Encode: true, Curtain: Curtain{}}, nil
+}
+func (f *fakeAPI) Renew(_ context.Context, _ string, old credentials.Secret) (Publish, error) {
+ f.mu.Lock()
+ f.renews = append(f.renews, old)
+ f.mu.Unlock()
+ return Publish{WHIPURL: "https://whip.example/x", Ticket: credentials.NewSecret(f.newKey),
+ ExpiresAt: time.Now().Add(4 * time.Hour)}, nil
+}
+func (f *fakeAPI) Curtain(_ context.Context, _ string, _ credentials.Secret, up bool, reason string) error {
+ f.mu.Lock()
+ f.curtains = append(f.curtains, reason)
+ f.mu.Unlock()
+ return nil
+}
+func (f *fakeAPI) PostFrame(context.Context, string, credentials.Secret, []byte) error {
+ atomic.AddInt32(&f.frames, 1)
+ return nil
+}
+func (f *fakeAPI) End(context.Context, string, credentials.Secret) error {
+ atomic.AddInt32(&f.ended, 1)
+ return nil
+}
+func (f *fakeAPI) Revoke(context.Context, string, credentials.Secret) error {
+ atomic.AddInt32(&f.revoked, 1)
+ return nil
+}
+
+type fakePub struct {
+ paused atomic.Bool
+ muted atomic.Bool
+ key atomic.Value
+ state atomic.Value
+}
+
+func (p *fakePub) Run(ctx context.Context, src video.Source) {
+ go func() { _ = src.Run(ctx, func(video.Sample) {}) }()
+ <-ctx.Done()
+}
+func (p *fakePub) SetPaused(v bool) { p.paused.Store(v) }
+func (p *fakePub) SetMuted(v bool) { p.muted.Store(v) }
+func (p *fakePub) SetPublishKey(k credentials.Secret) {
+ if !k.IsZero() {
+ p.key.Store(k)
+ }
+}
+func (p *fakePub) State() string {
+ if s, _ := p.state.Load().(string); s != "" {
+ return s
+ }
+ return video.StateLive
+}
+func (p *fakePub) Frames() int64 { return 1 }
+func (p *fakePub) LastError() string { return "" }
+
+type fakeSource struct {
+ out chan string
+ once sync.Once
+}
+
+func (s *fakeSource) Name() string { return "fake" }
+func (s *fakeSource) RequestKeyframe() {}
+func (s *fakeSource) Run(ctx context.Context, sink func(video.Sample)) error {
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case v := <-s.out:
+ sink(video.Sample{Data: []byte(v), Duration: 10 * time.Millisecond})
+ }
+ }
+}
+
+func newTestAgent(t *testing.T, api *fakeAPI, o AgentOptions) (*Agent, *fakePub) {
+ t.Helper()
+ if o.ID == "" {
+ o.ID = testID
+ }
+ if o.Publish.WHIPURL == "" {
+ o.Publish = Publish{WHIPURL: "https://whip.example/whip/agent/" + testID, Ticket: credentials.NewSecret(testTicket),
+ ExpiresAt: time.Now().Add(4 * time.Hour)}
+ }
+ if o.Capture.Kind == "" {
+ o.Capture = Capture{Kind: CaptureTest, Width: 320, Height: 240}
+ }
+ if o.Log == nil {
+ o.Log = slog.New(slog.NewTextHandler(io.Discard, nil))
+ }
+ if o.Snapshot == nil {
+ o.Snapshot = func(context.Context) ([]byte, error) { return blackJPEG(64, 36), nil }
+ }
+ var pub *fakePub
+ if o.NewPublisher == nil {
+ pub = &fakePub{}
+ o.NewPublisher = func(video.Options) publisher { return pub }
+ }
+ if o.NewSource == nil {
+ o.NewSource = func(Capture, EncodeOptions) video.Source { return &fakeSource{out: make(chan string, 8)} }
+ }
+ a, err := NewAgent(o)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return a, pub
+}
+
+func TestAgentSnapshotModeWhenLiveSaysNoEncode(t *testing.T) {
+ var no atomic.Bool
+ api := &fakeAPI{}
+ api.state = func() State {
+ if no.Load() {
+ return State{Encode: false}
+ }
+ return State{Encode: true}
+ }
+ a, pub := newTestAgent(t, api, AgentOptions{Client: api, StateEvery: 10 * time.Millisecond,
+ SnapshotEvery: 15 * time.Millisecond, CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}},
+ CurtainJPEG: blackJPEG(16, 9)})
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() { a.Run(ctx); close(done) }()
+ no.Store(true)
+ deadline := time.Now().Add(5 * time.Second)
+ for a.Status().Encode {
+ if time.Now().After(deadline) {
+ t.Fatal("encode never went false")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ time.Sleep(30 * time.Millisecond)
+ before := atomic.LoadInt32(&api.frames)
+ time.Sleep(80 * time.Millisecond)
+ after := atomic.LoadInt32(&api.frames)
+ if after <= before {
+ t.Fatalf("no snapshot while encode is false (%d → %d)", before, after)
+ }
+ if !pub.paused.Load() {
+ t.Fatal("the publisher was not paused in snapshot mode")
+ }
+ cancel()
+ <-done
+ if atomic.LoadInt32(&api.ended) != 1 {
+ t.Fatalf("End called %d times", api.ended)
+ }
+}
+
+func TestAgentRenewsAtThreeHours(t *testing.T) {
+ api := &fakeAPI{newKey: testTicket[:len(testTicket)-1] + "Z"}
+ a, pub := newTestAgent(t, api, AgentOptions{Client: api, Now: func() time.Time { return time.Unix(1_800_000_000, 0) }})
+ if d := a.renewDelay(); d != 3*time.Hour {
+ t.Fatalf("renewDelay = %v, want 3h", d)
+ }
+ // A ticket that dies sooner than three hours is renewed before it does.
+ a.expiresAt = a.opt.Now().Add(1 * time.Hour)
+ if d := a.renewDelay(); d != 45*time.Minute {
+ t.Fatalf("renewDelay with a one-hour ticket = %v, want 45m", d)
+ }
+ // And the renewal replaces the key the next WHIP session reconnects with.
+ a2, pub2 := newTestAgent(t, api, AgentOptions{Client: api, RenewAfter: 20 * time.Millisecond,
+ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})
+ ctx, cancel := context.WithCancel(context.Background())
+ go a2.Run(ctx)
+ time.Sleep(80 * time.Millisecond)
+ cancel()
+ api.mu.Lock()
+ n := len(api.renews)
+ api.mu.Unlock()
+ if n == 0 {
+ t.Fatal("the ticket was never renewed")
+ }
+ got, _ := pub2.key.Load().(credentials.Secret)
+ if got.Reveal() != api.newKey {
+ t.Fatalf("publisher key after renewal = %v", got)
+ }
+ _ = pub
+}
+
+func TestDeskSourceCurtainBlanksEveryFrame(t *testing.T) {
+ api := &fakeAPI{}
+ src := &fakeSource{out: make(chan string, 8)}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api,
+ NewSource: func(Capture, EncodeOptions) video.Source { return src },
+ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true, Duration: 100 * time.Millisecond}}})
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var mu sync.Mutex
+ var seen []string
+ done := make(chan struct{})
+ go func() {
+ _ = a.ds.Run(ctx, func(s video.Sample) {
+ mu.Lock()
+ seen = append(seen, string(s.Data))
+ mu.Unlock()
+ })
+ close(done)
+ }()
+ send := func(v string) {
+ select {
+ case src.out <- v:
+ case <-time.After(time.Second):
+ t.Fatal("source blocked")
+ }
+ }
+ time.Sleep(20 * time.Millisecond)
+ send("RAW1")
+ time.Sleep(20 * time.Millisecond)
+ a.SetCurtain(true, "manual")
+ time.Sleep(20 * time.Millisecond)
+ send("RAW2")
+ time.Sleep(150 * time.Millisecond)
+ mu.Lock()
+ got := append([]string(nil), seen...)
+ mu.Unlock()
+ if !containsStr(got, "RAW1") {
+ t.Fatalf("no frame before the curtain: %v", got)
+ }
+ if containsStr(got, "RAW2") {
+ t.Fatalf("a captured frame leaked through the curtain: %v", got)
+ }
+ if !containsStr(got, "CURTAIN") {
+ t.Fatalf("the curtain frame was not sent: %v", got)
+ }
+ a.SetCurtain(false, "manual")
+ time.Sleep(30 * time.Millisecond)
+ send("RAW3")
+ time.Sleep(30 * time.Millisecond)
+ mu.Lock()
+ got = append([]string(nil), seen...)
+ mu.Unlock()
+ if !containsStr(got, "RAW3") {
+ t.Fatalf("no frame after the curtain came down: %v", got)
+ }
+ cancel()
+ <-done
+}
+
+func TestTheTicketNeverReachesALogOrStatus(t *testing.T) {
+ var buf bytes.Buffer
+ api := &fakeAPI{}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Log: slog.New(slog.NewTextHandler(&buf, nil)),
+ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() { a.Run(ctx); close(done) }()
+ time.Sleep(40 * time.Millisecond)
+ cancel()
+ <-done
+ if strings.Contains(buf.String(), "wpt_") || strings.Contains(buf.String(), testTicket) {
+ t.Fatalf("a ticket reached the log: %s", buf.String())
+ }
+ st := a.Status()
+ b, _ := json.Marshal(st)
+ if strings.Contains(string(b), "wpt_") {
+ t.Fatalf("a ticket reached status: %s", b)
+ }
+ // The Secret type is enough on its own: %v, %+v and JSON all redact it.
+ pub := Publish{WHIPURL: "https://whip.example/x", Ticket: credentials.NewSecret(testTicket)}
+ for _, s := range []string{fmt.Sprintf("%v", pub.Ticket), fmt.Sprintf("%+v", pub), fmt.Sprintf("%#v", pub.Ticket)} {
+ if strings.Contains(s, testTicket) {
+ t.Fatalf("a Secret printed its value: %s", s)
+ }
+ }
+}
+
+func TestRepeatedRetriesDownshiftTo720p(t *testing.T) {
+ api := &fakeAPI{}
+ var widths []int
+ var mu sync.Mutex
+ pub := &fakePub{}
+ pub.state.Store(video.StateRetrying)
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api, StateEvery: 5 * time.Millisecond,
+ CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}}, CurtainJPEG: blackJPEG(16, 9),
+ NewPublisher: func(video.Options) publisher { return pub },
+ NewSource: func(_ Capture, e EncodeOptions) video.Source {
+ mu.Lock()
+ widths = append(widths, e.Width)
+ mu.Unlock()
+ return &fakeSource{out: make(chan string, 1)}
+ }})
+ ctx, cancel := context.WithCancel(context.Background())
+ go a.Run(ctx)
+ time.Sleep(200 * time.Millisecond)
+ cancel()
+ mu.Lock()
+ got := append([]int(nil), widths...)
+ mu.Unlock()
+ seen720 := false
+ for _, w := range got {
+ if w == 1280 {
+ seen720 = true
+ }
+ }
+ if !seen720 {
+ t.Fatalf("no 720p restart: %v", got)
+ }
+}
+
+type testClock struct{ t time.Time }
+
+func (c *testClock) now() time.Time { return c.t }
+func (c *testClock) add(d time.Duration) { c.t = c.t.Add(d) }
+
+func waitCurtains(t *testing.T, api *fakeAPI, want int) []string {
+ t.Helper()
+ deadline := time.Now().Add(2 * time.Second)
+ for {
+ api.mu.Lock()
+ got := append([]string(nil), api.curtains...)
+ api.mu.Unlock()
+ if len(got) >= want {
+ return got
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("only %d curtain reports: %v", len(got), got)
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+}
+
+func TestPluginSilentForThreeSecondsBlanksAndMutes(t *testing.T) {
+ clock := &testClock{t: time.Unix(1_800_000_000, 0)}
+ api := &fakeAPI{}
+ pub := &fakePub{}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",
+ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,
+ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})
+ a.pub = pub
+ clock.add(3 * time.Second)
+ a.watchCurtain()
+ if !a.curtainNow2() {
+ t.Fatal("a silent plugin did not raise the curtain")
+ }
+ if !pub.muted.Load() {
+ t.Fatal("a silent plugin did not mute the audio")
+ }
+ if !a.ds.isUp() {
+ t.Fatal("the curtain gate is not up")
+ }
+ got := waitCurtains(t, api, 1)
+ if got[0] != "detector" {
+ t.Fatalf("reported reason %q, want detector", got[0])
+ }
+}
+
+func TestPluginExitRaisesAtOnceAndRefusesADown(t *testing.T) {
+ clock := &testClock{t: time.Unix(1_800_000_000, 0)}
+ api := &fakeAPI{}
+ pub := &fakePub{}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",
+ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,
+ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})
+ a.pub = pub
+ a.PluginLost()
+ if !a.curtainNow2() || !pub.muted.Load() {
+ t.Fatal("PluginLost did not blank and mute at once")
+ }
+ // The owner's Curtain now must not uncover the desk while the plugin is gone.
+ a.SetCurtain(false, "manual")
+ if !a.curtainNow2() {
+ t.Fatal("a down was accepted while the plugin is lost")
+ }
+ // Nor may the plugin's own clear heartbeat: the supervisor says it is down.
+ clock.add(3 * time.Second)
+ a.ObserveCurtain(false, "detector")
+ a.watchCurtain()
+ if !a.curtainNow2() {
+ t.Fatal("a heartbeat cleared the supervisor's verdict")
+ }
+}
+
+func TestPluginBackClearsAfterTwoSeconds(t *testing.T) {
+ clock := &testClock{t: time.Unix(1_800_000_000, 0)}
+ api := &fakeAPI{}
+ pub := &fakePub{}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",
+ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,
+ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})
+ a.pub = pub
+ a.PluginLost()
+ waitCurtains(t, api, 1)
+ clock.add(3 * time.Second)
+ a.PluginBack()
+ a.ObserveCurtain(false, "detector")
+ // The plugin keeps beating once a second; each clear report refreshes its liveness without resetting the hold.
+ clock.add(time.Second)
+ a.ObserveCurtain(false, "detector")
+ clock.add(900 * time.Millisecond)
+ a.ObserveCurtain(false, "detector")
+ a.watchCurtain()
+ if !a.curtainNow2() {
+ t.Fatal("the curtain came down before two clear seconds")
+ }
+ clock.add(150 * time.Millisecond)
+ a.ObserveCurtain(false, "detector")
+ a.watchCurtain()
+ if a.curtainNow2() {
+ t.Fatal("the curtain did not come down after two clear seconds")
+ }
+ if pub.muted.Load() {
+ t.Fatal("the audio stayed muted after the curtain came down")
+ }
+ got := waitCurtains(t, api, 2)
+ if got[len(got)-1] != "detector" {
+ t.Fatalf("last reason %q", got[len(got)-1])
+ }
+}
+
+func TestAPluginThatIsBackButReportsUpKeepsTheCurtainUp(t *testing.T) {
+ clock := &testClock{t: time.Unix(1_800_000_000, 0)}
+ api := &fakeAPI{}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api, Now: clock.now, Plugin: "computer",
+ CurtainLostAfter: 2 * time.Second, CurtainClearHold: 2 * time.Second,
+ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true}}, CurtainJPEG: blackJPEG(16, 9)})
+ a.PluginLost()
+ clock.add(5 * time.Second)
+ a.PluginBack()
+ a.ObserveCurtain(true, "password_field")
+ clock.add(5 * time.Second)
+ a.watchCurtain()
+ if !a.curtainNow2() {
+ t.Fatal("a clear hold lowered a curtain the plugin says is up")
+ }
+}
+
+func (a *Agent) curtainNow2() bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.curtain
+}
+
+func TestCurtainUpBeforeRunBlanksTheFirstFrames(t *testing.T) {
+ api := &fakeAPI{}
+ src := &fakeSource{out: make(chan string, 4)}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api,
+ NewSource: func(Capture, EncodeOptions) video.Source { return src },
+ CurtainFrames: []video.Sample{{Data: []byte("CURTAIN"), Key: true, Duration: 100 * time.Millisecond}}})
+ a.SetCurtain(true, "password_field")
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var mu sync.Mutex
+ var seen []string
+ go func() {
+ _ = a.ds.Run(ctx, func(s video.Sample) {
+ mu.Lock()
+ seen = append(seen, string(s.Data))
+ mu.Unlock()
+ })
+ }()
+ time.Sleep(20 * time.Millisecond)
+ select {
+ case src.out <- "RAW":
+ default:
+ }
+ time.Sleep(150 * time.Millisecond)
+ mu.Lock()
+ got := append([]string(nil), seen...)
+ mu.Unlock()
+ if containsStr(got, "RAW") || !containsStr(got, "CURTAIN") {
+ t.Fatalf("frames before the first poll: %v", got)
+ }
+}
+
+func TestAgentRevokeIsTheKillSwitch(t *testing.T) {
+ api := &fakeAPI{}
+ a, _ := newTestAgent(t, api, AgentOptions{Client: api, CurtainFrames: []video.Sample{{Data: []byte("C"), Key: true}},
+ CurtainJPEG: blackJPEG(16, 9)})
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ done := make(chan struct{})
+ go func() { a.Run(ctx); close(done) }()
+ time.Sleep(20 * time.Millisecond)
+ a.Revoke()
+ select {
+ case <-done:
+ case <-time.After(2 * time.Second):
+ t.Fatal("Revoke did not end the agent")
+ }
+ time.Sleep(20 * time.Millisecond)
+ if atomic.LoadInt32(&api.revoked) == 0 {
+ t.Fatal("Revoke was not sent to Live")
+ }
+ if atomic.LoadInt32(&api.ended) != 0 {
+ t.Fatal("End was sent after the kill switch")
+ }
+}
+
+func containsStr(xs []string, want string) bool {
+ for _, x := range xs {
+ if strings.Contains(x, want) {
+ return true
+ }
+ }
+ return false
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,124 @@
@@ -0,0 +1,124 @@
+package stream
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "image"
+ "image/jpeg"
+ "os/exec"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+// PrivateScreenText is what the desk shows while the curtain is up. Viewers never see the reason, only this.
+const PrivateScreenText = "Private screen"
+
+// MakeCurtain returns the picture and the pre-encoded frames the desk substitutes while its curtain is up. It uses
+// ffmpeg when it is installed (the words "Private screen" on a dark screen) and falls back to a black frame, which is
+// still a blank: the curtain never fails open.
+func MakeCurtain(ctx context.Context, ffmpeg string, w, h int) ([]video.Sample, []byte) {
+ w, h = even(w), even(h)
+ if w <= 0 || h <= 0 {
+ w, h = 1280, 720
+ }
+ if w > 1920 {
+ h = h * 1920 / w
+ w = 1920
+ }
+ if h > 1080 {
+ w = w * 1080 / h
+ h = 1080
+ }
+ w, h = even(w), even(h)
+ if ffmpeg == "" {
+ ffmpeg = "ffmpeg"
+ }
+ if path, err := exec.LookPath(ffmpeg); err == nil {
+ if frames, err := ffmpegCurtainFrames(ctx, path, w, h); err == nil && len(frames) > 0 {
+ if jpg, err := ffmpegCurtainJPEG(ctx, path, w, h); err == nil && len(jpg) > 0 {
+ return frames, jpg
+ }
+ return frames, blackJPEG(w, h)
+ }
+ if jpg, err := ffmpegCurtainJPEG(ctx, path, w, h); err == nil && len(jpg) > 0 {
+ return blackFrames(w, h), jpg
+ }
+ }
+ return blackFrames(w, h), blackJPEG(w, h)
+}
+
+func drawtext(w, h int) string {
+ size := h / 18
+ if size < 14 {
+ size = 14
+ }
+ return fmt.Sprintf("drawtext=text='%s':fontcolor=0xe8e8ee:fontsize=%d:x=(w-text_w)/2:y=(h-text_h)/2",
+ PrivateScreenText, size)
+}
+
+func ffmpegCurtainFrames(ctx context.Context, ffmpeg string, w, h int) ([]video.Sample, error) {
+ ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ args := []string{"-hide_banner", "-loglevel", "error", "-nostdin",
+ "-f", "lavfi", "-i", fmt.Sprintf("color=c=0x101014:s=%dx%d:r=10", w, h), "-t", "1",
+ "-vf", drawtext(w, h),
+ "-c:v", "libx264", "-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "main",
+ "-pix_fmt", "yuv420p", "-bf", "0", "-g", "10", "-f", "h264", "pipe:1"}
+ out, err := exec.CommandContext(ctx, ffmpeg, args...).Output()
+ if err != nil {
+ return nil, err
+ }
+ var first *video.Sample
+ err = video.AccessUnits(bytes.NewReader(out), func(au []byte, key bool) {
+ if first == nil && key {
+ cp := append([]byte(nil), au...)
+ first = &video.Sample{Data: cp, Key: true, Duration: 100 * time.Millisecond}
+ }
+ })
+ if err != nil {
+ return nil, err
+ }
+ if first == nil {
+ return nil, fmt.Errorf("no keyframe in the curtain")
+ }
+ return []video.Sample{*first}, nil
+}
+
+func ffmpegCurtainJPEG(ctx context.Context, ffmpeg string, w, h int) ([]byte, error) {
+ ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ args := []string{"-hide_banner", "-loglevel", "error", "-nostdin",
+ "-f", "lavfi", "-i", fmt.Sprintf("color=c=0x101014:s=%dx%d", w, h), "-frames:v", "1",
+ "-vf", drawtext(w, h), "-c:v", "mjpeg", "-q:v", "5", "-f", "image2pipe", "pipe:1"}
+ return exec.CommandContext(ctx, ffmpeg, args...).Output()
+}
+
+func blackFrames(w, h int) []video.Sample {
+ enc, err := video.NewPCMEncoder(w, h)
+ if err != nil {
+ return nil
+ }
+ f := video.NewFrame(w, h)
+ for i := range f.Y {
+ f.Y[i] = 16
+ }
+ for i := range f.Cb {
+ f.Cb[i], f.Cr[i] = 128, 128
+ }
+ return []video.Sample{{Data: enc.Encode(f, true, nil), Key: true, Duration: 100 * time.Millisecond}}
+}
+
+func blackJPEG(w, h int) []byte {
+ img := image.NewRGBA(image.Rect(0, 0, w, h))
+ // A dark, quiet card; the planner draws the words when ffmpeg is there.
+ for i := 0; i < len(img.Pix); i += 4 {
+ img.Pix[i], img.Pix[i+1], img.Pix[i+2], img.Pix[i+3] = 16, 16, 20, 255
+ }
+ var buf bytes.Buffer
+ if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 80}); err != nil {
+ return nil
+ }
+ return buf.Bytes()
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
+package stream
+
+import (
+ "errors"
+ "image"
+ "strconv"
+
+ "github.com/makiuchi-d/gozxing"
+ "github.com/makiuchi-d/gozxing/qrcode"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+// DecodeProbe reads a probe frame's QR payload: the millisecond clock the desk drew it with. It is the same decode the
+// headless viewer does (ZXing there; gozxing here), so the probe can be checked without a browser.
+func DecodeProbe(f *video.Frame) (int64, error) {
+ return decodeQRImage(frameGray(f))
+}
+
+func frameGray(f *video.Frame) image.Image {
+ img := image.NewGray(image.Rect(0, 0, f.W, f.H))
+ for y := 0; y < f.H; y++ {
+ copy(img.Pix[y*img.Stride:y*img.Stride+f.W], f.Y[y*f.W:(y+1)*f.W])
+ }
+ return img
+}
+
+func decodeQRImage(img image.Image) (int64, error) {
+ bmp, err := gozxing.NewBinaryBitmapFromImage(img)
+ if err != nil {
+ return 0, err
+ }
+ res, err := qrcode.NewQRCodeReader().Decode(bmp, nil)
+ if err != nil {
+ return 0, err
+ }
+ ms, err := strconv.ParseInt(res.GetText(), 10, 64)
+ if err != nil {
+ return 0, errors.New("stream: the probe QR does not hold a millisecond clock")
+ }
+ return ms, nil
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,285 @@
@@ -0,0 +1,285 @@
+package stream
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os/exec"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+// Capture kinds, one per platform the design names plus the browser desk's own surface and a synthetic source for the
+// probe and tests.
+const (
+ CaptureX11 = "x11grab"
+ CapturePipeWire = "pipewire"
+ CaptureDDAGrab = "ddagrab"
+ CaptureAVFoundation = "avfoundation"
+ CaptureBrowser = "browser"
+ CaptureTest = "test"
+)
+
+// Capture is how the core grabs the desk's picture. The computer plugin builds it (it knows the desk's display and
+// whether it is a browser desk) and the core runs ffmpeg with it.
+type Capture struct {
+ Kind string `json:"kind"`
+ Display string `json:"display,omitempty"` // x11grab/pipewire: :99
+ Pipe string `json:"pipe,omitempty"` // browser: the FIFO the page's screencast writes JPEGs into
+ Device string `json:"device,omitempty"` // avfoundation: "1:none"
+ PipeWire string `json:"pipewire,omitempty"` // pipewire: the node id, when the plugin has one
+ Width int `json:"width,omitempty"`
+ Height int `json:"height,omitempty"`
+}
+
+// EncodeOptions is the design's encoder recipe. The zero value is 1080p30, CBR 4 Mb/s, a keyframe every two seconds,
+// 0 B-frames, zerolatency and stillimage, and VFR 5–30 with duplicate frames dropped.
+type EncodeOptions struct {
+ Width, Height int
+ FPS int
+ BitrateKbps int
+ Keyframe time.Duration
+ // NoDecimate keeps every captured frame (probe and tests); the desk otherwise drops duplicates, which is what makes
+ // the stream variable-rate.
+ NoDecimate bool
+ // Still keeps x264's stillimage tuning (mostly static screens). Off for the probe, where the QR moves.
+ Still bool
+}
+
+// DefaultEncode is the recipe measured in the design.
+func DefaultEncode() EncodeOptions {
+ return EncodeOptions{Width: 1920, Height: 1080, FPS: 30, BitrateKbps: 4000, Keyframe: 2 * time.Second, Still: true}
+}
+
+// LowBandwidth is the 720p rung the desk falls to on a low bandwidth estimate.
+func (o EncodeOptions) LowBandwidth() EncodeOptions {
+ o.Width, o.Height = 1280, 720
+ return o
+}
+
+func (o EncodeOptions) withDefaults() EncodeOptions {
+ if o.FPS <= 0 {
+ o.FPS = 30
+ }
+ if o.Keyframe <= 0 {
+ o.Keyframe = 2 * time.Second
+ }
+ if o.BitrateKbps <= 0 {
+ o.BitrateKbps = 4000
+ }
+ if o.Width <= 0 || o.Height <= 0 {
+ o.Width, o.Height = 1920, 1080
+ }
+ o.Width, o.Height = even(o.Width), even(o.Height)
+ return o
+}
+
+func even(n int) int {
+ if n%2 != 0 {
+ return n - 1
+ }
+ return n
+}
+
+// Args builds the ffmpeg command line for a capture and the design's recipe. It is pure so the recipe can be checked
+// without a display or a camera.
+func Args(ffmpeg string, c Capture, o EncodeOptions) ([]string, error) {
+ if ffmpeg == "" {
+ ffmpeg = "ffmpeg"
+ }
+ o = o.withDefaults()
+ if c.Width > 0 && c.Height > 0 { // the desk's own size, unless the caller asked for a ladder rung
+ if o.Width == 1920 && o.Height == 1080 {
+ o.Width, o.Height = even(c.Width), even(c.Height)
+ }
+ }
+ args := []string{"-hide_banner", "-loglevel", "error", "-nostdin"}
+
+ var filter []string
+ switch c.Kind {
+ case CaptureX11:
+ display := c.Display
+ if display == "" {
+ return nil, errors.New("stream: x11grab needs a display (:N)")
+ }
+ if !strings.Contains(display, ".") {
+ display += ".0" // the plugin may already name the screen (":90.0")
+ }
+ args = append(args, "-f", "x11grab", "-draw_mouse", "1", "-framerate", strconv.Itoa(o.FPS),
+ "-video_size", fmt.Sprintf("%dx%d", o.Width, o.Height), "-i", display)
+ case CapturePipeWire:
+ // ffmpeg 7.1's pipewire input; the plugin only chooses it when ffmpeg reports the device.
+ node := c.PipeWire
+ if node == "" {
+ return nil, errors.New("stream: pipewire needs a node id")
+ }
+ args = append(args, "-f", "pipewire", "-framerate", strconv.Itoa(o.FPS), "-i", node)
+ case CaptureDDAGrab:
+ args = append(args, "-f", "lavfi", "-i",
+ fmt.Sprintf("ddagrab=output_idx=0:framerate=%d:video_size=%dx%d", o.FPS, o.Width, o.Height))
+ filter = append(filter, "hwdownload", "format=bgra")
+ case CaptureAVFoundation:
+ device := c.Device
+ if device == "" {
+ device = "1:none"
+ }
+ args = append(args, "-f", "avfoundation", "-capture_cursor", "1", "-framerate", strconv.Itoa(o.FPS), "-i", device)
+ case CaptureBrowser:
+ if c.Pipe == "" {
+ return nil, errors.New("stream: a browser desk needs its screencast pipe")
+ }
+ args = append(args, "-f", "image2pipe", "-framerate", strconv.Itoa(o.FPS), "-i", c.Pipe)
+ case CaptureTest:
+ args = append(args, "-f", "lavfi", "-i", fmt.Sprintf("testsrc2=size=%dx%d:rate=%d", o.Width, o.Height, o.FPS))
+ default:
+ return nil, fmt.Errorf("stream: unknown capture %q", c.Kind)
+ }
+
+ // Variable rate: drop frames that did not change. -fps_mode vfr keeps the capture's timestamps, so a still screen
+ // costs about 5 frames a second and a moving one up to 30. The source's timing is the only clock; nothing is
+ // duplicated to a fixed rate.
+ if !o.NoDecimate {
+ filter = append(filter, "mpdecimate=hi=64*12:lo=64*5:frac=0.33")
+ }
+ if len(filter) > 0 {
+ args = append(args, "-vf", strings.Join(filter, ","))
+ }
+ // The cap, not a constant rate: -fps_mode vfr keeps the capture's own timestamps and -r would contradict it.
+ args = append(args, "-fps_mode", "vfr")
+
+ // H.264 main, no B-frames, zerolatency, keyframe every two seconds of wall clock even when frames were dropped.
+ args = append(args,
+ "-c:v", "libx264",
+ "-preset", "ultrafast",
+ "-tune", tune(o.Still),
+ "-profile:v", "main",
+ "-pix_fmt", "yuv420p",
+ "-bf", "0",
+ "-g", strconv.Itoa(o.FPS*int(o.Keyframe.Seconds())),
+ "-keyint_min", strconv.Itoa(o.FPS*int(o.Keyframe.Seconds())),
+ "-sc_threshold", "0",
+ "-force_key_frames", fmt.Sprintf("expr:gte(t,n_forced*%g)", o.Keyframe.Seconds()),
+ "-b:v", fmt.Sprintf("%dk", o.BitrateKbps),
+ "-maxrate", fmt.Sprintf("%dk", o.BitrateKbps),
+ "-bufsize", fmt.Sprintf("%dk", o.BitrateKbps),
+ "-x264-params", "nal-hrd=cbr",
+ "-f", "h264", "pipe:1",
+ )
+ return args, nil
+}
+
+func tune(still bool) string {
+ if still {
+ return "zerolatency,stillimage"
+ }
+ return "zerolatency"
+}
+
+// FFmpegSource runs one ffmpeg capture+encode pipeline and hands its Annex B access units to the publisher. It restarts
+// the child with backoff when it exits, like a camera source does.
+type FFmpegSource struct {
+ FFmpeg string
+ Capture Capture
+ Options EncodeOptions
+ Log *slog.Logger
+ keyReq atomic.Bool
+}
+
+func (s *FFmpegSource) Name() string { return "ffmpeg:" + s.Capture.Kind }
+func (s *FFmpegSource) RequestKeyframe() { s.keyReq.Store(true) }
+
+// Run restarts ffmpeg until ctx ends. A keyframe request cannot reach an already-running ffmpeg; the two-second
+// force_key_frames expression covers a new viewer, and a restart covers the rest.
+func (s *FFmpegSource) Run(ctx context.Context, sink func(video.Sample)) error {
+ backoff := time.Second
+ for ctx.Err() == nil {
+ start := time.Now()
+ err := s.runOnce(ctx, sink)
+ if ctx.Err() != nil {
+ return nil
+ }
+ if time.Since(start) > time.Minute {
+ backoff = time.Second
+ }
+ if s.Log != nil {
+ s.Log.Warn("stream capture exited; restarting", "capture", s.Capture.Kind, "err", err, "in", backoff)
+ }
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-time.After(backoff):
+ }
+ if backoff < 30*time.Second {
+ backoff *= 2
+ }
+ }
+ return nil
+}
+
+func (s *FFmpegSource) runOnce(ctx context.Context, sink func(video.Sample)) error {
+ ff := s.FFmpeg
+ if ff == "" {
+ ff = "ffmpeg"
+ }
+ args, err := Args(ff, s.Capture, s.Options)
+ if err != nil {
+ return err
+ }
+ cmd := exec.CommandContext(ctx, ff, args...)
+ var stderr strings.Builder
+ cmd.Stderr = &limitedWriter{w: &stderr, n: 4096}
+ out, err := cmd.StdoutPipe()
+ if err != nil {
+ return err
+ }
+ if err := cmd.Start(); err != nil {
+ return err
+ }
+ last := time.Now()
+ perr := video.AccessUnits(out, func(au []byte, key bool) {
+ now := time.Now()
+ d := now.Sub(last)
+ if d < time.Millisecond {
+ d = time.Millisecond
+ }
+ if d > time.Second {
+ d = time.Second
+ }
+ last = now
+ sink(video.Sample{Data: au, Key: key, Duration: d})
+ })
+ werr := cmd.Wait()
+ if perr != nil {
+ return perr
+ }
+ if werr != nil {
+ return fmt.Errorf("%w: %s", werr, strings.TrimSpace(stderr.String()))
+ }
+ return errors.New("ffmpeg exited")
+}
+
+type limitedWriter struct {
+ w interface{ Write([]byte) (int, error) }
+ n int
+}
+
+func (l *limitedWriter) Write(p []byte) (int, error) {
+ if l.n <= 0 {
+ return len(p), nil
+ }
+ q := p
+ if len(q) > l.n {
+ q = q[:l.n]
+ }
+ l.n -= len(q)
+ _, _ = l.w.Write(q)
+ return len(p), nil
+}
+
+// probeCapture is the source the latency probe draws its QR clock on.
+func probeCapture(w, h int) Capture { return Capture{Kind: CaptureTest, Width: w, Height: h} }
new file mode 100644
new file mode 100644
@@ -0,0 +1,163 @@
@@ -0,0 +1,163 @@
+package stream
+
+import (
+ "context"
+ "os/exec"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+func hasArg(args []string, want string) bool {
+ for i := 0; i < len(args); i++ {
+ if args[i] == want {
+ return true
+ }
+ }
+ return false
+}
+
+func argAfter(args []string, flag string) string {
+ out := ""
+ for i := 0; i+1 < len(args); i++ {
+ if args[i] == flag {
+ out = args[i+1]
+ }
+ }
+ return out
+}
+
+func TestEncoderRecipe(t *testing.T) {
+ args, err := Args("ffmpeg", Capture{Kind: CaptureX11, Display: ":99", Width: 1600, Height: 900}, DefaultEncode())
+ if err != nil {
+ t.Fatal(err)
+ }
+ checks := map[string]string{
+ "-bf": "0",
+ "-profile:v": "main",
+ "-preset": "ultrafast",
+ "-tune": "zerolatency,stillimage",
+ "-g": "60",
+ "-keyint_min": "60",
+ "-force_key_frames": "expr:gte(t,n_forced*2)",
+ "-b:v": "4000k",
+ "-maxrate": "4000k",
+ "-bufsize": "4000k",
+ "-fps_mode": "vfr",
+ "-x264-params": "nal-hrd=cbr",
+ "-f": "h264",
+ }
+ for flag, want := range checks {
+ if got := argAfter(args, flag); got != want {
+ t.Errorf("%s = %q, want %q (args %v)", flag, got, want, args)
+ }
+ }
+ if !strings.Contains(argAfter(args, "-vf"), "mpdecimate") {
+ t.Errorf("no duplicate-frame dropping: %v", args)
+ }
+ if got := argAfter(args, "-video_size"); got != "1600x900" {
+ t.Errorf("capture size = %q", got)
+ }
+ if !hasArg(args, "pipe:1") {
+ t.Errorf("no Annex B output: %v", args)
+ }
+}
+
+func TestEncoderLowBandwidthRung(t *testing.T) {
+ args, err := Args("ffmpeg", Capture{Kind: CaptureX11, Display: ":99", Width: 1920, Height: 1080},
+ DefaultEncode().LowBandwidth())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := argAfter(args, "-video_size"); got != "1280x720" {
+ t.Errorf("720p rung = %q", got)
+ }
+}
+
+func TestEachCaptureKind(t *testing.T) {
+ cases := []Capture{
+ {Kind: CaptureX11, Display: ":97", Width: 800, Height: 600},
+ {Kind: CaptureX11, Display: ":97.0", Width: 800, Height: 600},
+ {Kind: CapturePipeWire, PipeWire: "42", Width: 800, Height: 600},
+ {Kind: CaptureDDAGrab, Width: 800, Height: 600},
+ {Kind: CaptureAVFoundation, Device: "2:none"},
+ {Kind: CaptureBrowser, Pipe: "/run/ov/page.fifo"},
+ {Kind: CaptureTest, Width: 800, Height: 600},
+ }
+ for _, c := range cases {
+ args, err := Args("ffmpeg", c, EncodeOptions{Width: 800, Height: 600, FPS: 30})
+ if err != nil {
+ t.Errorf("%s: %v", c.Kind, err)
+ continue
+ }
+ joined := strings.Join(args, " ")
+ switch c.Kind {
+ case CaptureX11:
+ if !strings.Contains(joined, "-f x11grab") || !strings.Contains(joined, ":97.0") || strings.Contains(joined, ":97.0.0") {
+ t.Errorf("x11grab args: %s", joined)
+ }
+ case CapturePipeWire:
+ if !strings.Contains(joined, "-f pipewire") || !strings.Contains(joined, " 42 ") {
+ t.Errorf("pipewire args: %s", joined)
+ }
+ case CaptureDDAGrab:
+ if !strings.Contains(joined, "ddagrab=output_idx=0") || !strings.Contains(argAfter(args, "-vf"), "hwdownload") {
+ t.Errorf("ddagrab args: %s", joined)
+ }
+ case CaptureAVFoundation:
+ if !strings.Contains(joined, "-f avfoundation") || !strings.Contains(joined, "2:none") {
+ t.Errorf("avfoundation args: %s", joined)
+ }
+ case CaptureBrowser:
+ if !strings.Contains(joined, "-f image2pipe") || !strings.Contains(joined, "/run/ov/page.fifo") {
+ t.Errorf("browser args: %s", joined)
+ }
+ case CaptureTest:
+ if !strings.Contains(joined, "testsrc2=") {
+ t.Errorf("test args: %s", joined)
+ }
+ }
+ }
+}
+
+func TestCaptureNeedsItsInput(t *testing.T) {
+ if _, err := Args("ffmpeg", Capture{Kind: CaptureX11}, DefaultEncode()); err == nil {
+ t.Error("x11grab without a display was accepted")
+ }
+ if _, err := Args("ffmpeg", Capture{Kind: CaptureBrowser}, DefaultEncode()); err == nil {
+ t.Error("a browser desk without a pipe was accepted")
+ }
+ if _, err := Args("ffmpeg", Capture{Kind: "nope"}, DefaultEncode()); err == nil {
+ t.Error("an unknown capture was accepted")
+ }
+}
+
+func TestFFmpegSourcePublishesAccessUnits(t *testing.T) {
+ ff, err := exec.LookPath("ffmpeg")
+ if err != nil {
+ t.Skip("no ffmpeg on this machine")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+ src := &FFmpegSource{FFmpeg: ff, Capture: Capture{Kind: CaptureTest, Width: 320, Height: 240},
+ Options: EncodeOptions{Width: 320, Height: 240, FPS: 15, Keyframe: time.Second}}
+ samples := make(chan video.Sample, 8)
+ go func() {
+ _ = src.Run(ctx, func(s video.Sample) {
+ select {
+ case samples <- s:
+ default:
+ }
+ })
+ }()
+ select {
+ case s := <-samples:
+ if len(s.Data) == 0 || s.Duration <= 0 {
+ t.Fatalf("empty sample: %+v", s)
+ }
+ case <-time.After(10 * time.Second):
+ t.Fatal("no access units from ffmpeg")
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,403 @@
@@ -0,0 +1,403 @@
+// Package stream is Node's part of Agent Streams S1: the desk stream. It publishes a desk's screen to OpenVibe.Live
+// over WHIP with a publish ticket (wpt_…), polls the stream's state, sends snapshots while nobody watches, renews the
+// ticket at three hours, blanks the picture behind the curtain, and measures glass-to-glass latency with a QR probe.
+//
+// The design is plan-state-2026-10-22/AGENT-STREAMS-S1-DESIGN-2026-11-06.md. Tickets and tokens are credentials.Secret:
+// they go on the wire and nowhere else, never into a log line, an error or `openvibe-node status`.
+package stream
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+)
+
+// DefaultLiveURL is where the agent-stream API lives.
+const DefaultLiveURL = "https://openvibe.live"
+
+// ErrTicketInvalid is Live refusing a wpt_ (401 invalid_ticket): it is wrong, expired, revoked or names another stream.
+var ErrTicketInvalid = errors.New("the publish ticket was refused (invalid_ticket)")
+
+// ErrAlreadyPublishing is a second WHIP session for a stream that already has one (409 already_publishing).
+var ErrAlreadyPublishing = errors.New("this stream already has a WHIP session (already_publishing)")
+
+var (
+ idRe = regexp.MustCompile(`^ags_[0-9A-HJKMNP-TV-Z]{26}$`)
+ ticketRe = regexp.MustCompile(`wpt_[A-Za-z0-9_-]{43}`)
+ viewTickRe = regexp.MustCompile(`wvt_[A-Za-z0-9_-]{43}`)
+ bearerAnyRe = regexp.MustCompile(`(?i)bearer\s+[A-Za-z0-9._~+/-]{8,}`)
+)
+
+// Task is what the agent is doing and where that task lives (live.agent-stream@1 task).
+type Task struct {
+ Source string `json:"source"` // codes, actor, desk or node
+ Ref string `json:"ref"`
+ Title string `json:"title"`
+ URL string `json:"url,omitempty"`
+}
+
+// CreateRequest is the body of POST /api/streams/agent.
+type CreateRequest struct {
+ Title string `json:"title"`
+ Task Task `json:"task"`
+ Visibility string `json:"visibility"`
+ AllowSites []string `json:"allow_sites,omitempty"`
+}
+
+// Stream is live.agent-stream@1, the fields Node needs.
+type Stream struct {
+ ID string `json:"id"`
+ Owner string `json:"owner"`
+ Channel string `json:"channel"`
+ Title string `json:"title"`
+ Task Task `json:"task"`
+ Visibility string `json:"visibility"`
+ Mode string `json:"mode"`
+ State string `json:"state"`
+ Viewers int `json:"viewers"`
+ WorkedSeconds int64 `json:"worked_seconds"`
+ PageURL string `json:"page_url"`
+ Label string `json:"label"`
+ StartedAt *string `json:"started_at"`
+ EndedAt *string `json:"ended_at"`
+}
+
+// Publish is live.agent-stream-publish@1: where the desk publishes and with what.
+type Publish struct {
+ WHIPURL string `json:"whip_url"`
+ Ticket credentials.Secret `json:"-"`
+ ExpiresAt time.Time `json:"-"`
+}
+
+// publishWire is Publish on the wire; the ticket is read once and wrapped in a Secret.
+type publishWire struct {
+ WHIPURL string `json:"whip_url"`
+ Ticket string `json:"ticket"`
+ ExpiresAt string `json:"expires_at"`
+}
+
+// Curtain is the desk's privacy screen as Live records it.
+type Curtain struct {
+ Up bool `json:"up"`
+ Since *time.Time `json:"since,omitempty"`
+ Reason string `json:"reason,omitempty"`
+}
+
+// State is live.agent-stream-state@1, what the desk reads every ten seconds.
+type State struct {
+ Viewers int `json:"viewers"`
+ Encode bool `json:"encode"`
+ Curtain Curtain `json:"curtain"`
+ EndsAt *time.Time `json:"ends_at"`
+}
+
+// CreateResult is the 201 answer of POST /api/streams/agent.
+type CreateResult struct {
+ Stream Stream `json:"stream"`
+ Publish Publish `json:"publish"`
+}
+
+// Client talks to Live's agent-stream API. Token buys the node token that POST /api/streams/agent authenticates with;
+// every other call uses the stream's publish ticket.
+type Client struct {
+ base string
+ token func(context.Context) (credentials.Secret, error)
+ http *http.Client
+ ua string
+}
+
+// ClientOptions configure a Client.
+type ClientOptions struct {
+ BaseURL string
+ Token func(context.Context) (credentials.Secret, error)
+ HTTPClient *http.Client
+ UserAgent string
+}
+
+// NewClient returns a client for base (DefaultLiveURL when empty). Plain http is accepted only for a loopback host,
+// which is what tests use.
+func NewClient(o ClientOptions) (*Client, error) {
+ base := strings.TrimSuffix(o.BaseURL, "/")
+ if base == "" {
+ base = DefaultLiveURL
+ }
+ u, err := url.Parse(base)
+ if err != nil || u.Host == "" || u.User != nil {
+ return nil, errors.New("stream: the Live URL is not an origin")
+ }
+ if u.Scheme != "https" && !(u.Scheme == "http" && isLoopback(u.Hostname())) {
+ return nil, errors.New("stream: the Live URL must be https")
+ }
+ if o.Token == nil {
+ return nil, errors.New("stream: no node token source (pair this machine with OpenVibe.Network)")
+ }
+ c := &Client{base: base, token: o.Token, http: o.HTTPClient, ua: o.UserAgent}
+ if c.http == nil {
+ c.http = &http.Client{Timeout: 20 * time.Second}
+ }
+ if c.ua == "" {
+ c.ua = "openvibe-node"
+ }
+ return c, nil
+}
+
+func isLoopback(host string) bool {
+ if host == "localhost" {
+ return true
+ }
+ ip := net.ParseIP(host)
+ return ip != nil && ip.IsLoopback()
+}
+
+// Create calls POST /api/streams/agent with the machine's node token (audience openvibe.live). It is the only call
+// that carries a token; the answer's ticket is stored in a Secret and never logged.
+func (c *Client) Create(ctx context.Context, req CreateRequest) (CreateResult, error) {
+ var out CreateResult
+ if err := validateCreate(req); err != nil {
+ return out, err
+ }
+ tok, err := c.token(ctx)
+ if err != nil {
+ return out, err
+ }
+ var wire struct {
+ Stream Stream `json:"stream"`
+ Publish publishWire `json:"publish"`
+ }
+ if err := c.do(ctx, http.MethodPost, "/api/streams/agent", tok, req, &wire); err != nil {
+ return out, err
+ }
+ pub, err := parsePublish(wire.Publish)
+ if err != nil {
+ return out, err
+ }
+ out.Stream, out.Publish = wire.Stream, pub
+ return out, nil
+}
+
+// State calls GET /api/streams/agent/:id/state with the ticket.
+func (c *Client) State(ctx context.Context, id string, ticket credentials.Secret) (State, error) {
+ var out State
+ if err := checkID(id); err != nil {
+ return out, err
+ }
+ err := c.do(ctx, http.MethodGet, "/api/streams/agent/"+id+"/state", ticket, nil, &out)
+ return out, err
+}
+
+// Renew calls POST /api/streams/agent/:id/ticket with the ticket and returns the new one. The desk renews at 3 h.
+func (c *Client) Renew(ctx context.Context, id string, ticket credentials.Secret) (Publish, error) {
+ var out Publish
+ if err := checkID(id); err != nil {
+ return out, err
+ }
+ var wire publishWire
+ if err := c.do(ctx, http.MethodPost, "/api/streams/agent/"+id+"/ticket", ticket, nil, &wire); err != nil {
+ return out, err
+ }
+ return parsePublish(wire)
+}
+
+// Revoke calls DELETE /api/streams/agent/:id/ticket: the kill switch. Live ends the WHIP session at once.
+func (c *Client) Revoke(ctx context.Context, id string, ticket credentials.Secret) error {
+ if err := checkID(id); err != nil {
+ return err
+ }
+ return c.do(ctx, http.MethodDelete, "/api/streams/agent/"+id+"/ticket", ticket, nil, nil)
+}
+
+// Curtain reports the desk's privacy screen: POST /api/streams/agent/:id/curtain {up, reason}.
+func (c *Client) Curtain(ctx context.Context, id string, ticket credentials.Secret, up bool, reason string) error {
+ if err := checkID(id); err != nil {
+ return err
+ }
+ if !validReason(reason) {
+ return fmt.Errorf("stream: curtain reason %q is not one of password_field, secret_file, env_print, manual, site, detector", reason)
+ }
+ return c.do(ctx, http.MethodPost, "/api/streams/agent/"+id+"/curtain", ticket,
+ map[string]any{"up": up, "reason": reason}, nil)
+}
+
+// PostFrame sends a snapshot (JPEG or WebP, at most 200 KB): PUT /api/streams/agent/:id/frame.
+func (c *Client) PostFrame(ctx context.Context, id string, ticket credentials.Secret, frame []byte) error {
+ if err := checkID(id); err != nil {
+ return err
+ }
+ if len(frame) == 0 || len(frame) > MaxFrameBytes {
+ return fmt.Errorf("stream: a snapshot is 1 to %d bytes (got %d)", MaxFrameBytes, len(frame))
+ }
+ typ := http.DetectContentType(frame)
+ if typ != "image/jpeg" && typ != "image/webp" {
+ return fmt.Errorf("stream: a snapshot must be JPEG or WebP (got %s)", typ)
+ }
+ return c.doRaw(ctx, http.MethodPut, "/api/streams/agent/"+id+"/frame", ticket, typ, frame, nil)
+}
+
+// End calls POST /api/streams/agent/:id/end.
+func (c *Client) End(ctx context.Context, id string, ticket credentials.Secret) error {
+ if err := checkID(id); err != nil {
+ return err
+ }
+ return c.do(ctx, http.MethodPost, "/api/streams/agent/"+id+"/end", ticket, nil, nil)
+}
+
+// MaxFrameBytes is the snapshot cap in the design.
+const MaxFrameBytes = 200 * 1024
+
+func validateCreate(r CreateRequest) error {
+ if strings.TrimSpace(r.Title) == "" || len(r.Title) > 200 {
+ return errors.New("stream: title is 1 to 200 characters")
+ }
+ switch r.Task.Source {
+ case "codes", "actor", "desk", "node":
+ default:
+ return errors.New("stream: task.source is codes, actor, desk or node")
+ }
+ if r.Task.Ref == "" || len(r.Task.Ref) > 200 || r.Task.Title == "" || len(r.Task.Title) > 200 {
+ return errors.New("stream: task.ref and task.title are required (at most 200 characters)")
+ }
+ if r.Task.URL != "" {
+ if u, err := url.Parse(r.Task.URL); err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
+ return errors.New("stream: task.url must be an http(s) URL")
+ }
+ }
+ switch r.Visibility {
+ case "private", "unlisted", "public":
+ default:
+ return errors.New("stream: visibility is private, unlisted or public")
+ }
+ if len(r.AllowSites) > 50 {
+ return errors.New("stream: allow_sites carries at most 50 hosts")
+ }
+ for _, s := range r.AllowSites {
+ if u, err := url.Parse(s); err != nil || u.Host == "" {
+ return fmt.Errorf("stream: allow_sites entry %q is not a URL", s)
+ }
+ }
+ return nil
+}
+
+func checkID(id string) error {
+ if !idRe.MatchString(id) {
+ return fmt.Errorf("stream: %q is not an agent-stream id (ags_…)", id)
+ }
+ return nil
+}
+
+func validReason(r string) bool {
+ switch r {
+ case "password_field", "secret_file", "env_print", "manual", "site", "detector":
+ return true
+ }
+ return false
+}
+
+func parsePublish(w publishWire) (Publish, error) {
+ var out Publish
+ if w.WHIPURL == "" || w.Ticket == "" || w.ExpiresAt == "" {
+ return out, errors.New("stream: the publish part is missing whip_url, ticket or expires_at")
+ }
+ u, err := url.Parse(w.WHIPURL)
+ if err != nil || u.Host == "" || u.User != nil {
+ return out, errors.New("stream: the publish whip_url is not a URL")
+ }
+ if !ticketRe.MatchString(w.Ticket) || !strings.HasPrefix(w.Ticket, "wpt_") {
+ return out, errors.New("stream: the publish ticket is not a wpt_…")
+ }
+ exp, err := time.Parse(time.RFC3339, w.ExpiresAt)
+ if err != nil {
+ return out, errors.New("stream: expires_at is not RFC 3339")
+ }
+ out.WHIPURL, out.Ticket, out.ExpiresAt = w.WHIPURL, credentials.NewSecret(w.Ticket), exp
+ return out, nil
+}
+
+func (c *Client) do(ctx context.Context, method, path string, auth credentials.Secret, body any, out any) error {
+ var raw []byte
+ typ := ""
+ if body != nil {
+ b, err := json.Marshal(body)
+ if err != nil {
+ return err
+ }
+ raw, typ = b, "application/json"
+ }
+ return c.doRaw(ctx, method, path, auth, typ, raw, out)
+}
+
+func (c *Client) doRaw(ctx context.Context, method, path string, auth credentials.Secret, typ string, body []byte, out any) error {
+ var rdr io.Reader
+ if body != nil {
+ rdr = bytes.NewReader(body)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr)
+ if err != nil {
+ return err
+ }
+ if typ != "" {
+ req.Header.Set("Content-Type", typ)
+ }
+ // A ticket is sent only to the configured Live origin, never after a redirect to another host.
+ req.Header.Set("User-Agent", c.ua)
+ if !auth.IsZero() {
+ req.Header.Set("Authorization", "Bearer "+auth.Reveal())
+ }
+ resp, err := c.http.Do(req)
+ if err != nil {
+ return errors.New(redact(err.Error()))
+ }
+ defer resp.Body.Close()
+ raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+ if resp.StatusCode/100 != 2 {
+ var pe struct {
+ Code string `json:"code"`
+ Error string `json:"error"`
+ Detail string `json:"detail"`
+ }
+ _ = json.Unmarshal(raw, &pe)
+ code := pe.Code
+ if code == "" {
+ code = pe.Error
+ }
+ switch {
+ case resp.StatusCode == http.StatusUnauthorized && (code == "invalid_ticket" || code == ""):
+ return ErrTicketInvalid
+ case resp.StatusCode == http.StatusConflict && code == "already_publishing":
+ return ErrAlreadyPublishing
+ }
+ msg := strings.TrimSpace(code + " " + pe.Detail)
+ if msg == "" {
+ msg = strings.TrimSpace(string(raw))
+ if len(msg) > 200 {
+ msg = msg[:200]
+ }
+ }
+ return fmt.Errorf("stream: Live answered HTTP %d %s", resp.StatusCode, redact(msg))
+ }
+ if out != nil {
+ if err := json.Unmarshal(raw, out); err != nil {
+ return errors.New("stream: Live's answer is not valid JSON")
+ }
+ }
+ return nil
+}
+
+// redact removes anything that looks like a publish or view ticket, and any bearer value, from a message that may be
+// logged or returned in an error.
+func redact(s string) string {
+ s = ticketRe.ReplaceAllString(s, "[ticket]")
+ s = viewTickRe.ReplaceAllString(s, "[view-ticket]")
+ s = bearerAnyRe.ReplaceAllString(s, "Bearer [redacted]")
+ return s
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,193 @@
@@ -0,0 +1,193 @@
+package stream
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+)
+
+const (
+ testTicket = "wpt_0123456789abcdefghijklmnopqrstuvwxyzABCDEFG"
+ testID = "ags_01J8Z4M2Q0R7T9YV3K6N8P1W2X"
+)
+
+type recorded struct {
+ method, path, auth, ctype string
+ body []byte
+}
+
+func liveFake(t *testing.T, h func(w http.ResponseWriter, r *http.Request, rec recorded)) (*Client, *[]recorded) {
+ t.Helper()
+ var mu sync.Mutex
+ var got []recorded
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ b, _ := io.ReadAll(r.Body)
+ rec := recorded{method: r.Method, path: r.URL.Path, auth: r.Header.Get("Authorization"), ctype: r.Header.Get("Content-Type"), body: b}
+ mu.Lock()
+ got = append(got, rec)
+ mu.Unlock()
+ h(w, r, rec)
+ }))
+ t.Cleanup(srv.Close)
+ c, err := NewClient(ClientOptions{BaseURL: srv.URL, Token: func(context.Context) (credentials.Secret, error) {
+ return credentials.NewSecret("node-token"), nil
+ }, UserAgent: "test"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return c, &got
+}
+
+func TestCreateParsesPublishAndNeverLogsTheTicket(t *testing.T) {
+ c, got := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {
+ if rec.method != "POST" || rec.path != "/api/streams/agent" {
+ t.Errorf("request %s %s", rec.method, rec.path)
+ }
+ if rec.auth != "Bearer node-token" {
+ t.Errorf("create auth = %q", rec.auth)
+ }
+ var req CreateRequest
+ if err := json.Unmarshal(rec.body, &req); err != nil {
+ t.Error(err)
+ }
+ if req.Title != "fix the tests" || req.Task.Source != "codes" || req.Visibility != "private" {
+ t.Errorf("create body = %+v", req)
+ }
+ w.WriteHeader(http.StatusCreated)
+ io.WriteString(w, `{"stream":{"id":"`+testID+`","title":"fix the tests","state":"live"},
+ "publish":{"whip_url":"https://whip.openvibe.live/whip/agent/`+testID+`","ticket":"`+testTicket+`","expires_at":"2026-11-06T16:00:00Z"}}`)
+ })
+ res, err := c.Create(context.Background(), CreateRequest{Title: "fix the tests", Visibility: "private",
+ Task: Task{Source: "codes", Ref: "job_1", Title: "fix"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.Stream.ID != testID || res.Publish.WHIPURL == "" {
+ t.Fatalf("create = %+v", res)
+ }
+ if res.Publish.Ticket.Reveal() != testTicket {
+ t.Fatalf("ticket = %v", res.Publish.Ticket)
+ }
+ if s := res.Publish.Ticket.String(); strings.Contains(s, "wpt_") {
+ t.Fatalf("Secret prints its value: %q", s)
+ }
+ // The ticket appears only in the wire answer; nothing this package hands out in an error or status does.
+ b, _ := json.Marshal(res)
+ if strings.Contains(string(b), "wpt_") {
+ t.Fatalf("the ticket reached JSON: %s", b)
+ }
+ _ = got
+}
+
+func TestTicketCallsUseTheTicketAsBearer(t *testing.T) {
+ c, got := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {
+ switch {
+ case strings.HasSuffix(rec.path, "/state"):
+ io.WriteString(w, `{"viewers":2,"encode":true,"curtain":{"up":false},"ends_at":"2026-11-06T16:00:00Z"}`)
+ case rec.method == "POST" && strings.HasSuffix(rec.path, "/ticket"):
+ io.WriteString(w, `{"whip_url":"https://whip.openvibe.live/whip/agent/`+testID+`","ticket":"`+testTicket+`","expires_at":"2026-11-06T17:00:00Z"}`)
+ case rec.method == "DELETE":
+ w.WriteHeader(http.StatusNoContent)
+ case strings.HasSuffix(rec.path, "/curtain"):
+ if string(rec.body) != `{"reason":"manual","up":true}` {
+ t.Errorf("curtain body = %s", rec.body)
+ }
+ w.WriteHeader(http.StatusNoContent)
+ case strings.HasSuffix(rec.path, "/frame"):
+ if rec.ctype != "image/jpeg" || len(rec.body) < 4 {
+ t.Errorf("frame = %q %d bytes", rec.ctype, len(rec.body))
+ }
+ w.WriteHeader(http.StatusNoContent)
+ case strings.HasSuffix(rec.path, "/end"):
+ w.WriteHeader(http.StatusNoContent)
+ }
+ })
+ ctx := context.Background()
+ tick := credentials.NewSecret(testTicket)
+ if st, err := c.State(ctx, testID, tick); err != nil || st.Viewers != 2 || !st.Encode {
+ t.Fatalf("state = %+v %v", st, err)
+ }
+ if p, err := c.Renew(ctx, testID, tick); err != nil || p.Ticket.Reveal() != testTicket {
+ t.Fatalf("renew = %+v %v", p, err)
+ }
+ if err := c.Revoke(ctx, testID, tick); err != nil {
+ t.Fatal(err)
+ }
+ if err := c.Curtain(ctx, testID, tick, true, "manual"); err != nil {
+ t.Fatal(err)
+ }
+ jpeg := []byte{0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0}
+ if err := c.PostFrame(ctx, testID, tick, jpeg); err != nil {
+ t.Fatal(err)
+ }
+ if err := c.End(ctx, testID, tick); err != nil {
+ t.Fatal(err)
+ }
+ for _, r := range *got {
+ if r.path == "/api/streams/agent" {
+ continue
+ }
+ if r.auth != "Bearer "+testTicket {
+ t.Errorf("%s %s auth = %q", r.method, r.path, r.auth)
+ }
+ }
+}
+
+func TestTicketErrors(t *testing.T) {
+ c, _ := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {
+ switch {
+ case strings.HasSuffix(rec.path, "/state"):
+ w.WriteHeader(http.StatusUnauthorized)
+ io.WriteString(w, `{"code":"invalid_ticket"}`)
+ case strings.HasSuffix(rec.path, "/end"):
+ w.WriteHeader(http.StatusConflict)
+ io.WriteString(w, `{"code":"already_publishing"}`)
+ case strings.HasSuffix(rec.path, "/frame"):
+ w.WriteHeader(http.StatusBadRequest)
+ io.WriteString(w, `{"code":"invalid_frame","detail":"ticket `+testTicket+`"}`)
+ }
+ })
+ ctx := context.Background()
+ tick := credentials.NewSecret(testTicket)
+ if _, err := c.State(ctx, testID, tick); !errors.Is(err, ErrTicketInvalid) {
+ t.Fatalf("state: %v", err)
+ }
+ if err := c.End(ctx, testID, tick); !errors.Is(err, ErrAlreadyPublishing) {
+ t.Fatalf("end: %v", err)
+ }
+ err := c.PostFrame(ctx, testID, tick, []byte{0xff, 0xd8, 0xff, 0xe0})
+ if err == nil || strings.Contains(err.Error(), "wpt_") {
+ t.Fatalf("frame error leaks a ticket: %v", err)
+ }
+}
+
+func TestCreateValidation(t *testing.T) {
+ c, _ := liveFake(t, func(w http.ResponseWriter, r *http.Request, rec recorded) {
+ t.Error("invalid request reached Live")
+ })
+ bad := []CreateRequest{
+ {Title: "", Task: Task{Source: "codes", Ref: "r", Title: "t"}, Visibility: "private"},
+ {Title: "t", Task: Task{Source: "other", Ref: "r", Title: "t"}, Visibility: "private"},
+ {Title: "t", Task: Task{Source: "codes", Ref: "r", Title: "t"}, Visibility: "friends"},
+ {Title: "t", Task: Task{Source: "codes", Ref: "r", Title: "t"}, Visibility: "private", AllowSites: []string{"not a url"}},
+ }
+ for i, r := range bad {
+ if _, err := c.Create(context.Background(), r); err == nil {
+ t.Errorf("case %d accepted", i)
+ }
+ }
+ if err := c.Curtain(context.Background(), testID, credentials.NewSecret(testTicket), true, "because"); err == nil {
+ t.Error("an unknown curtain reason was accepted")
+ }
+ if err := c.PostFrame(context.Background(), testID, credentials.NewSecret(testTicket), make([]byte, MaxFrameBytes+1)); err == nil {
+ t.Error("an oversized frame was accepted")
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,219 @@
@@ -0,0 +1,219 @@
+package stream
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/url"
+ "strings"
+ "sync"
+ "time"
+
+ qrcode "github.com/skip2/go-qrcode"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+// Probe QR: each frame carries the millisecond clock the desk drew it. The headless viewer decodes it and subtracts
+// the instant it was decoded, which is the glass-to-glass latency.
+const (
+ ProbeDefaultSeconds = 60
+ ProbeDefaultWidth = 640
+ ProbeDefaultHeight = 360
+ ProbeDefaultFPS = 10
+)
+
+// ProbeOptions configure `openvibe-node stream --probe`.
+type ProbeOptions struct {
+ WHIPURL string
+ Ticket credentials.Secret
+ Seconds int
+ Width int
+ Height int
+ FPS int
+ Loopback bool
+ Log *slog.Logger
+ Now func() time.Time
+ // NewPublisher is for tests.
+ NewPublisher func(video.Options) publisher
+}
+
+// ProbeSource draws a QR of the current millisecond clock in the top-left corner of every frame, on the built-in PCM
+// encoder (no ffmpeg, no display). It is the clock the viewer reads.
+type ProbeSource struct {
+ W, H, FPS int
+ Now func() time.Time
+ mu sync.Mutex
+ lastMS int64
+ keyReq bool
+}
+
+func (p *ProbeSource) Name() string { return "probe" }
+
+func (p *ProbeSource) RequestKeyframe() {
+ p.mu.Lock()
+ p.keyReq = true
+ p.mu.Unlock()
+}
+
+// LastMS is the clock value of the last frame drawn.
+func (p *ProbeSource) LastMS() int64 {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.lastMS
+}
+
+func (p *ProbeSource) Run(ctx context.Context, sink func(video.Sample)) error {
+ if p.W <= 0 || p.H <= 0 {
+ p.W, p.H = ProbeDefaultWidth, ProbeDefaultHeight
+ }
+ if p.FPS <= 0 {
+ p.FPS = ProbeDefaultFPS
+ }
+ if p.Now == nil {
+ p.Now = time.Now
+ }
+ w, h := mbSize(p.W), mbSize(p.H)
+ enc, err := video.NewPCMEncoder(w, h)
+ if err != nil {
+ return err
+ }
+ frame := time.Second / time.Duration(p.FPS)
+ tick := time.NewTicker(frame)
+ defer tick.Stop()
+ var prev *video.Frame
+ for n := 0; ; n++ {
+ now := p.Now()
+ f := video.NewFrame(w, h)
+ drawProbe(f, now.UnixMilli())
+ p.mu.Lock()
+ p.lastMS = now.UnixMilli()
+ key := prev == nil || n%10 == 0 || p.keyReq
+ p.keyReq = false
+ p.mu.Unlock()
+ var coded []int
+ if !key {
+ coded = video.ChangedMBs(prev, f, 4)
+ }
+ sink(video.Sample{Data: enc.Encode(f, key, coded), Key: key, Duration: frame})
+ prev = f
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-tick.C:
+ }
+ }
+}
+
+// mbSize rounds a picture to the PCM encoder's macroblock grid.
+func mbSize(n int) int {
+ if n < 16 {
+ return 16
+ }
+ return n / 16 * 16
+}
+
+// drawProbe paints the QR on white with a quiet zone, plus a moving marker so the encoder sees motion.
+func drawProbe(f *video.Frame, ms int64) {
+ for i := range f.Y {
+ f.Y[i] = 235
+ }
+ for i := range f.Cb {
+ f.Cb[i], f.Cr[i] = 128, 128
+ }
+ q, err := qrcode.New(fmt.Sprintf("%d", ms), qrcode.Medium)
+ if err != nil {
+ return
+ }
+ bmp := q.Bitmap()
+ // Four modules of quiet zone, then whole-pixel modules. At 640×360 the code is about 200 px square.
+ scale := min(f.W, f.H) / (len(bmp) + 8)
+ if scale < 2 {
+ scale = 2
+ }
+ size := len(bmp) * scale
+ x0, y0 := 8, 8
+ for y := 0; y < size && y0+y < f.H; y++ {
+ for x := 0; x < size && x0+x < f.W; x++ {
+ if bmp[y/scale][x/scale] {
+ f.Y[(y0+y)*f.W+(x0+x)] = 16
+ }
+ }
+ }
+ // A marker sweeping along the bottom shows the stream is moving (and keeps VFR from dropping every frame).
+ pos := int(ms/100) % (f.W - 32)
+ for y := f.H - 24; y < f.H-8; y++ {
+ for x := pos; x < pos+32; x++ {
+ f.Y[y*f.W+x] = 16
+ }
+ }
+}
+
+// Probe publishes the QR clock until seconds elapse (or ctx ends) and reports the frames written.
+func Probe(ctx context.Context, o ProbeOptions) (int64, error) {
+ if o.Seconds <= 0 {
+ o.Seconds = ProbeDefaultSeconds
+ }
+ if o.Width <= 0 {
+ o.Width = ProbeDefaultWidth
+ }
+ if o.Height <= 0 {
+ o.Height = ProbeDefaultHeight
+ }
+ if o.FPS <= 0 {
+ o.FPS = ProbeDefaultFPS
+ }
+ if o.Now == nil {
+ o.Now = time.Now
+ }
+ if o.Log == nil {
+ o.Log = slog.Default()
+ }
+ if err := checkProbeURL(o.WHIPURL); err != nil {
+ return 0, err
+ }
+ if o.Ticket.IsZero() || !strings.HasPrefix(o.Ticket.Reveal(), "wpt_") {
+ return 0, errors.New("stream: --probe needs the stream's publish ticket (--ticket wpt_…)")
+ }
+ opts := video.Options{WHIPURL: o.WHIPURL, PublishKey: o.Ticket, Audio: true,
+ UserAgent: "openvibe-node/stream-probe", IncludeLoopback: o.Loopback, Log: o.Log}
+ src := &ProbeSource{W: o.Width, H: o.Height, FPS: o.FPS, Now: o.Now}
+ ctx, cancel := context.WithTimeout(ctx, time.Duration(o.Seconds)*time.Second)
+ defer cancel()
+ var pub publisher
+ if o.NewPublisher != nil {
+ pub = o.NewPublisher(opts)
+ } else {
+ pub = video.NewPublisher(opts)
+ }
+ done := make(chan struct{})
+ go func() { pub.Run(ctx, src); close(done) }()
+ start := time.Now()
+ tick := time.NewTicker(5 * time.Second)
+ defer tick.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ <-done
+ return pub.Frames(), nil
+ case <-tick.C:
+ o.Log.Info("stream probe publishing", "elapsed", time.Since(start).Round(time.Second).String())
+ }
+ }
+}
+
+func checkProbeURL(raw string) error {
+ if raw == "" {
+ return errors.New("stream: --probe needs a WHIP URL (--whip-url)")
+ }
+ u, err := url.Parse(raw)
+ if err != nil || u.Host == "" || u.User != nil {
+ return errors.New("stream: --whip-url is not a URL")
+ }
+ if u.Scheme != "https" && !(u.Scheme == "http" && isLoopback(u.Hostname())) {
+ return errors.New("stream: --whip-url must be https")
+ }
+ return nil
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,97 @@
@@ -0,0 +1,97 @@
+package stream
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/credentials"
+ "github.com/OpenVibers/OpenVibe.Node/internal/video"
+)
+
+func TestProbeSourceQRRoundTrips(t *testing.T) {
+ want := int64(1_800_000_000_123)
+ src := &ProbeSource{W: 640, H: 360, FPS: 10, Now: func() time.Time { return time.UnixMilli(want) }}
+ got := make(chan video.Sample, 2)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go func() { _ = src.Run(ctx, func(s video.Sample) { got <- s }) }()
+ select {
+ case s := <-got:
+ f := video.NewFrame(640, 360)
+ // The PCM encoder wrote an IDR of the frame; decode the source's own drawing straight from a fresh frame.
+ _ = s
+ drawProbe(f, want)
+ ms, err := DecodeProbe(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ms != want {
+ t.Fatalf("decoded %d, want %d", ms, want)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("no probe frame")
+ }
+}
+
+func TestProbeSourceClockAdvances(t *testing.T) {
+ base := time.UnixMilli(1_800_000_000_000)
+ i := 0
+ src := &ProbeSource{W: 640, H: 360, FPS: 20, Now: func() time.Time {
+ v := base.Add(time.Duration(i) * 50 * time.Millisecond)
+ i++
+ return v
+ }}
+ samples := make(chan video.Sample, 4)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go func() { _ = src.Run(ctx, func(s video.Sample) { samples <- s }) }()
+ for n := 0; n < 3; n++ {
+ select {
+ case <-samples:
+ case <-time.After(2 * time.Second):
+ t.Fatal("no probe frame")
+ }
+ }
+ if src.LastMS() < base.UnixMilli()+50 {
+ t.Fatalf("the clock did not advance: %d", src.LastMS())
+ }
+}
+
+type countingPub struct{ frames int64 }
+
+func (c *countingPub) Run(ctx context.Context, s video.Source) {
+ _ = s.Run(ctx, func(video.Sample) { c.frames++ })
+}
+func (c *countingPub) SetPaused(bool) {}
+func (c *countingPub) SetMuted(bool) {}
+func (c *countingPub) SetPublishKey(credentials.Secret) {}
+func (c *countingPub) State() string { return video.StateLive }
+func (c *countingPub) Frames() int64 { return c.frames }
+func (c *countingPub) LastError() string { return "" }
+
+func TestProbePublishesUntilTimeIsUp(t *testing.T) {
+ pub := &countingPub{}
+ n, err := Probe(context.Background(), ProbeOptions{
+ WHIPURL: "http://127.0.0.1:1/whip/agent/" + testID,
+ Ticket: credentials.NewSecret(testTicket),
+ Seconds: 1, Width: 320, Height: 180, FPS: 20,
+ NewPublisher: func(video.Options) publisher { return pub },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n < 5 {
+ t.Fatalf("probe published %d frames in a second", n)
+ }
+}
+
+func TestProbeNeedsAURLAndATicket(t *testing.T) {
+ if _, err := Probe(context.Background(), ProbeOptions{Seconds: 1, NewPublisher: func(video.Options) publisher { return &countingPub{} }}); err == nil {
+ t.Error("no URL accepted")
+ }
+ if _, err := Probe(context.Background(), ProbeOptions{WHIPURL: "https://whip.openvibe.live/whip/agent/" + testID,
+ Seconds: 1, NewPublisher: func(video.Options) publisher { return &countingPub{} }}); err == nil {
+ t.Error("no ticket accepted")
+ }
+}
@@ -37,6 +37,11 @@ type Options struct {
@@ -37,6 +37,11 @@ type Options struct {
// IncludeLoopback offers loopback ICE candidates (tests on one machine).
// IncludeLoopback offers loopback ICE candidates (tests on one machine).
IncludeLoopback bool
IncludeLoopback bool
ConnectTimeout time.Duration
ConnectTimeout time.Duration
+ // Audio adds a silent Opus track (Agent Streams S1): S4's voice then needs no renegotiation. The packets are the
+ // 20 ms silence frame from pion's own e2e test, sent whether or not video is flowing.
+ Audio bool
+ // UserAgent names the Node on the WHIP request.
+ UserAgent string
}
}
// States reported by Publisher.State.
// States reported by Publisher.State.
@@ -59,6 +64,9 @@ type Publisher struct {
@@ -59,6 +64,9 @@ type Publisher struct {
track *webrtc.TrackLocalStaticSample
track *webrtc.TrackLocalStaticSample
needK bool
needK bool
src Source
src Source
+ pause atomic.Bool
+ mute atomic.Bool
+ key atomic.Value // credentials.Secret: the publish ticket, replaceable by renewal
}
}
func NewPublisher(opt Options) *Publisher {
func NewPublisher(opt Options) *Publisher {
@@ -74,17 +82,35 @@ func NewPublisher(opt Options) *Publisher {
@@ -74,17 +82,35 @@ func NewPublisher(opt Options) *Publisher {
p := &Publisher{opt: opt, log: opt.Log.With("component", "video")}
p := &Publisher{opt: opt, log: opt.Log.With("component", "video")}
p.state.Store(StateOff)
p.state.Store(StateOff)
p.lastEr.Store("")
p.lastEr.Store("")
+ p.key.Store(opt.PublishKey)
return p
return p
}
}
// State is off, connecting, live or retrying.
// State is off, connecting, live or retrying.
func (p *Publisher) State() string { return p.state.Load().(string) }
func (p *Publisher) State() string { return p.state.Load().(string) }
-// Frames counts samples written to a live session.
+// Frames counts video samples written to a live session.
func (p *Publisher) Frames() int64 { return p.frames.Load() }
func (p *Publisher) Frames() int64 { return p.frames.Load() }
func (p *Publisher) LastError() string { return p.lastEr.Load().(string) }
func (p *Publisher) LastError() string { return p.lastEr.Load().(string) }
+// SetPaused stops sending video samples while keeping the WHIP session and its audio track up. Snapshot mode uses it:
+// Live asked for no encoding, but the stream must not look dropped.
+func (p *Publisher) SetPaused(paused bool) { p.pause.Store(paused) }
+
+// SetMuted stops the audio track's packets. The curtain calls it: a desk whose screen is blanked must not carry the
+// desk's sound either (S1's track is silent, but S4's voice uses the same switch).
+func (p *Publisher) SetMuted(muted bool) { p.mute.Store(muted) }
+
+// SetPublishKey replaces the ticket a renewed stream reconnects with. It is never logged.
+func (p *Publisher) SetPublishKey(k credentials.Secret) {
+ if !k.IsZero() {
+ p.key.Store(k)
+ }
+}
+
+func (p *Publisher) publishKey() credentials.Secret { return p.key.Load().(credentials.Secret) }
+
// Run publishes src until ctx ends, reconnecting with backoff.
// Run publishes src until ctx ends, reconnecting with backoff.
func (p *Publisher) Run(ctx context.Context, src Source) {
func (p *Publisher) Run(ctx context.Context, src Source) {
p.mu.Lock()
p.mu.Lock()
@@ -107,7 +133,7 @@ func (p *Publisher) Run(ctx context.Context, src Source) {
@@ -107,7 +133,7 @@ func (p *Publisher) Run(ctx context.Context, src Source) {
}
}
msg := "ended"
msg := "ended"
if err != nil {
if err != nil {
- msg = scrub(err.Error(), p.opt.PublishKey)
+ msg = scrub(err.Error(), p.publishKey())
}
}
p.lastEr.Store(msg)
p.lastEr.Store(msg)
p.state.Store(StateRetrying)
p.state.Store(StateRetrying)
@@ -135,7 +161,7 @@ func (p *Publisher) write(s Sample) {
@@ -135,7 +161,7 @@ func (p *Publisher) write(s Sample) {
needK = false
needK = false
}
}
p.mu.Unlock()
p.mu.Unlock()
- if track == nil || needK {
+ if track == nil || needK || p.pause.Load() {
return
return
}
}
if err := track.WriteSample(media.Sample{Data: s.Data, Duration: s.Duration}); err == nil {
if err := track.WriteSample(media.Sample{Data: s.Data, Duration: s.Duration}); err == nil {
@@ -143,6 +169,25 @@ func (p *Publisher) write(s Sample) {
@@ -143,6 +169,25 @@ func (p *Publisher) write(s Sample) {
}
}
}
}
+// OpusSilence is one 20 ms silence frame (pion/webrtc e2e_test.go's silentOpusFrame: 8 kHz mono, 20 ms).
+var OpusSilence = []byte{0xf8, 0xff, 0xfe}
+
+func (p *Publisher) sendSilence(ctx context.Context, track *webrtc.TrackLocalStaticSample) {
+ tick := time.NewTicker(20 * time.Millisecond)
+ defer tick.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-tick.C:
+ if p.mute.Load() {
+ continue
+ }
+ _ = track.WriteSample(media.Sample{Data: OpusSilence, Duration: 20 * time.Millisecond})
+ }
+ }
+}
+
func (p *Publisher) requestKeyframe() {
func (p *Publisher) requestKeyframe() {
p.mu.Lock()
p.mu.Lock()
src := p.src
src := p.src
@@ -162,6 +207,15 @@ func (p *Publisher) newAPI() (*webrtc.API, error) {
@@ -162,6 +207,15 @@ func (p *Publisher) newAPI() (*webrtc.API, error) {
}, webrtc.RTPCodecTypeVideo); err != nil {
}, webrtc.RTPCodecTypeVideo); err != nil {
return nil, err
return nil, err
}
}
+ if p.opt.Audio {
+ if err := m.RegisterCodec(webrtc.RTPCodecParameters{
+ RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2,
+ SDPFmtpLine: "minptime=10;useinbandfec=1"},
+ PayloadType: 111,
+ }, webrtc.RTPCodecTypeAudio); err != nil {
+ return nil, err
+ }
+ }
ir := &interceptor.Registry{}
ir := &interceptor.Registry{}
if err := webrtc.RegisterDefaultInterceptors(m, ir); err != nil {
if err := webrtc.RegisterDefaultInterceptors(m, ir); err != nil {
return nil, err
return nil, err
@@ -198,6 +252,18 @@ func (p *Publisher) session(ctx context.Context) error {
@@ -198,6 +252,18 @@ func (p *Publisher) session(ctx context.Context) error {
if err != nil {
if err != nil {
return err
return err
}
}
+ var audio *webrtc.TrackLocalStaticSample
+ var silenceOnce sync.Once
+ if p.opt.Audio {
+ audio, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus,
+ ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"}, "audio", "openvibe-node")
+ if err != nil {
+ return err
+ }
+ if _, err := pc.AddTransceiverFromTrack(audio, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}); err != nil {
+ return err
+ }
+ }
go func() { // RTCP: a viewer's PLI or FIR asks for a keyframe.
go func() { // RTCP: a viewer's PLI or FIR asks for a keyframe.
for {
for {
pkts, _, err := tr.Sender().ReadRTCP()
pkts, _, err := tr.Sender().ReadRTCP()
@@ -267,8 +333,11 @@ func (p *Publisher) session(ctx context.Context) error {
@@ -267,8 +333,11 @@ func (p *Publisher) session(ctx context.Context) error {
p.track, p.needK = track, true
p.track, p.needK = track, true
p.mu.Unlock()
p.mu.Unlock()
p.requestKeyframe()
p.requestKeyframe()
+ if audio != nil {
+ silenceOnce.Do(func() { go p.sendSilence(ctx, audio) })
+ }
p.state.Store(StateLive)
p.state.Store(StateLive)
- p.log.Info("video live", "whip", scrub(redactURL(p.opt.WHIPURL), p.opt.PublishKey))
+ p.log.Info("video live", "whip", scrub(redactURL(p.opt.WHIPURL), p.publishKey()))
case webrtc.PeerConnectionStateDisconnected:
case webrtc.PeerConnectionStateDisconnected:
disconnectedAt = time.Now()
disconnectedAt = time.Now()
case webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed:
case webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed:
@@ -288,17 +357,21 @@ func (p *Publisher) post(ctx context.Context, sdp string) (answer, resource stri
@@ -288,17 +357,21 @@ func (p *Publisher) post(ctx context.Context, sdp string) (answer, resource stri
return "", "", err
return "", "", err
}
}
req.Header.Set("Content-Type", "application/sdp")
req.Header.Set("Content-Type", "application/sdp")
- if !p.opt.PublishKey.IsZero() {
- req.Header.Set("Authorization", "Bearer "+p.opt.PublishKey.Reveal())
+ key := p.publishKey()
+ if !key.IsZero() {
+ req.Header.Set("Authorization", "Bearer "+key.Reveal())
+ }
+ if p.opt.UserAgent != "" {
+ req.Header.Set("User-Agent", p.opt.UserAgent)
}
}
resp, err := p.opt.HTTPClient.Do(req)
resp, err := p.opt.HTTPClient.Do(req)
if err != nil {
if err != nil {
- return "", "", fmt.Errorf("whip: %s", scrub(err.Error(), p.opt.PublishKey))
+ return "", "", fmt.Errorf("whip: %s", scrub(err.Error(), key))
}
}
defer resp.Body.Close()
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
- return "", "", fmt.Errorf("whip: HTTP %d %s", resp.StatusCode, scrub(string(bytes.TrimSpace(body[:min(len(body), 200)])), p.opt.PublishKey))
+ return "", "", fmt.Errorf("whip: HTTP %d %s", resp.StatusCode, scrub(string(bytes.TrimSpace(body[:min(len(body), 200)])), key))
}
}
loc := resp.Header.Get("Location")
loc := resp.Header.Get("Location")
if loc != "" {
if loc != "" {
@@ -321,9 +394,10 @@ func (p *Publisher) delete(resource string) {
@@ -321,9 +394,10 @@ func (p *Publisher) delete(resource string) {
if err != nil {
if err != nil {
return
return
}
}
- // The publish key goes only to the WHIP endpoint's own origin, whatever Location the server returned.
- if !p.opt.PublishKey.IsZero() && sameOrigin(resource, p.opt.WHIPURL) {
- req.Header.Set("Authorization", "Bearer "+p.opt.PublishKey.Reveal())
+ // The ticket goes only to the WHIP endpoint's own origin, whatever Location the server returned.
+ key := p.publishKey()
+ if !key.IsZero() && sameOrigin(resource, p.opt.WHIPURL) {
+ req.Header.Set("Authorization", "Bearer "+key.Reveal())
}
}
if resp, err := p.opt.HTTPClient.Do(req); err == nil {
if resp, err := p.opt.HTTPClient.Do(req); err == nil {
resp.Body.Close()
resp.Body.Close()
@@ -23,7 +23,7 @@ func TestPublishTestPatternOverWHIP(t *testing.T) {
@@ -23,7 +23,7 @@ func TestPublishTestPatternOverWHIP(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
done := make(chan struct{})
go func() { p.Run(ctx, &TestPattern{W: 160, H: 96, FPS: 20}); close(done) }()
go func() { p.Run(ctx, &TestPattern{W: 160, H: 96, FPS: 20}); close(done) }()
- deadline := time.Now().Add(15 * time.Second)
+ deadline := time.Now().Add(40 * time.Second)
for rx.Packets.Load() < 20 {
for rx.Packets.Load() < 20 {
if time.Now().After(deadline) {
if time.Now().After(deadline) {
t.Fatalf("no media: state %s frames %d packets %d err %q", p.State(), p.Frames(), rx.Packets.Load(), p.LastError())
t.Fatalf("no media: state %s frames %d packets %d err %q", p.State(), p.Frames(), rx.Packets.Load(), p.LastError())
@@ -43,6 +43,52 @@ func TestPublishTestPatternOverWHIP(t *testing.T) {
@@ -43,6 +43,52 @@ func TestPublishTestPatternOverWHIP(t *testing.T) {
}
}
}
}
+func TestPublishWithSilentAudio(t *testing.T) {
+ rx := &WHIPReceiver{PublishKey: "pk_audio"}
+ srv := httptest.NewServer(rx)
+ defer srv.Close()
+ defer rx.Close()
+ p := NewPublisher(Options{WHIPURL: srv.URL + "/whip/agent/ags_x", PublishKey: credentials.NewSecret("pk_audio"),
+ Audio: true, IncludeLoopback: true, Log: slog.New(slog.NewTextHandler(io.Discard, nil))})
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() { p.Run(ctx, &TestPattern{W: 160, H: 96, FPS: 20}); close(done) }()
+ deadline := time.Now().Add(40 * time.Second)
+ for rx.AudioPackets.Load() < 5 || p.Frames() == 0 {
+ if time.Now().After(deadline) {
+ t.Fatalf("no audio+video: state %s frames %d audio %d err %q", p.State(), p.Frames(), rx.AudioPackets.Load(), p.LastError())
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ video := p.Frames()
+ p.SetPaused(true)
+ time.Sleep(500 * time.Millisecond)
+ if grew := p.Frames() - video; grew > 2 {
+ t.Fatalf("paused video kept flowing: %d frames", grew)
+ }
+ if rx.AudioPackets.Load() < 5 {
+ t.Fatal("audio stopped")
+ }
+ // The curtain mutes the track: no audio packets while muted, and they resume when it comes down.
+ p.SetMuted(true)
+ time.Sleep(300 * time.Millisecond)
+ muted := rx.AudioPackets.Load()
+ time.Sleep(500 * time.Millisecond)
+ if grew := rx.AudioPackets.Load() - muted; grew > 2 {
+ t.Fatalf("muted audio kept flowing: %d packets", grew)
+ }
+ p.SetMuted(false)
+ deadline = time.Now().Add(3 * time.Second)
+ for rx.AudioPackets.Load() <= muted+2 {
+ if time.Now().After(deadline) {
+ t.Fatal("audio did not resume after unmute")
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ cancel()
+ <-done
+}
+
func TestPublishRetriesOnBadKey(t *testing.T) {
func TestPublishRetriesOnBadKey(t *testing.T) {
rx := &WHIPReceiver{PublishKey: "right"}
rx := &WHIPReceiver{PublishKey: "right"}
srv := httptest.NewServer(rx)
srv := httptest.NewServer(rx)
@@ -59,11 +105,18 @@ func TestPublishRetriesOnBadKey(t *testing.T) {
@@ -59,11 +105,18 @@ func TestPublishRetriesOnBadKey(t *testing.T) {
}
}
time.Sleep(10 * time.Millisecond)
time.Sleep(10 * time.Millisecond)
}
}
+ // A renewed ticket replaces the key the next session reconnects with.
+ rx.SetPublishKey("right")
+ p.SetPublishKey(credentials.NewSecret("right"))
+ deadline = time.Now().Add(10 * time.Second)
+ for p.State() != StateLive {
+ if time.Now().After(deadline) {
+ t.Fatalf("renewed key never published: state %s err %q", p.State(), p.LastError())
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
cancel()
cancel()
<-done
<-done
- if e := p.LastError(); e == "" || contains(e, "wrong-key-value") {
- t.Fatalf("last error %q", e)
- }
}
}
// The receiver's key can change while it serves (a fake Bot rotating the publish key): under -race this is the check
// The receiver's key can change while it serves (a fake Bot rotating the publish key): under -race this is the check
@@ -17,9 +17,10 @@ import (
@@ -17,9 +17,10 @@ import (
type WHIPReceiver struct {
type WHIPReceiver struct {
PublishKey string
PublishKey string
- Packets atomic.Int64
- Deletes atomic.Int64
- BadAuth atomic.Int64
+ Packets atomic.Int64
+ AudioPackets atomic.Int64
+ Deletes atomic.Int64
+ BadAuth atomic.Int64
mu sync.Mutex
mu sync.Mutex
pcs map[string]*webrtc.PeerConnection
pcs map[string]*webrtc.PeerConnection
@@ -78,11 +79,15 @@ func (w *WHIPReceiver) post(rw http.ResponseWriter, r *http.Request) {
@@ -78,11 +79,15 @@ func (w *WHIPReceiver) post(rw http.ResponseWriter, r *http.Request) {
return
return
}
}
pc.OnTrack(func(t *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {
pc.OnTrack(func(t *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {
+ audio := t.Kind() == webrtc.RTPCodecTypeAudio
for {
for {
if _, _, err := t.ReadRTP(); err != nil {
if _, _, err := t.ReadRTP(); err != nil {
return
return
}
}
w.Packets.Add(1)
w.Packets.Add(1)
+ if audio {
+ w.AudioPackets.Add(1)
+ }
}
}
})
})
if err := pc.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: string(offer)}); err != nil {
if err := pc.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: string(offer)}); err != nil {
@@ -69,6 +69,14 @@ Command kinds and their answers (the ack's ``data``):
@@ -69,6 +69,14 @@ Command kinds and their answers (the ack's ``data``):
against ``max_desks``. ``describe`` lists the kinds this machine runs (``desk.kinds``). Starting and stopping act
against ``max_desks``. ``describe`` lists the kinds this machine runs (``desk.kinds``). Starting and stopping act
(refused at observe, approved at ask), ``status`` runs at every level, and the kill switch stops the desk. Whole desks
(refused at observe, approved at ask), ``status`` runs at every level, and the kill switch stops the desk. Whole desks
need Linux with Xvfb, browser desks a Chromium-family browser; config ``allow_desk: false`` turns both off.
need Linux with Xvfb, browser desks a Chromium-family browser; config ``allow_desk: false`` turns both off.
+* ``desk.stream`` ``{"action": "start"|"stop"|"status"|"curtain", "desk"?, "allow_sites"?, "up"?, "reason"?}``
+ (Agent Streams S1, ``stream.py`` and ``curtain.py``): the agent's own stream on OpenVibe.Live. ``start`` answers
+ ``{streaming, sid, capture, curtain, detectors}`` with the capture source the core's ffmpeg should use (x11grab,
+ PipeWire, ddagrab, AVFoundation, or a browser desk's page through a FIFO); the core mints the stream and the ``wpt_``
+ ticket on Live, so neither reaches this plugin. The privacy curtain's detectors (password field, secret-named
+ window/file, an environment print, the hotkey, the browser's allow-list) run here and report ``curtain`` events;
+ a detector that fails or is unsure puts the curtain up and never down. Config ``allow_desk_stream: false`` turns it
+ off. The core publishes over WHIP and snapshots while nobody watches.
* **A desk host for other people** (``desk_isolation: "sandbox"``, Node v0.32, Desks plan D5): every desk is a browser
* **A desk host for other people** (``desk_isolation: "sandbox"``, Node v0.32, Desks plan D5): every desk is a browser
desk whose browser runs under ``openvibe-node desk-sandbox`` (the jobs' sandbox: its own namespaces and cgroup as
desk whose browser runs under ``openvibe-node desk-sandbox`` (the jobs' sandbox: its own namespaces and cgroup as
``worker.run_as``, ``desk_memory_mb``, ``desk_cpu_millis``, ``desk_disk_mb`` and ``desk_pids`` each, only the public
``worker.run_as``, ``desk_memory_mb``, ``desk_cpu_millis``, ``desk_disk_mb`` and ``desk_pids`` each, only the public
@@ -108,10 +116,16 @@ import urllib.parse
@@ -108,10 +116,16 @@ import urllib.parse
from openvibe_plugin import Deferred, Fault, Plugin
from openvibe_plugin import Deferred, Fault, Plugin
from .browser import KEY_CODES, Browser, BrowserError, find_binary
from .browser import KEY_CODES, Browser, BrowserError, find_binary
+from .curtain import (Curtain, EnvPrintDetector, Hotkey, PasswordFieldDetector, SecretFileDetector, SiteDetector,
+ _active_window, install_sigusr1)
from .desk import MAX_DESKS_CAP, BrowserDesk, Desks, PrivateDesk, default_max_desks
from .desk import MAX_DESKS_CAP, BrowserDesk, Desks, PrivateDesk, default_max_desks
from .egress import EgressProxy, not_public
from .egress import EgressProxy, not_public
+from .stream import PageScreencast, capture_kinds, desktop_capture
from .files import Files, Handles
from .files import Files, Handles
+# The desk stream's curtain heartbeat: the core declares the curtain lost if it hears nothing for two seconds, so this
+# must stay well under that (and the test patches it to run fast).
+DESK_STREAM_HEARTBEAT_S = 1.0
LEVELS = ("observe", "ask", "trusted", "full")
LEVELS = ("observe", "ask", "trusted", "full")
RANK = {name: i for i, name in enumerate(LEVELS)}
RANK = {name: i for i, name in enumerate(LEVELS)}
READ_OPS = frozenset({"list", "stat", "read"})
READ_OPS = frozenset({"list", "stat", "read"})
@@ -366,6 +380,14 @@ class Computer(Plugin):
@@ -366,6 +380,14 @@ class Computer(Plugin):
self.allow_screen_stream = True
self.allow_screen_stream = True
self.stream = None # the live screen: {"sid", "until", "fps", "tool", "dir", "stop": Event}
self.stream = None # the live screen: {"sid", "until", "fps", "tool", "dir", "stop": Event}
self.stream_mu = threading.Lock()
self.stream_mu = threading.Lock()
+ # The agent's desk stream (Agent Streams S1): capture, the curtain's detectors and the hotkey.
+ self.allow_desk_stream = True
+ self.desk_stream = None
+ self.desk_stream_mu = threading.Lock()
+ self.desk_stream_allow_sites = []
+ self.desk_stream_hotkey = "ctrl+alt+p"
+ self.desk_stream_detectors = None
+ self._sigusr1_old = None
self.browser_mu = threading.Lock()
self.browser_mu = threading.Lock()
self.browser_op_mu = threading.Lock()
self.browser_op_mu = threading.Lock()
self.browser_factory = browser_factory or self._new_browser
self.browser_factory = browser_factory or self._new_browser
@@ -428,6 +450,18 @@ class Computer(Plugin):
@@ -428,6 +450,18 @@ class Computer(Plugin):
self.notify = config.get("notify", True) is not False
self.notify = config.get("notify", True) is not False
self.allow_browser = config.get("allow_browser", True) is not False
self.allow_browser = config.get("allow_browser", True) is not False
self.allow_screen_stream = config.get("allow_screen_stream", True) is not False
self.allow_screen_stream = config.get("allow_screen_stream", True) is not False
+ self.allow_desk_stream = config.get("allow_desk_stream", True) is not False
+ self.desk_stream_hotkey = str(config.get("desk_stream_hotkey") or "ctrl+alt+p")
+ sites = config.get("desk_stream_allow_sites") or []
+ if not isinstance(sites, list) or not all(isinstance(u, str) and u for u in sites) or len(sites) > 50:
+ raise Fault("bad_value", "desk_stream_allow_sites is a list of up to 50 URLs")
+ self.desk_stream_allow_sites = list(sites)
+ dets = config.get("desk_stream_detectors")
+ if dets is not None:
+ known = ("password_field", "secret_file", "env_print", "manual", "site")
+ if not isinstance(dets, list) or not all(d in known for d in dets):
+ raise Fault("bad_value", "desk_stream_detectors is a list of %s" % ", ".join(known))
+ self.desk_stream_detectors = list(dets)
self.allow_desk = config.get("allow_desk", True) is not False
self.allow_desk = config.get("allow_desk", True) is not False
self.desks.max_desks = _int(config.get("max_desks"), "max_desks", 1, MAX_DESKS_CAP, default_max_desks())
self.desks.max_desks = _int(config.get("max_desks"), "max_desks", 1, MAX_DESKS_CAP, default_max_desks())
self.desk_idle_s = _int(config.get("desk_idle_s"), "desk_idle_s", 60, 86400, 900)
self.desk_idle_s = _int(config.get("desk_idle_s"), "desk_idle_s", 60, 86400, 900)
@@ -492,6 +526,10 @@ class Computer(Plugin):
@@ -492,6 +526,10 @@ class Computer(Plugin):
caps["browser"] = {"level": self.level, "browser": os.path.basename(binary), "headless": self.browser_headless,
caps["browser"] = {"level": self.level, "browser": os.path.basename(binary), "headless": self.browser_headless,
"egress": self.browser_egress}
"egress": self.browser_egress}
kinds = self._desk_kinds()
kinds = self._desk_kinds()
+ if self.allow_desk_stream and (kinds or tool):
+ caps["desk.stream"] = {"capture": capture_kinds(self.platform), "browser": "browser" in kinds,
+ "curtain": ["password_field", "secret_file", "env_print", "manual", "site", "detector"],
+ "hotkey": self.desk_stream_hotkey, "allow_sites": True}
if kinds:
if kinds:
# A browser desk's screen and input are its page's own (the browser's pipe); a whole desk's need X tools.
# A browser desk's screen and input are its page's own (the browser's pipe); a whole desk's need X tools.
whole, page = "desktop" in kinds, "browser" in kinds
whole, page = "desktop" in kinds, "browser" in kinds
@@ -941,6 +979,8 @@ class Computer(Plugin):
@@ -941,6 +979,8 @@ class Computer(Plugin):
return self._start(cmd, lambda: self._screen(tool, width, env=desk_env))
return self._start(cmd, lambda: self._screen(tool, width, env=desk_env))
if cmd.kind == "screen_stream":
if cmd.kind == "screen_stream":
return self._screen_stream(v, env=desk_env, desk=desk_name, page=page)
return self._screen_stream(v, env=desk_env, desk=desk_name, page=page)
+ if cmd.kind == "desk.stream":
+ return self._desk_stream_op(cmd, v, level)
if cmd.kind == "fs":
if cmd.kind == "fs":
op = v.get("op")
op = v.get("op")
if op in READ_OPS:
if op in READ_OPS:
@@ -988,7 +1028,7 @@ class Computer(Plugin):
@@ -988,7 +1028,7 @@ class Computer(Plugin):
return
return
if kind in SHARED_DESK_KINDS and v.get("desk", "main") != "main":
if kind in SHARED_DESK_KINDS and v.get("desk", "main") != "main":
return
return
- if kind == "screen_stream":
+ if kind in ("screen_stream", "desk.stream"):
raise Fault("not_allowed", "this desk host shows a desk as pictures (screen), not live: its one stream is its operator's")
raise Fault("not_allowed", "this desk host shows a desk as pictures (screen), not live: its one stream is its operator's")
raise Fault("not_allowed", "this machine is a desk host for other people: only its browser desks take commands")
raise Fault("not_allowed", "this machine is a desk host for other people: only its browser desks take commands")
@@ -1646,7 +1686,10 @@ class Computer(Plugin):
@@ -1646,7 +1686,10 @@ class Computer(Plugin):
left = deadline - self._deadline_clock()() - 1.0
left = deadline - self._deadline_clock()() - 1.0
if left < 1:
if left < 1:
raise Fault("expired", "no time was left to run it")
raise Fault("expired", "no time was left to run it")
- return self._exec(argv, cwd, stdin, min(timeout, left), env=env)
+ out = self._exec(argv, cwd, stdin, min(timeout, left), env=env)
+ # The desk stream's environment-print detector sees the output before it leaves the machine.
+ self._desk_stream_saw_output(argv, out)
+ return out
return run, self._exec_summary(argv, cwd.path if cwd is not None else None, stdin, desk)
return run, self._exec_summary(argv, cwd.path if cwd is not None else None, stdin, desk)
@@ -1852,6 +1895,7 @@ class Computer(Plugin):
@@ -1852,6 +1895,7 @@ class Computer(Plugin):
offset = _int(v.get("offset"), "offset", 0, 2 ** 53, 0)
offset = _int(v.get("offset"), "offset", 0, 2 ** 53, 0)
length = _int(v.get("length"), "length", 0, self.max_read, self.max_read)
length = _int(v.get("length"), "length", 0, self.max_read, self.max_read)
data, size = target.read(offset, length)
data, size = target.read(offset, length)
+ self._desk_stream_saw_path(target.path)
return {"path": target.path, "size": size, "offset": offset, "length": len(data),
return {"path": target.path, "size": size, "offset": offset, "length": len(data),
"truncated": offset + len(data) < size, "content_b64": base64.b64encode(data).decode("ascii")}
"truncated": offset + len(data) < size, "content_b64": base64.b64encode(data).decode("ascii")}
@@ -1990,18 +2034,22 @@ class Computer(Plugin):
@@ -1990,18 +2034,22 @@ class Computer(Plugin):
try:
try:
with lock(): # one action at a time in each browser; two desks' browsers act in parallel
with lock(): # one action at a time in each browser; two desks' browsers act in parallel
if action == "open":
if action == "open":
- return b.open(url)
- if action == "read":
- return b.read()
- if action == "screenshot":
- return b.screenshot(width)
- if action == "click":
- return b.click(selector=selector, text=None if selector else text)
- if action == "fill":
- return b.fill(selector, text)
- if action == "press":
- return b.press(key)
- return b.back()
+ out = b.open(url)
+ elif action == "read":
+ out = b.read()
+ elif action == "screenshot":
+ out = b.screenshot(width)
+ elif action == "click":
+ out = b.click(selector=selector, text=None if selector else text)
+ elif action == "fill":
+ out = b.fill(selector, text)
+ elif action == "press":
+ out = b.press(key)
+ else:
+ out = b.back()
+ if isinstance(out, dict):
+ self._desk_stream_saw_url(out.get("url"))
+ return out
except BrowserError as e:
except BrowserError as e:
raise Fault(e.code, e.message)
raise Fault(e.code, e.message)
@@ -2273,6 +2321,206 @@ class Computer(Plugin):
@@ -2273,6 +2321,206 @@ class Computer(Plugin):
def _stream_capable(self, tool):
def _stream_capable(self, tool):
return self._have_pillow() or self._native_jpeg(tool, "x.jpg") is not None
return self._have_pillow() or self._native_jpeg(tool, "x.jpg") is not None
+ # ---- the agent's desk stream (Agent Streams S1) ---------------------------------------------------------------
+ def _desk_stream_op(self, cmd, v, level):
+ """desk.stream: start/stop/status/curtain one agent stream. The core mints the wpt_ ticket and publishes; the
+ plugin says how to capture and runs the curtain's detectors (fail closed)."""
+ action = v.get("action")
+ if action == "status":
+ return self._desk_stream_status()
+ if action == "stop":
+ return {"streaming": False, "stopped": self._stop_desk_stream("stopped by its owner")}
+ if action == "curtain":
+ with self.desk_stream_mu:
+ st = self.desk_stream
+ if st is None:
+ raise Fault("not_ready", "no desk stream is running")
+ up = v.get("up") is True
+ reason = v.get("reason") if v.get("reason") in ("manual", "site", "detector") else "manual"
+ if up:
+ st["curtain"].raise_(reason)
+ else:
+ st["curtain"].lower()
+ return self._desk_stream_status()
+ if action != "start":
+ raise Fault("bad_value", "action is start, stop, status or curtain")
+ if not self.allow_desk_stream:
+ raise Fault("unsupported", "this machine does not stream its desks (allow_desk_stream is off)")
+ if self._shared():
+ raise Fault("not_allowed", "a desk host for other people shows a desk as pictures, never live")
+ desk = v.get("desk") or "main"
+ if desk != "main":
+ self.desks.check(desk)
+ if not self.desks.status(desk).get("running"):
+ raise Fault("not_ready", "start the desk first (desk start, desk %s)" % desk)
+ allow_sites = v.get("allow_sites")
+ if not isinstance(allow_sites, list):
+ allow_sites = self.desk_stream_allow_sites
+ allow_sites = [u for u in allow_sites if isinstance(u, str) and u][:50]
+ self._stop_desk_stream("replaced")
+ browser_desk = desk != "main" and self.desks.kind(desk) == "browser"
+ screencast = None
+ if browser_desk:
+ status = self.desks.status(desk)
+ width, height = status.get("width") or 1280, status.get("height") or 800
+ browser = self._desk_browser(desk)
+ try:
+ screencast = PageScreencast(browser, width=width, height=height).start()
+ except ValueError as e:
+ raise Fault("unsupported", str(e))
+ capture = {"kind": "browser", "pipe": screencast.fifo, "width": width, "height": height}
+ else:
+ if desk == "main":
+ if not self.allow_screen_stream:
+ raise Fault("unsupported", "this machine does not show its screen (allow_screen_stream is off)")
+ if not self._screen_tool():
+ raise Fault("unsupported", "no screen capture tool on this machine")
+ env = self.env
+ width, height = self._display_size(env)
+ else:
+ status = self.desks.status(desk)
+ env = self.desks.env(desk)
+ width, height = status.get("width") or 1280, status.get("height") or 800
+ try:
+ capture = desktop_capture(self.platform, env, width, height,
+ ffmpeg=self.which("ffmpeg") or "ffmpeg")
+ except ValueError as e:
+ raise Fault("unsupported", str(e))
+ current_url = (lambda: browser.last_navigation) if browser_desk else None
+ detectors = self._desk_stream_detectors(allow_sites, env if not browser_desk else None, current_url)
+ curtain = Curtain(detectors=list(detectors.values()), on_change=self._desk_stream_curtain_changed)
+ # Fail closed: the picture starts blank and only clears after every detector has been clear for two seconds.
+ curtain.raise_("detector")
+ curtain.start()
+ hotkey = Hotkey(self._desk_stream_toggle, self.desk_stream_hotkey)
+ hotkey.start()
+ self._sigusr1_old = install_sigusr1(self._desk_stream_toggle)
+ st = {"sid": os.urandom(8).hex(), "desk": desk, "capture": capture, "screencast": screencast,
+ "curtain": curtain, "hotkey": hotkey, "detectors": detectors, "allow_sites": allow_sites,
+ "since": self.clock()}
+ stop = threading.Event()
+ st["heartbeat"], st["heartbeat_thread"] = stop, threading.Thread(
+ target=self._desk_stream_heartbeat, args=(st, stop), name="computer-curtain-beat", daemon=True)
+ with self.desk_stream_mu:
+ self.desk_stream = st
+ st["heartbeat_thread"].start()
+ return {"streaming": True, "sid": st["sid"], "capture": capture, "curtain": curtain.snapshot(),
+ "detectors": sorted(detectors), "hotkey": self.desk_stream_hotkey}
+
+ def _desk_stream_heartbeat(self, st, stop):
+ """Tell the core the curtain state every second. Its watchdog blanks the desk if this goes quiet for two
+ seconds, so a hung plugin cannot leave a picture up; a clear state still has to last two seconds there."""
+ while not stop.is_set():
+ snap = st["curtain"].snapshot()
+ try:
+ self.emit_event("curtain", up=bool(snap["up"]), reason=snap["reason"] or "detector", heartbeat=True)
+ except Exception:
+ pass
+ stop.wait(DESK_STREAM_HEARTBEAT_S)
+
+ def _desk_stream_status(self):
+ with self.desk_stream_mu:
+ st = self.desk_stream
+ if st is None:
+ return {"streaming": False}
+ out = {"streaming": True, "sid": st["sid"], "desk": st["desk"], "capture": st["capture"],
+ "curtain": st["curtain"].snapshot(), "detectors": sorted(st["detectors"]), "allow_sites": st["allow_sites"]}
+ cast = st.get("screencast")
+ if cast is not None:
+ out["page"] = cast.status()
+ return out
+
+ def _stop_desk_stream(self, reason):
+ with self.desk_stream_mu:
+ st, self.desk_stream = self.desk_stream, None
+ if st is None:
+ return False
+ cast = st.get("screencast")
+ if cast is not None:
+ cast.stop()
+ beat = st.get("heartbeat")
+ if beat is not None:
+ beat.set()
+ thread = st.get("heartbeat_thread")
+ if thread is not None and thread is not threading.current_thread() and thread.is_alive():
+ thread.join(1.0)
+ st["hotkey"].stop()
+ st["curtain"].stop()
+ if self._sigusr1_old is not None:
+ try:
+ signal.signal(signal.SIGUSR1, self._sigusr1_old)
+ except (ValueError, OSError):
+ pass
+ self._sigusr1_old = None
+ self.log("desk stream ended (%s)", reason)
+ return True
+
+ def _desk_stream_detectors(self, allow_sites, env=None, current_url=None):
+ wanted = self.desk_stream_detectors or ("password_field", "secret_file", "env_print", "site")
+ out = {}
+ if "password_field" in wanted:
+ out["password_field"] = PasswordFieldDetector()
+ if "secret_file" in wanted:
+ out["secret_file"] = SecretFileDetector(window=(lambda: _active_window(env)) if env else None)
+ if "env_print" in wanted:
+ out["env_print"] = EnvPrintDetector()
+ if "site" in wanted and allow_sites:
+ out["site"] = SiteDetector(allow_sites, current_url=current_url)
+ return out
+
+ def _desk_stream_curtain_changed(self, up, reason):
+ self.emit_event("curtain", up=bool(up), reason=reason)
+ self.log("curtain %s (%s)", "up" if up else "down", reason)
+
+ def _desk_stream_toggle(self, _reason="manual"):
+ with self.desk_stream_mu:
+ st = self.desk_stream
+ if st is None:
+ return
+ if st["curtain"].snapshot()["up"]:
+ st["curtain"].lower()
+ else:
+ st["curtain"].raise_("manual")
+
+ def _desk_stream_saw_path(self, path):
+ with self.desk_stream_mu:
+ st = self.desk_stream
+ if st is not None and "secret_file" in st["detectors"]:
+ st["detectors"]["secret_file"].saw_path(path)
+
+ def _desk_stream_saw_output(self, argv, out):
+ with self.desk_stream_mu:
+ st = self.desk_stream
+ if st is None or "env_print" not in st["detectors"]:
+ return
+ text = ""
+ if isinstance(out, dict):
+ text = (out.get("stdout") or "") + "\n" + (out.get("stderr") or "")
+ st["detectors"]["env_print"].saw_output(argv, text)
+ if "secret_file" in st["detectors"]:
+ for a in argv:
+ st["detectors"]["secret_file"].saw_path(a)
+
+ def _desk_stream_saw_url(self, url):
+ with self.desk_stream_mu:
+ st = self.desk_stream
+ if st is not None and "site" in st["detectors"]:
+ st["detectors"]["site"].saw_url(url)
+
+ def _display_size(self, env):
+ """The display's pixel size for x11grab, or 1920×1080 where there is no xdotool."""
+ if not (env or {}).get("DISPLAY"):
+ return 1920, 1080
+ try:
+ r = subprocess.run(["xdotool", "getdisplaygeometry"], capture_output=True, text=True, timeout=3,
+ env=self._child_env(env) if env is not None else None)
+ w, h = [int(x) for x in r.stdout.split()]
+ if w >= 320 and h >= 180:
+ return w, h
+ except (OSError, ValueError, subprocess.SubprocessError):
+ pass
+ return 1920, 1080
+
def _screen_stream(self, v, env=None, desk=None, page=False):
def _screen_stream(self, v, env=None, desk=None, page=False):
action = v.get("action")
action = v.get("action")
if action == "stop":
if action == "stop":
@@ -227,6 +227,7 @@ class Browser:
@@ -227,6 +227,7 @@ class Browser:
self.behind = [] # (target, session) of the pages it was opened from, newest last
self.behind = [] # (target, session) of the pages it was opened from, newest last
self.tab_events = [] # Target.targetCreated / targetDestroyed seen since the last _follow
self.tab_events = [] # Target.targetCreated / targetDestroyed seen since the last _follow
self.tab_change = None # "opened" or "closed" since the last _mark: said in the action's receipt
self.tab_change = None # "opened" or "closed" since the last _mark: said in the action's receipt
+ self.last_navigation = None # the URL of the last main-frame navigation (a desk stream's allow-list reads it)
self.follow_mu = threading.Lock()
self.follow_mu = threading.Lock()
self.after_start = None # called once the browser runs (a kept profile's session cookies go back in)
self.after_start = None # called once the browser runs (a kept profile's session cookies go back in)
@@ -398,6 +399,10 @@ class Browser:
@@ -398,6 +399,10 @@ class Browser:
waiting[1] = {"error": {"message": "Session closed: its page closed"}}
waiting[1] = {"error": {"message": "Session closed: its page closed"}}
waiting[0].set()
waiting[0].set()
self.events.append(msg["method"])
self.events.append(msg["method"])
+ if msg["method"] == "Page.frameNavigated":
+ frame = (msg.get("params") or {}).get("frame") or {}
+ if not frame.get("parentId") and frame.get("url"):
+ self.last_navigation = frame["url"]
if msg["method"] in ("Target.targetCreated", "Target.targetDestroyed"):
if msg["method"] in ("Target.targetCreated", "Target.targetDestroyed"):
self.tab_events = (self.tab_events + [(msg["method"], msg.get("params") or {})])[-50:]
self.tab_events = (self.tab_events + [(msg["method"], msg.get("params") or {})])[-50:]
if len(self.events) > 500:
if len(self.events) > 500:
new file mode 100644
new file mode 100644
@@ -0,0 +1,494 @@
@@ -0,0 +1,494 @@
+"""The desk stream's privacy curtain (Agent Streams S1).
+
+The curtain is a state, not a button: while it is up, no captured frame reaches the publisher and the desk reports
+``{"up": true, "reason": …}`` to OpenVibe.Live, which pauses every viewer's consumer as well. A detector answers with a
+reason when it sees something private, with ``None`` when it is clear, and may raise when it cannot tell; **a detector
+that raises, or that is unsure, puts the curtain up and never down** (the design's fail-closed rule). The curtain comes
+down only when every detector has been clear for two seconds, or when the owner lowers it.
+
+Detectors:
+ * ``password_field`` — the focused accessible element is a password field (AT-SPI on Linux, UI Automation on Windows,
+ the AX API on macOS; the probe is injected so each platform, and the tests, supply their own);
+ * ``secret_file`` — the focused window's title or class looks secret, or a secret-named file was just read;
+ * ``env_print`` — an ``exec`` printed environment values or ran ``env``/``printenv``;
+ * ``manual`` — the person pressed the hotkey or chose Curtain now;
+ * ``site`` — a browser desk navigated outside its allow-list;
+ * ``detector`` — a detector failed or could not decide.
+
+The hotkey is Ctrl+Alt+P by default on X11 (XGrabKey through ctypes, no extra dependency). ``SIGUSR1`` toggles the
+curtain too, which is what a service can be given without a display. Neither the reason nor a detector's error text is
+allowed to carry anything a viewer sees: viewers only ever see "Private screen".
+"""
+from __future__ import annotations
+
+import ctypes
+import os
+import re
+import signal
+import subprocess
+import sys
+import threading
+import time
+from urllib.parse import urlparse
+
+CURTAIN_CLEAR_HOLD_S = 2.0
+CURTAIN_POLL_S = 0.25
+# How long a one-off signal (a secret file read, an environment print) keeps the curtain up once it is seen.
+CURTAIN_EVENT_HOLD_S = 10.0
+
+# A path or window title that looks secret. `.env`, `*credentials*`, a private key, PEMs and password managers.
+SECRET_NAME = re.compile(
+ r"(^|[/\\])(\.env(\.[A-Za-z0-9_-]+)?|[^/\\]*credentials[^/\\]*|id_rsa|id_dsa|id_ecdsa|id_ed25519|"
+ r"[^/\\]*\.(pem|key|p12|pfx|kdbx|ovpn)|[^/\\]*secret[^/\\]*)$",
+ re.IGNORECASE)
+SECRET_WINDOW = re.compile(
+ r"(keepass|1password|bitwarden|lastpass|dashlane|nordpass|password|credential|secret|\benv\b)", re.IGNORECASE)
+
+REASONS = ("password_field", "secret_file", "env_print", "manual", "site", "detector")
+
+# A line from `env`/`printenv`, or an exported assignment. Three or more in one output is an environment print.
+ENV_LINE = re.compile(r"^[A-Z_][A-Z0-9_]{2,}=.*$")
+
+
+def looks_secret_name(name):
+ return bool(name) and bool(SECRET_NAME.search(name))
+
+
+def looks_secret_window(title, klass=""):
+ return bool(title and SECRET_WINDOW.search(title)) or bool(klass and SECRET_WINDOW.search(klass))
+
+
+class Detector:
+ """A curtain detector. poll() returns None when clear, a reason when it sees something private, and may raise."""
+ reason = "detector"
+
+ def poll(self):
+ return None
+
+
+class PasswordFieldDetector(Detector):
+ reason = "password_field"
+
+ def __init__(self, probe=None):
+ self.probe = probe or _password_field_focused
+
+ def poll(self):
+ v = self.probe()
+ if v is None:
+ # Unsure is not clear.
+ return self.reason
+ return self.reason if v else None
+
+
+class SecretFileDetector(Detector):
+ reason = "secret_file"
+
+ def __init__(self, window=None, clock=time.monotonic):
+ self.window = window or _active_window
+ self.clock = clock
+ self.until = 0.0
+
+ def saw_path(self, name):
+ if looks_secret_name(name):
+ self.until = max(self.until, self.clock() + CURTAIN_EVENT_HOLD_S)
+
+ def poll(self):
+ title, klass = self.window()
+ if looks_secret_window(title, klass):
+ return self.reason
+ if self.clock() < self.until:
+ return self.reason
+ return None
+
+
+class EnvPrintDetector(Detector):
+ reason = "env_print"
+
+ def __init__(self, clock=time.monotonic):
+ self.clock = clock
+ self.until = 0.0
+
+ def saw_output(self, argv, output):
+ """The exec wrapper calls this with what the command printed, before the output leaves the machine."""
+ if not output:
+ return
+ cmd = os.path.basename(argv[0]) if argv else ""
+ if cmd in ("env", "printenv") or any(a in ("-0", "--null") for a in argv):
+ self.until = max(self.until, self.clock() + CURTAIN_EVENT_HOLD_S)
+ return
+ hits = 0
+ for line in output.splitlines():
+ if ENV_LINE.match(line):
+ hits += 1
+ if hits >= 3:
+ self.until = max(self.until, self.clock() + CURTAIN_EVENT_HOLD_S)
+ return
+
+ def poll(self):
+ return self.reason if self.clock() < self.until else None
+
+
+class SiteDetector(Detector):
+ reason = "site"
+
+ def __init__(self, allow_sites=None, current_url=None):
+ self.allow = [u for u in (allow_sites or []) if u]
+ self.current_url = current_url
+ self.last = None
+
+ def allows(self, url):
+ if not self.allow:
+ return True
+ host = urlparse(url).hostname or ""
+ for entry in self.allow:
+ want = urlparse(entry).hostname if "://" in entry else entry
+ if not want:
+ continue
+ if host == want or host.endswith("." + want):
+ return True
+ return False
+
+ def saw_url(self, url):
+ self.last = url
+ if url and not self.allows(url):
+ return self.reason
+ return None
+
+ def poll(self):
+ if self.current_url is not None:
+ try:
+ url = self.current_url()
+ except Exception:
+ return self.reason
+ else:
+ url = self.last
+ return self.saw_url(url)
+
+
+class ManualDetector(Detector):
+ reason = "manual"
+
+ def __init__(self):
+ self.raised = False
+ self.lock = threading.Lock()
+
+ def raise_(self):
+ with self.lock:
+ self.raised = True
+
+ def lower(self):
+ with self.lock:
+ self.raised = False
+
+ def poll(self):
+ with self.lock:
+ return self.reason if self.raised else None
+
+
+class Curtain:
+ """The fail-closed curtain. evaluate() is pure enough to test with an injected clock and detectors."""
+
+ def __init__(self, detectors=None, clock=time.monotonic, clear_hold=CURTAIN_CLEAR_HOLD_S, on_change=None):
+ self.detectors = list(detectors or [])
+ self.clock = clock
+ self.clear_hold = clear_hold
+ self.on_change = on_change
+ self.lock = threading.RLock()
+ self.up = False
+ self.reason = None
+ self.since = None
+ self.clear_since = None
+ self.error = None
+ self._stop = threading.Event()
+ self._thread = None
+ self.manual = ManualDetector()
+ self.detectors.append(self.manual)
+
+ def add(self, detector):
+ with self.lock:
+ self.detectors.append(detector)
+ return detector
+
+ def evaluate(self):
+ """One pass: any raised or unsure detector raises the curtain; all clear for two seconds lowers it."""
+ with self.lock:
+ reason = None
+ for det in self.detectors:
+ try:
+ got = det.poll()
+ except Exception as e: # a detector that fails is never clear
+ got = "detector"
+ self.error = "%s: %s" % (getattr(det, "reason", "detector"), e)
+ if got:
+ reason = got if got in REASONS else "detector"
+ break
+ now = self.clock()
+ if reason:
+ self.clear_since = None
+ return self._set(True, reason, now)
+ if not self.up:
+ self.clear_since = None
+ return False
+ if self.clear_since is None:
+ self.clear_since = now
+ if now - self.clear_since >= self.clear_hold:
+ return self._set(False, None, now)
+ return True
+
+ def _set(self, up, reason, now):
+ if up == self.up:
+ if up and reason and reason != self.reason:
+ self.reason = reason
+ if self.on_change:
+ try:
+ self.on_change(up, reason)
+ except Exception:
+ pass
+ return self.up
+ self.up = up
+ self.reason = reason
+ self.since = now if up else None
+ self.clear_since = None
+ if self.on_change:
+ try:
+ self.on_change(up, reason or "detector")
+ except Exception:
+ pass
+ return self.up
+
+ def raise_(self, reason="manual"):
+ if reason == "manual":
+ self.manual.raise_()
+ with self.lock:
+ self._set(True, reason if reason in REASONS else "manual", self.clock())
+
+ def lower(self):
+ """The owner's own button: lower now (the detectors may raise it again on the next pass)."""
+ self.manual.lower()
+ with self.lock:
+ self.clear_since = self.clock()
+ return self._set(False, None, self.clock())
+
+ def snapshot(self):
+ with self.lock:
+ return {"up": self.up, "reason": self.reason if self.up else None,
+ "since": None if self.since is None else int(self.since)}
+
+ def start(self):
+ if self._thread is not None:
+ return
+ self._stop.clear()
+ self._thread = threading.Thread(target=self._loop, name="computer-curtain", daemon=True)
+ self._thread.start()
+
+ def _loop(self):
+ while not self._stop.is_set():
+ try:
+ self.evaluate()
+ except Exception:
+ self.raise_("detector")
+ self._stop.wait(CURTAIN_POLL_S)
+
+ def stop(self):
+ self._stop.set()
+ thread, self._thread = self._thread, None
+ if thread is not None and thread is not threading.current_thread() and thread.is_alive():
+ thread.join(1.0)
+
+
+# ---- platform probes -----------------------------------------------------------------------------------------------
+
+def _active_window(env=None):
+ """(title, WM_CLASS) of the focused X11 window, or ("", "") when there is none (no display, no xdotool)."""
+ run_env = None
+ if env is not None:
+ run_env = dict(os.environ)
+ run_env.update({k: v for k, v in env.items() if isinstance(v, str)})
+ if not (run_env or os.environ).get("DISPLAY"):
+ return "", ""
+ try:
+ win = subprocess.run(["xdotool", "getactivewindow"], capture_output=True, text=True, timeout=2, env=run_env)
+ except (OSError, subprocess.SubprocessError):
+ return "", ""
+ wid = win.stdout.strip()
+ if not wid:
+ return "", ""
+ try:
+ title = subprocess.run(["xdotool", "getwindowname", wid], capture_output=True, text=True, timeout=2,
+ env=run_env).stdout.strip()
+ except (OSError, subprocess.SubprocessError):
+ title = ""
+ try:
+ klass = subprocess.run(["xprop", "-id", wid, "WM_CLASS"], capture_output=True, text=True, timeout=2,
+ env=run_env).stdout.strip()
+ except (OSError, subprocess.SubprocessError):
+ klass = ""
+ return title, klass
+
+
+def _password_field_focused():
+ """True when the focused AT-SPI element is a password field, False when it is not, None when it cannot tell.
+
+ On Windows and macOS the probe is the platform's accessibility API; this module's default is the Linux one. A
+ missing accessibility stack is None (unsure), which raises the curtain: the stream is never shown blind.
+ """
+ if sys.platform == "win32":
+ return _password_field_windows()
+ if sys.platform == "darwin":
+ return _password_field_macos()
+ try:
+ import pyatspi
+ except Exception:
+ return None
+ try:
+ desktop = pyatspi.Registry.getDesktop(0)
+ focused = _focused_accessible(desktop, pyatspi, 0)
+ if focused is None:
+ return False
+ return focused.getRole() == pyatspi.ROLE_PASSWORD_TEXT
+ except Exception:
+ return None
+
+
+def _focused_accessible(node, pyatspi, depth):
+ """The accessible with STATE_FOCUSED, walking at most a dozen levels (accessibility trees can be huge)."""
+ if depth > 12:
+ return None
+ try:
+ state = node.getState()
+ if state is not None and state.contains(pyatspi.STATE_FOCUSED):
+ return node
+ except Exception:
+ pass
+ try:
+ children = node
+ except Exception:
+ return None
+ try:
+ for child in children:
+ got = _focused_accessible(child, pyatspi, depth + 1)
+ if got is not None:
+ return got
+ except Exception:
+ return None
+ return None
+
+
+def _password_field_windows():
+ # UI Automation through PowerShell: the focused element's ControlType is Edit and IsPassword is true.
+ try:
+ script = ("Add-Type -AssemblyName UIAutomationClient;"
+ "$e=[System.Windows.Automation.AutomationElement]::FocusedElement;"
+ "if($e -and $e.Current.IsPassword){'yes'}else{'no'}")
+ r = subprocess.run(["powershell", "-NoProfile", "-Command", script], capture_output=True, text=True, timeout=3)
+ except (OSError, subprocess.SubprocessError):
+ return None
+ if "yes" in r.stdout:
+ return True
+ if "no" in r.stdout:
+ return False
+ return None
+
+
+def _password_field_macos():
+ # The AX API is only reachable from a signed/trusted helper; without it the answer is unsure.
+ return None
+
+
+# ---- the hotkey ---------------------------------------------------------------------------------------------------
+
+class Hotkey:
+ """A global curtain hotkey. X11: Ctrl+Alt+P through XGrabKey (ctypes, no dependency). Elsewhere: unavailable, and
+ the owner uses Curtain now on Live or `openvibe-node stream` (SIGUSR1) instead."""
+
+ def __init__(self, callback, combo="ctrl+alt+p"):
+ self.callback = callback
+ self.combo = combo
+ self._stop = threading.Event()
+ self._thread = None
+ self.error = None
+
+ def available(self):
+ return sys.platform.startswith("linux") and bool(os.environ.get("DISPLAY"))
+
+ def start(self):
+ if not self.available() or self._thread is not None:
+ return False
+ self._thread = threading.Thread(target=self._run, name="computer-curtain-hotkey", daemon=True)
+ self._thread.start()
+ return True
+
+ def stop(self):
+ self._stop.set()
+ thread, self._thread = self._thread, None
+ if thread is not None and thread.is_alive():
+ thread.join(1.0)
+
+ def _run(self):
+ try:
+ x11 = ctypes.CDLL("libX11.so.6")
+ except OSError as e:
+ self.error = str(e)
+ return
+ # Declare the few calls used; keycodes and masks are constants in X11/X.h.
+ x11.XOpenDisplay.restype = ctypes.c_void_p
+ x11.XOpenDisplay.argtypes = [ctypes.c_char_p]
+ x11.XDefaultRootWindow.restype = ctypes.c_ulong
+ x11.XDefaultRootWindow.argtypes = [ctypes.c_void_p]
+ x11.XStringToKeysym.restype = ctypes.c_ulong
+ x11.XStringToKeysym.argtypes = [ctypes.c_char_p]
+ x11.XKeysymToKeycode.restype = ctypes.c_ubyte
+ x11.XKeysymToKeycode.argtypes = [ctypes.c_void_p, ctypes.c_ulong]
+ x11.XGrabKey.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_uint, ctypes.c_ulong, ctypes.c_int,
+ ctypes.c_int, ctypes.c_int]
+ x11.XPending.argtypes = [ctypes.c_void_p]
+ x11.XNextEvent.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+ x11.XCloseDisplay.argtypes = [ctypes.c_void_p]
+ x11.XCloseDisplay.restype = ctypes.c_int
+ dpy = x11.XOpenDisplay(None)
+ if not dpy:
+ self.error = "no X display"
+ return
+ try:
+ key = self.combo.split("+")[-1]
+ mods = 0
+ parts = [p.strip().lower() for p in self.combo.split("+")[:-1]]
+ if "ctrl" in parts or "control" in parts:
+ mods |= 4 # ControlMask
+ if "alt" in parts or "mod1" in parts:
+ mods |= 8 # Mod1Mask
+ if "shift" in parts:
+ mods |= 1 # ShiftMask
+ keysym = x11.XStringToKeysym(key.encode("ascii"))
+ keycode = x11.XKeysymToKeycode(dpy, keysym)
+ root = x11.XDefaultRootWindow(dpy)
+ # Grab the four CapsLock/NumLock variants, else the hotkey stops working when either is on.
+ for extra in (0, 2, 16, 18): # LockMask, Mod2Mask
+ x11.XGrabKey(dpy, keycode, mods | extra, root, 1, 1, 1)
+ event = ctypes.create_string_buffer(192) # XEvent is 24 longs; this is plenty for KeyPress.
+ while not self._stop.is_set():
+ while x11.XPending(dpy):
+ x11.XNextEvent(dpy, event)
+ try:
+ self.callback("manual")
+ except Exception:
+ pass
+ self._stop.wait(0.05)
+ finally:
+ x11.XCloseDisplay(dpy)
+
+
+def install_sigusr1(toggle):
+ """A service's local curtain hotkey: SIGUSR1 toggles it (kill -USR1 <pid>). Returns the previous handler."""
+ try:
+ old = signal.getsignal(signal.SIGUSR1)
+
+ def handler(signum, frame):
+ try:
+ toggle()
+ except Exception:
+ pass
+ signal.signal(signal.SIGUSR1, handler)
+ return old
+ except (ValueError, AttributeError, OSError):
+ return None
new file mode 100644
new file mode 100644
@@ -0,0 +1,178 @@
@@ -0,0 +1,178 @@
+"""How the desk stream captures a picture (Agent Streams S1).
+
+For a whole desk (an Xvfb display) and the machine's own screen the core runs ffmpeg itself, with the display and size
+this module reports: x11grab or PipeWire on Linux, ddagrab on Windows, AVFoundation on macOS. A browser desk has no
+display to grab, so its own page is captured here over the DevTools pipe at about five frames a second (the floor of
+the design's 5–30 VFR) and written as JPEGs into a small FIFO that the core's ffmpeg reads with `-f image2pipe`.
+
+The FIFO is 0600 in a 0700 folder and is removed with the stream. Nothing here calls the owner's screen: a browser desk
+uses its own headless browser, and a named desk's display is its Xvfb one.
+"""
+from __future__ import annotations
+
+import errno
+import os
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+PAGE_STREAM_FPS = 5
+PAGE_STREAM_WIDTH = 1280
+PAGE_STREAM_QUALITY = 70
+FIFO_NAME = "frames.fifo"
+
+
+def ffmpeg_has_pipewire(ffmpeg):
+ """Whether this ffmpeg has the pipewire input device (7.1 and later)."""
+ try:
+ out = subprocess.run([ffmpeg or "ffmpeg", "-hide_banner", "-devices"],
+ capture_output=True, text=True, timeout=5).stdout
+ except (OSError, subprocess.SubprocessError):
+ return False
+ return "pipewire" in out
+
+
+def capture_kinds(platform=None):
+ platform = platform or sys.platform
+ if platform == "win32":
+ return ["ddagrab"]
+ if platform == "darwin":
+ return ["avfoundation"]
+ return ["x11grab", "pipewire"]
+
+
+def desktop_capture(platform, env, width, height, ffmpeg="", pipewire_node=None):
+ """The capture spec for a display (a desk's Xvfb or the owner's screen), or a Fault-worthy message."""
+ width, height = _size(width, height)
+ platform = platform or sys.platform
+ if platform == "win32":
+ return {"kind": "ddagrab", "width": width, "height": height}
+ if platform == "darwin":
+ return {"kind": "avfoundation", "device": os.environ.get("OPENVIBE_AVFOUNDATION_SCREEN", "1:none"),
+ "width": width, "height": height}
+ display = (env or {}).get("DISPLAY")
+ if display:
+ # x11grab takes `:N.0`; the desk already carries its own display number.
+ if not display.endswith(".0"):
+ display += ".0"
+ return {"kind": "x11grab", "display": display, "width": width, "height": height}
+ if (env or {}).get("WAYLAND_DISPLAY"):
+ node = pipewire_node or os.environ.get("OPENVIBE_PIPEWIRE_NODE")
+ if not node:
+ raise ValueError("a Wayland desk needs OPENVIBE_PIPEWIRE_NODE set to the screen-cast node id")
+ if not ffmpeg_has_pipewire(ffmpeg):
+ raise ValueError("this ffmpeg has no pipewire input; install ffmpeg 7.1+ or set a display")
+ return {"kind": "pipewire", "pipewire": str(node), "width": width, "height": height}
+ raise ValueError("no display to capture on this machine")
+
+
+def _size(width, height):
+ try:
+ width, height = int(width), int(height)
+ except (TypeError, ValueError):
+ width, height = 1280, 800
+ width = max(320, min(3840, width)) & ~15
+ height = max(180, min(2160, height)) & ~15
+ return width, height
+
+
+class PageScreencast:
+ """Write a browser desk's page into a FIFO as JPEGs for the core's ffmpeg (image2pipe)."""
+
+ def __init__(self, browser, width=PAGE_STREAM_WIDTH, height=800, fps=PAGE_STREAM_FPS, clock=time.monotonic):
+ if not hasattr(os, "mkfifo"):
+ raise ValueError("a browser desk's stream needs a named pipe (Linux or macOS), which this system has not")
+ self.browser = browser
+ self.width = width
+ self.height = height
+ self.fps = max(1, min(10, fps))
+ self.clock = clock
+ self.stop_event = threading.Event()
+ self.thread = None
+ self.dir = tempfile.mkdtemp(prefix="openvibe-page-stream-")
+ self.fifo = os.path.join(self.dir, FIFO_NAME)
+ os.mkfifo(self.fifo, 0o600)
+ self.error = None
+ self.frames = 0
+
+ def start(self):
+ self.thread = threading.Thread(target=self._loop, name="computer-page-stream", daemon=True)
+ self.thread.start()
+ return self
+
+ def _loop(self):
+ fd = None
+ try:
+ # OPENVIBE's ffmpeg opens the read end; wait for it without blocking the plugin's stop.
+ while not self.stop_event.is_set():
+ try:
+ fd = os.open(self.fifo, os.O_WRONLY | os.O_NONBLOCK)
+ break
+ except OSError as e:
+ if e.errno not in (errno.ENXIO, errno.ENOENT):
+ self.error = str(e)
+ return
+ self.stop_event.wait(0.1)
+ if fd is None:
+ return
+ os.set_blocking(fd, False)
+ frame = os.path.join(self.dir, "frame-%d.jpg" % os.getpid())
+ period = 1.0 / self.fps
+ failures = 0
+ while not self.stop_event.is_set():
+ started = self.clock()
+ try:
+ self.browser.frame(frame, self.width, PAGE_STREAM_QUALITY)
+ with open(frame, "rb") as f:
+ data = f.read()
+ view = memoryview(data)
+ while view:
+ try:
+ n = os.write(fd, view)
+ except BlockingIOError:
+ # ffmpeg is behind: drop this frame rather than grow a queue.
+ n = len(view)
+ view = view[n:]
+ self.frames += 1
+ failures = 0
+ except OSError as e:
+ if e.errno == errno.EPIPE:
+ return
+ self.error = str(e)
+ return
+ except Exception as e: # a page that is navigating, a browser that restarted: try again briefly
+ failures += 1
+ if failures >= 10:
+ self.error = str(e)
+ return
+ self.stop_event.wait(max(0.0, period - (self.clock() - started)))
+ finally:
+ if fd is not None:
+ try:
+ os.close(fd)
+ except OSError:
+ pass
+ self._cleanup()
+
+ def _cleanup(self):
+ try:
+ os.unlink(self.fifo)
+ except OSError:
+ pass
+ try:
+ os.rmdir(self.dir)
+ except OSError:
+ pass
+
+ def status(self):
+ return {"fifo": self.fifo, "width": self.width, "height": self.height, "fps": self.fps,
+ "frames": self.frames, "error": self.error}
+
+ def stop(self):
+ self.stop_event.set()
+ thread, self.thread = self.thread, None
+ if thread is not None and thread is not threading.current_thread() and thread.is_alive():
+ thread.join(2.0)
+ self._cleanup()
new file mode 100644
new file mode 100644
@@ -0,0 +1,292 @@
@@ -0,0 +1,292 @@
+"""The agent desk stream (Agent Streams S1): the curtain's triggers and its fail-closed rule, the capture each platform
+chooses, a browser desk's screencast into its FIFO, and the plugin's desk.stream start/stop/status."""
+import io
+import os
+import shutil
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from types import SimpleNamespace
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+sys.path.insert(0, os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(HERE)), "sdk"))
+
+import openvibe_computer # noqa: E402
+from openvibe_computer import Computer # noqa: E402
+from openvibe_computer.curtain import (Curtain, EnvPrintDetector, PasswordFieldDetector, SecretFileDetector,
+ SiteDetector, looks_secret_name, looks_secret_window) # noqa: E402
+from openvibe_computer.stream import PageScreencast, desktop_capture # noqa: E402
+from openvibe_plugin import Fault # noqa: E402
+
+
+class Clock:
+ def __init__(self):
+ self.t = 0.0
+
+ def __call__(self):
+ return self.t
+
+
+class Answer:
+ def __init__(self, value):
+ self.value = value
+
+ def poll(self):
+ return self.value
+
+
+class Boom:
+ reason = "password_field"
+
+ def poll(self):
+ raise RuntimeError("the accessibility bus went away")
+
+
+class FakeDesk:
+ kind = "desktop"
+
+ def __init__(self, display=":90"):
+ self.display = display
+
+ def available(self):
+ return True
+
+ def window_manager(self):
+ return "metacity"
+
+ def running(self):
+ return True
+
+ def status(self):
+ return {"running": True, "display": self.display, "width": 800, "height": 600, "kind": "desktop"}
+
+ def env(self):
+ return {"DISPLAY": self.display, "DBUS_SESSION_BUS_ADDRESS": "unix:path=/tmp/ov-test"}
+
+
+def curtain(detectors=(), clock=None):
+ clock = clock or Clock()
+ return Curtain(list(detectors), clock=clock, clear_hold=2.0), clock
+
+
+class CurtainTest(unittest.TestCase):
+ def test_site_detector_follows_a_navigation_it_did_not_see(self):
+ nav = {"url": "https://example.com/a"}
+ c, clock = curtain([SiteDetector(["https://example.com"], current_url=lambda: nav["url"])])
+ self.assertFalse(c.evaluate())
+ nav["url"] = "https://bank.example/login"
+ self.assertTrue(c.evaluate())
+ nav["url"] = "https://docs.example.com/"
+ clock.t = 10.0
+ self.assertTrue(c.evaluate())
+ clock.t = 12.1
+ self.assertFalse(c.evaluate())
+
+ def test_each_trigger_raises(self):
+ cases = [
+ ("password", PasswordFieldDetector(probe=lambda: True), None),
+ ("secret window", SecretFileDetector(window=lambda: ("~/.env", "")), None),
+ ("secret class", SecretFileDetector(window=lambda: ("Documents", "WM_CLASS(STRING) = keepassxc")), None),
+ ("env", EnvPrintDetector(), ("env", "HOME=/root\nPATH=/bin\n")),
+ ("site", SiteDetector(["https://example.com"]), "https://evil.example/x"),
+ ]
+ for name, det, action in cases:
+ with self.subTest(name=name):
+ c, clock = curtain([det])
+ if isinstance(det, EnvPrintDetector):
+ det.saw_output(list(action), "HOME=/root\nPATH=/bin\n")
+ elif isinstance(det, SiteDetector):
+ det.saw_url(action)
+ self.assertTrue(c.evaluate(), name)
+ self.assertTrue(c.snapshot()["up"], name)
+ c, _ = curtain([])
+ c.raise_("manual")
+ self.assertTrue(c.evaluate())
+ self.assertEqual(c.snapshot()["reason"], "manual")
+
+ def test_secret_names(self):
+ for name in (".env", "/home/a/.env.local", "/home/a/credentials.json", "id_rsa", "/k/server.pem", "/k/x.key"):
+ self.assertTrue(looks_secret_name(name), name)
+ for name in ("notes.txt", "/home/a/environment.md", "keyboard.png"):
+ self.assertFalse(looks_secret_name(name), name)
+ self.assertTrue(looks_secret_window("Passwords — Bitwarden"))
+ self.assertFalse(looks_secret_window("OpenVibe — Dashboard"))
+
+ def test_detector_exception_is_fail_closed(self):
+ c, _ = curtain([Boom()])
+ self.assertTrue(c.evaluate())
+ self.assertTrue(c.snapshot()["up"])
+ self.assertEqual(c.snapshot()["reason"], "detector")
+
+ def test_unsure_is_not_clear(self):
+ c, _ = curtain([PasswordFieldDetector(probe=lambda: None)])
+ self.assertTrue(c.evaluate())
+ self.assertEqual(c.snapshot()["reason"], "password_field")
+
+ def test_clear_for_two_seconds_before_down(self):
+ state = {"pw": True}
+ c, clock = curtain([PasswordFieldDetector(probe=lambda: state["pw"])])
+ self.assertTrue(c.evaluate())
+ state["pw"] = False
+ clock.t = 10.0
+ self.assertTrue(c.evaluate(), "the hold starts now")
+ clock.t = 11.9
+ self.assertTrue(c.evaluate(), "still inside the two seconds")
+ clock.t = 12.1
+ self.assertFalse(c.evaluate(), "two clear seconds lower it")
+ self.assertFalse(c.snapshot()["up"])
+
+ def test_owner_lowers_now(self):
+ c, _ = curtain([])
+ c.raise_("manual")
+ self.assertTrue(c.snapshot()["up"])
+ c.lower()
+ self.assertFalse(c.snapshot()["up"])
+ # But the next pass still has to see every detector clear; a raised detector raises it again.
+ c.add(Answer("site"))
+ self.assertTrue(c.evaluate())
+
+ def test_change_callback_reports_the_reason_never_a_detector_error(self):
+ seen = []
+ c = Curtain([Boom()], clock=Clock(), on_change=lambda up, why: seen.append((up, why)))
+ c.evaluate()
+ self.assertEqual(seen, [(True, "detector")])
+ self.assertNotIn("accessibility", seen[0][1])
+
+
+class CaptureTest(unittest.TestCase):
+ def test_each_platform(self):
+ self.assertEqual(desktop_capture("linux", {"DISPLAY": ":99"}, 1600, 900),
+ {"kind": "x11grab", "display": ":99.0", "width": 1600, "height": 896})
+ self.assertEqual(desktop_capture("win32", {}, 1600, 900)["kind"], "ddagrab")
+ self.assertEqual(desktop_capture("darwin", {}, 1600, 900)["kind"], "avfoundation")
+
+ def test_no_display_is_refused(self):
+ with self.assertRaises(ValueError):
+ desktop_capture("linux", {}, 1280, 800)
+
+ def test_browser_screencast_writes_jpeg_frames(self):
+ if not hasattr(os, "mkfifo"):
+ self.skipTest("a named pipe is Linux/macOS only")
+ try:
+ from PIL import Image
+ except ImportError:
+ self.skipTest("pillow not installed")
+ tmp = tempfile.mkdtemp(prefix="ov-page-")
+ self.addCleanup(lambda: shutil.rmtree(tmp, ignore_errors=True))
+ jpeg = os.path.join(tmp, "one.jpg")
+ Image.new("RGB", (64, 48), (10, 20, 30)).save(jpeg, "JPEG")
+
+ class FakeBrowser:
+ def frame(self, path, width, quality):
+ shutil.copy(jpeg, path)
+
+ cast = PageScreencast(FakeBrowser(), width=64, height=48, fps=10)
+ got = []
+
+ def read_one():
+ with open(cast.fifo, "rb") as f:
+ got.append(f.read(4))
+
+ reader = threading.Thread(target=read_one)
+ reader.start()
+ cast.start()
+ deadline = time.time() + 5
+ while not got and time.time() < deadline:
+ time.sleep(0.02)
+ cast.stop()
+ reader.join(2)
+ self.assertTrue(got and got[0][:2] == b"\xff\xd8", "a JPEG frame reached the FIFO")
+
+ def test_capture_kinds(self):
+ from openvibe_computer.stream import capture_kinds
+ self.assertEqual(capture_kinds("win32"), ["ddagrab"])
+ self.assertEqual(capture_kinds("darwin"), ["avfoundation"])
+ self.assertIn("x11grab", capture_kinds("linux"))
+
+
+class PluginTest(unittest.TestCase):
+ def make(self, **config):
+ self.dir = tempfile.mkdtemp(prefix="ov-stream-")
+ self.addCleanup(lambda: shutil.rmtree(self.dir, ignore_errors=True))
+ p = Computer(which=lambda n: "/usr/bin/" + n if n in ("Xvfb", "xdotool", "import", "xprop") else None,
+ env={"DISPLAY": ":0"}, platform="linux")
+ p.setup({"level": "trusted", "roots": [self.dir], **config})
+ p.desks.desks["private"] = FakeDesk()
+ # The detectors need a real accessibility stack and display; the plugin's start path is what this tests.
+ p._desk_stream_detectors = lambda allow_sites, env=None, current_url=None: {}
+ return p
+
+ def cmd(self, value):
+ return SimpleNamespace(kind="desk.stream", value=value, id="c1", deadline=time.monotonic() + 30)
+
+ def test_describe_offers_desk_stream(self):
+ p = self.make()
+ caps = p.describe({})
+ self.assertIn("desk.stream", caps)
+ self.assertIn("x11grab", caps["desk.stream"]["capture"])
+ off = Computer(which=lambda n: "/usr/bin/" + n if n in ("Xvfb",) else None, env={"DISPLAY": ":0"}, platform="linux")
+ off.setup({"level": "trusted", "roots": [self.dir], "allow_desk_stream": False})
+ self.assertNotIn("desk.stream", off.describe({"allow_desk_stream": False, "roots": [self.dir]}))
+
+ def test_start_status_stop(self):
+ p = self.make()
+ started = p.handle(self.cmd({"action": "start", "desk": "private", "title": "t"}))
+ self.assertTrue(started["streaming"])
+ self.assertEqual(started["capture"]["kind"], "x11grab")
+ self.assertEqual(started["capture"]["display"], ":90.0")
+ self.assertTrue(started["curtain"]["up"], "the curtain starts up (fail closed)")
+ self.assertTrue(p.handle(self.cmd({"action": "status"}))["streaming"])
+ p.handle(self.cmd({"action": "curtain", "up": False}))
+ self.assertFalse(p.handle(self.cmd({"action": "status"}))["curtain"]["up"])
+ self.assertTrue(p.handle(self.cmd({"action": "stop"}))["stopped"])
+ self.assertFalse(p.handle(self.cmd({"action": "status"}))["streaming"])
+
+ def test_start_is_refused_when_off(self):
+ p = self.make(allow_desk_stream=False)
+ with self.assertRaises(Fault) as e:
+ p.handle(self.cmd({"action": "start", "desk": "private"}))
+ self.assertEqual(e.exception.code, "unsupported")
+
+ def test_curtain_requires_a_stream(self):
+ p = self.make()
+ with self.assertRaises(Fault) as e:
+ p.handle(self.cmd({"action": "curtain", "up": True}))
+ self.assertEqual(e.exception.code, "not_ready")
+
+ def test_a_running_stream_beats_its_curtain_state(self):
+ old = openvibe_computer.DESK_STREAM_HEARTBEAT_S
+ openvibe_computer.DESK_STREAM_HEARTBEAT_S = 0.02
+ self.addCleanup(lambda: setattr(openvibe_computer, "DESK_STREAM_HEARTBEAT_S", old))
+ p = self.make()
+ events = []
+ p.emit_event = lambda name, **fields: events.append((name, fields))
+ p.handle(self.cmd({"action": "start", "desk": "private"}))
+ deadline = time.time() + 2
+ while sum(1 for n, f in events if n == "curtain" and f.get("heartbeat")) < 2 and time.time() < deadline:
+ time.sleep(0.01)
+ beats = [f for n, f in events if n == "curtain" and f.get("heartbeat")]
+ self.assertGreaterEqual(len(beats), 2, events)
+ self.assertTrue(beats[0]["up"], "the first beat reports the fail-closed curtain up")
+ p.handle(self.cmd({"action": "stop"}))
+ n = len(beats)
+ time.sleep(0.1)
+ self.assertEqual(len([f for x, f in events if x == "curtain" and f.get("heartbeat")]), n,
+ "the heartbeat stops with the stream")
+
+ def test_secret_read_raises_the_curtain(self):
+ p = self.make()
+ p.handle(self.cmd({"action": "start", "desk": "private"}))
+ p._desk_stream_detectors = lambda allow_sites, env=None, current_url=None: {}
+ p._desk_stream_saw_path("/home/a/.env")
+ # No secret detector in this injected set: nothing changes, which is why the real set always includes one.
+ self.assertTrue(p.handle(self.cmd({"action": "status"}))["streaming"])
+ p.handle(self.cmd({"action": "stop"}))
+
+
+if __name__ == "__main__":
+ unittest.main()
new file mode 100755
new file mode 100755
@@ -0,0 +1,168 @@
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Headless viewer for the Agent Streams S1 latency probe.
+
+It joins a WebRTC stream over WHEP (the design's ``openvibe-node stream --probe`` publishes a QR of the millisecond
+clock it drew each frame), decodes that QR from every video frame, and reports the glass-to-glass latency:
+``now_ms - clock_in_the_QR``. The pass line is p95 under 1 s over 60 s.
+
+ scripts/stream-viewer.sh --whep https://openvibe.live/whep/agent/ags_… --seconds 60
+
+The view ticket is a secret: it comes from OPENVIBE_VIEW_TICKET (or ``--ticket-stdin``), never from argv. A WHEP
+endpoint is what this needs; Live's S1 player speaks mediasoup over /ws/broadcast, so until Live exposes WHEP the
+number is measured through a relay that does (the design allows a later WHEP endpoint for third-party players).
+
+Requires aiortc and OpenCV (scripts/stream-viewer.sh puts them in a cache venv). ``--self-test`` checks the statistics
+without a network.
+"""
+import argparse
+import asyncio
+import json
+import os
+import sys
+import time
+
+PASS_P95_MS = 1000.0
+
+
+class LatencyReport:
+ def __init__(self):
+ self.samples = []
+
+ def add(self, sent_ms, now_ms=None):
+ now_ms = time.time() * 1000.0 if now_ms is None else now_ms
+ delta = now_ms - sent_ms
+ if 0 <= delta <= 60_000:
+ self.samples.append(delta)
+ return delta
+ return None
+
+ def summary(self):
+ if not self.samples:
+ return {"frames": 0, "p50_ms": None, "p95_ms": None, "max_ms": None}
+ s = sorted(self.samples)
+ def pct(p):
+ return s[min(len(s) - 1, int(round(p / 100.0 * (len(s) - 1))))]
+ return {"frames": len(s), "p50_ms": round(pct(50), 1), "p95_ms": round(pct(95), 1),
+ "max_ms": round(s[-1], 1)}
+
+
+def self_test():
+ r = LatencyReport()
+ for ms in range(1, 101):
+ r.add(0, ms)
+ got = r.summary()
+ assert got["frames"] == 100 and got["p95_ms"] == 95.0, got
+ assert r.add(10_000, 1_000) is None, "a QR from the future is not a sample"
+ try:
+ import cv2 # noqa: F401
+ import numpy # noqa: F401
+ except ImportError as e:
+ print("self-test: statistics ok, QR decoder missing: %s" % e)
+ return 1
+ print("self-test: statistics and QR decoder ok")
+ return 0
+
+
+async def run_whep(args):
+ try:
+ import cv2
+ import numpy as np
+ from aiortc import RTCPeerConnection, RTCSessionDescription
+ except ImportError as e:
+ print("missing a viewer dependency (%s): run scripts/stream-viewer.sh" % e, file=sys.stderr)
+ return 2
+ import aiohttp
+
+ ticket = args.ticket
+ headers = {"Content-Type": "application/sdp", "Accept": "application/sdp"}
+ if ticket:
+ headers["Authorization"] = "Bearer " + ticket
+
+ pc = RTCPeerConnection()
+ report = LatencyReport()
+ decoder = cv2.QRCodeDetector()
+ seen = {"frames": 0, "qr": 0}
+
+ @pc.on("track")
+ def on_track(track):
+ if track.kind != "video":
+ return
+
+ async def consume():
+ while True:
+ try:
+ frame = await track.recv()
+ except Exception:
+ return
+ seen["frames"] += 1
+ try:
+ img = frame.to_ndarray(format="bgr24")
+ except Exception:
+ continue
+ ok, decoded, *_ = decoder.detectAndDecode(img)
+ if not ok or not decoded:
+ continue
+ try:
+ sent = int(decoded.strip())
+ except ValueError:
+ continue
+ seen["qr"] += 1
+ report.add(sent)
+ asyncio.ensure_future(consume())
+
+ pc.addTransceiver("video", direction="recvonly")
+ offer = await pc.createOffer()
+ await pc.setLocalDescription(offer)
+ # Wait for the ICE gathering to finish (a few seconds), then send the complete offer.
+ deadline = time.monotonic() + 10
+ while pc.iceGatheringState != "complete" and time.monotonic() < deadline:
+ await asyncio.sleep(0.1)
+ async with aiohttp.ClientSession() as session:
+ async with session.post(args.whep, data=pc.localDescription.sdp, headers=headers) as resp:
+ body = await resp.text()
+ if resp.status not in (200, 201):
+ print("WHEP answered HTTP %d: %s" % (resp.status, body[:200]), file=sys.stderr)
+ await pc.close()
+ return 3
+ await pc.setRemoteDescription(RTCSessionDescription(body, "answer"))
+ print("viewer: joined %s; measuring for %ds" % (args.whep, args.seconds), file=sys.stderr)
+ try:
+ await asyncio.sleep(args.seconds)
+ finally:
+ await pc.close()
+ out = report.summary()
+ out.update({"seconds": args.seconds, "video_frames": seen["frames"], "qr_frames": seen["qr"],
+ "p95_under_1s": out["p95_ms"] is not None and out["p95_ms"] < args.max_p95_ms})
+ if args.json:
+ print(json.dumps(out))
+ else:
+ if out["frames"] == 0:
+ print("no QR frame decoded in %ds: is --probe publishing to this stream?" % args.seconds, file=sys.stderr)
+ else:
+ print("glass-to-glass latency over %ds: p50 %.1f ms, p95 %.1f ms, max %.1f ms (%d frames)" %
+ (args.seconds, out["p50_ms"], out["p95_ms"], out["max_ms"], out["frames"]))
+ print("S1 pass line: p95 under %.0f ms → %s" % (args.max_p95_ms, "PASS" if out["p95_under_1s"] else "FAIL"))
+ return 0 if out["p95_under_1s"] else 1
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--whep", help="the WHEP endpoint of the stream")
+ ap.add_argument("--seconds", type=int, default=60)
+ ap.add_argument("--json", action="store_true")
+ ap.add_argument("--self-test", action="store_true")
+ ap.add_argument("--max-p95-ms", type=float, default=PASS_P95_MS)
+ ap.add_argument("--ticket-stdin", action="store_true", help="read the view ticket from one line of stdin")
+ args = ap.parse_args()
+ if args.self_test:
+ return self_test()
+ if not args.whep:
+ ap.error("--whep is required (or --self-test)")
+ args.ticket = os.environ.get("OPENVIBE_VIEW_TICKET", "")
+ if not args.ticket and args.ticket_stdin:
+ args.ticket = sys.stdin.readline().strip()
+ return asyncio.run(run_whep(args))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
new file mode 100755
new file mode 100755
@@ -0,0 +1,26 @@
@@ -0,0 +1,26 @@
+#!/bin/sh
+# The latency viewer's dependencies in a cache venv, then the viewer. See scripts/stream-viewer.py.
+set -eu
+
+here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cache=${XDG_CACHE_HOME:-$HOME/.cache}/openvibe-stream-viewer
+venv=$cache/venv
+
+if [ ! -x "$venv/bin/python" ]; then
+ command -v python3 >/dev/null || { echo "stream-viewer: python3 is required" >&2; exit 2; }
+ mkdir -p "$cache"
+ python3 -m venv "$venv" || {
+ echo "stream-viewer: could not create a venv under $cache; install python3-venv" >&2
+ exit 2
+ }
+fi
+
+if ! "$venv/bin/python" -c 'import aiortc, cv2, numpy' >/dev/null 2>&1; then
+ echo "stream-viewer: installing aiortc and OpenCV once into $venv" >&2
+ "$venv/bin/python" -m pip install --quiet --disable-pip-version-check aiortc opencv-python-headless numpy || {
+ echo "stream-viewer: pip install failed (no network?)" >&2
+ exit 2
+ }
+fi
+
+exec "$venv/bin/python" "$here/stream-viewer.py" "$@"