openvibers/openvibe.link Commits

Fix the independent review finding in the existing uncommitted… (#16)

OP OpenVibers committed 4 Oct 2026 · 79ccefcd99 · 19 files, +3294 −172

local checks ✓ 1/1 · CI ✓ 9/9 · review SHIP by OpenCode Qwen3.7 Plus

https://github.com/OpenVibers/OpenVibe.Node/pull/16

Changes

ddiff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
iindex 13c0a39..08d12eb 100644
---- a/.github/workflows/ci.yml
++++ b/.github/workflows/ci.yml
@@@ -24,12 +24,23 @@ jobs:
- run: pip install av numpy
- run: test -z "$(gofmt -l .)" || (gofmt -l . && exit 1)
- run: go vet ./...
-- # The job worker isolates jobs in unprivileged user and network namespaces, which Ubuntu's AppArmor restricts by
-- # default; its tests skip where they cannot run, and OPENVIBE_WORKER_TESTS=require makes them fail here instead.
++ # The job worker's tests skip where jobs cannot run sandboxed: unprivileged, the runner's cgroup is not delegated
++ # (and Ubuntu's AppArmor restricts unprivileged user namespaces).
- run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
- run: go test -race -count=1 ./...
-- env:
-- OPENVIBE_WORKER_TESTS: require
++ # So they run again as root (jobs as nobody), in a systemd scope with its cgroup delegated, where
++ # OPENVIBE_WORKER_TESTS=require makes them fail instead of skipping. Each test binary lies alone in a directory
++ # nobody can read, as a function's executable must (a job gets a copy of that directory).
++ - name: worker sandbox tests (root)
++ run: |
++ d=$(mktemp -d) && chmod 755 "$d"
++ for t in 'worker .' 'node ^(TestJobRunsOverTheLink|TestLocalStopEndsJobs)$'; do
++ set -- $t
++ mkdir -m 755 "$d/$1"
++ go test -race -c -o "$d/$1/$1.test" ./internal/$1
++ (cd internal/$1 && sudo systemd-run --scope --quiet -p Delegate=yes \
++ env OPENVIBE_WORKER_TESTS=require "$d/$1/$1.test" -test.count=1 -test.v -test.run "$2")
++ done
build:
name: cross-compile
ddiff --git a/docs/worker.md b/docs/worker.md
iindex 86570a1..84ef7a9 100644
---- a/docs/worker.md
++++ b/docs/worker.md
@@@ -13,17 +13,25 @@ the server. The worker is **off by default**; off, the Node refuses every job `c
"worker": {
"enabled": false,
"functions": [
-- {"name": "thumbnail", "version": "1.2.0", "command": ["/opt/fn/thumbnail/bin/run"], "env": {"MODE": "fast"}}
++ {"name": "thumbnail", "version": "1.2.0", "command": ["/opt/fn/thumbnail/bin/run"], "env": {"MODE": "fast"},
++ "artifact_dir": "/opt/fn/thumbnail"}
],
"run_as": {"uid": 2001, "gid": 2001},
"allow_same_user": false,
++ "egress": "none",
"caps": {
"max_ttl_ms": 600000,
"max_wall_ms": 300000,
"max_cpu_ms": 300000,
"max_mem_bytes": 536870912,
"max_output_bytes": 1048576,
-- "max_jobs": 1
++ "max_jobs": 1,
++ "max_cpu_millis": 1000,
++ "max_pids": 64,
++ "max_disk_bytes": 67108864,
++ "max_io_bps": 67108864,
++ "max_io_iops": 1000,
++ "max_vm_bytes": 8589934592
}
}
```
@@@ -31,42 +39,101 @@ the server. The worker is **off by default**; off, the Node refuses every job `c
- `functions`: `name` and `version` match a job's `artifact` exactly (the contract's patterns; listed once each);
`command[0]` is an absolute path. `env` is added to the job's environment. A per-entry `class` (`function`, the
default, or `code`) is planned and not accepted yet: the config loader refuses unknown keys, so leave it out
-- until the `code` class ships; only `function` is advertised in `status.capabilities.worker` today.
++ until the `code` class ships; only `function` is advertised in `status.capabilities.worker` today. `artifact_dir` (default: the directory
++ of `command[0]`) is the one host directory a job gets besides the system paths, as a **copy** made as the job
++ starts, read-only, at the same path: nothing the host adds to it or changes in it later reaches the job (a Unix
++ socket or a FIFO that appears there, say, through which a job could reach a host process). The Node opens the
++ directory and checks where it leads at that moment; the job's sandbox opens it again by its path, as the job's user
++ (who must be able to search every directory above it and list it, or the job fails), checks that it is the
++ directory the Node checked (or the job fails), and copies it from there, as the job's user, into a tmpfs of at most `max_disk_bytes` (it counts towards the job's memory, and its reads towards its
++ `io.max`; it must be done within the 10 s the sandbox has to stand). The copy holds directories, regular files and
++ symlinks (copied as symlinks, never followed) with their permission bits and modification times, without owner,
++ set-id or sticky bits; what the job's user may not read is left out; a socket, FIFO or device in the tree, or a
++ mount nested in it (a bind mount of the same filesystem included), fails the job. `artifact_dir` must not be `/`;
++ nor be, hold or lie in the Node's config or state directory, `/proc`, `/sys`, `/dev`, `/run`, `/var/run`,
++ `/var/lock`, `/etc`, `/boot` or `/root`; nor be or hold `/tmp`, `/var/tmp` or `/home`; nor hold a socket, FIFO or
++ device at boot (the tree is walked then, so the probe says so before any job). The probe refuses the whole worker
++ otherwise, and a job of such a function ends `failed`. A directory in `/usr` (or another system path; the default
++ for a command in `/usr/local/bin`, say) is copied too, over what the bind of `/usr` shows there, at the path it leads
++ to in the job's root, and the job fails unless its own path to it leads to the copy: point `artifact_dir` at the
++ function's own directory, since a copy of all of `/usr/bin` does not fit `max_disk_bytes`. Every directory of a
++ system path that its path crosses (as the job follows it: through `/lib` to `/usr/lib` or a symlink in `/usr`, say)
++ must belong to root (or the Node's own user) and be writable by no other user, nor by a group with any other member
++ (listed in `/etc/group` or by primary group in `/etc/passwd`; a group not in `/etc/group` counts as untrusted), or the probe and
++ each job refuse it: there a host user could switch a symlink or rename a directory on the path while the job runs,
++ and lead the job to a live host directory instead of its copy. A missing `artifact_dir` does not fail the probe; its
++ jobs end `failed`.
- `run_as`: the unprivileged uid and gid jobs run as (not 0; no supplementary groups). It needs the Node to run as
root, which the system install does. Create a dedicated user for it with no login and no groups.
--- `allow_same_user`: run jobs as the Node's own (non-root) user instead. A job can then read every file the Node can,
-- `credential.json` included, and use its groups (`gpio`, `dialout`, `video`): **for development only**. With neither
-- `run_as` nor `allow_same_user` the worker stays off.
++- `allow_same_user`: run jobs as the Node's own (non-root) user instead, with its groups (`gpio`, `dialout`,
++ `video`), on what the job's private root holds: **for development only**. With neither `run_as` nor
++ `allow_same_user` the worker stays off.
- `caps`: the owner's local caps. A job's `ttl_ms`, `limits.wall_ms`, `limits.cpu_ms` and `limits.mem_bytes` are
each clamped to them (the stricter value wins, as with `max_command_ms`). `max_output_bytes` caps the stdout bytes
-- one job sends (counted on the wire, after invalid UTF-8 became U+FFFD); `max_jobs` how many run at once.
--
--The class `function` — advertised as the reserved Fabric capability `worker:function` in `status.capabilities.worker`
--(the `platform.resource-offer@1` namespace OpenVibe.Run routes on) — is offered only when `enabled` is true, the OS is
--Linux, at least one `functions` entry is declared and a **boot-time probe** passes: it starts `/bin/sh` the way a job
--is started and checks that the process got user, network and PID namespaces of its own, that its CPU limit can be set and that its CPU and memory can be read. If
--the probe fails the Node logs why (`worker off: …`) and refuses every job, exactly as with the worker off. On Ubuntu
--24.04 and later, unprivileged user namespaces are restricted by AppArmor; a Node running as root with `run_as` is not
--affected.
++ one job sends (counted on the wire, after invalid UTF-8 became U+FFFD); `max_jobs` how many run at once. The
++ kernel enforces the rest on every job: `max_cpu_millis` (thousandths of a core, cgroup `cpu.max`), `max_pids`
++ (processes and threads: `pids.max` and `RLIMIT_NPROC`), `max_disk_bytes` (the size of its `/tmp` and of
++ its copy of `artifact_dir`, and `RLIMIT_FSIZE`), `max_io_bps` and `max_io_iops` (`io.max` read and write, on
++ every whole block device with a medium (a disk whose `dev` or `size` cannot be read in `/sys/block` fails the job
++ and the probe) but the hidden paths of an NVMe multipath disk, which only the limited disk
++ over them reaches: read back from the kernel when the job's cgroup is made and again once its sandbox stands; a device that
++ cannot be limited, or whose `io.max` is not the job's, fails the job and the probe), `max_vm_bytes`
++ (`RLIMIT_AS`, always set: there is no "none"; a runtime that reserves a large address space up front, such as
++ Node.js, the JVM or race-instrumented Go, needs a large value, e.g. 64 TiB for the last). `mem_bytes` is the job cgroup's `memory.max` (swap off: `memory.swap.max` is set to 0 and read back; without swap accounting, which makes that
++ control, the probe and each job are refused unless the host has no swap at all; an OOM kills the whole job).
++- `egress`: what a job may reach on the network. `none` (the default) is the only policy enforced: a network
++ namespace with nothing but a loopback that is down. `public` (internet but no private, link-local or Node
++ addresses) and `openvibe-only` (the platform's endpoints only) need an egress proxy in the job's namespace that is
++ not built yet (follow-up): with either, the probe fails and the worker stays off, never a silent `none` nor an
++ open network.
++
++The class `function` — advertised as the reserved Fabric capability `worker:function` in
++`status.capabilities.worker` (the `platform.resource-offer@1` namespace OpenVibe.Run routes on) — is offered only when `enabled` is true, the OS is Linux
++(amd64 or arm64: the architectures with a seccomp allowlist), at least one `functions` entry is declared and a **boot-time probe** passes: it starts `/bin/sh`
++the way a job is started and checks every control from the host: the sandbox was set up; the process has user,
++mount, network, PID, IPC, UTS and cgroup namespaces of its own, `NoNewPrivs` and a seccomp filter; nothing in its
++root but `/tmp`, `/proc` and the device nodes is mounted writable; it is in its own cgroup with every controller
++(cpu, memory, pids, io) and every block device holds its `io.max`; its rlimits (`RLIMIT_AS` included) are set; its
++root holds none of `/var`, `/etc/shadow`, `/etc/ssl/private`, `/etc/pki/tls/private` and the Node's directories; its CPU limit can be set and its usage read; `egress` is `none`; no function's `artifact_dir`
++is refused (above). If any check fails the Node logs why (`worker off: …`) and refuses every job,
++exactly as with the worker off: there is no weaker mode. A job whose sandbox fails at run time ends `failed` without
++its function having run.
++
++Requirements: Linux 5.14 or later with cgroup v2; the Node's cgroup **delegated** to it (`Delegate=yes` on its
++systemd unit, which the service `openvibe-node install` writes sets; running `install.sh` again rewrites an older
++install's service, or add it by hand: `sudo systemctl edit openvibe-node`, add `[Service]` and `Delegate=yes`,
++restart). The Node moves itself into an `openvibe-node` leaf of that cgroup and makes one `openvibe-job-*` cgroup per
++job beside it; a cgroup that holds any process other than the Node and its children (a login session's, say) is not
++rearranged, and the worker stays off. On Ubuntu 24.04 and later, unprivileged user namespaces are restricted by
++AppArmor; a Node running as root with `run_as` is not affected.
## How a job runs
1. **Admit** (before the `ack`): refused unless the stop latch is clear, the artifact is declared and fewer than
`max_jobs` jobs run. An id the worker already holds is answered from it (`ack` while it runs, its `job_exit` once
it ended) and never starts a second process.
--2. **Start**: a fresh working directory (`$TMPDIR/openvibe-job-*`, owned by `run_as`, removed afterwards); an
-- environment built from scratch (`PATH=/usr/local/bin:/usr/bin:/bin`, `HOME` and `TMPDIR` the working directory,
-- `LANG=C.UTF-8`, `OPENVIBE_JOB_ID`, and the function's `env`; nothing inherited from the Node); `args` as one line of
-- JSON on stdin; stderr discarded. The process starts in its own process group and in new user, network and PID
-- namespaces, with `PR_SET_PDEATHSIG` SIGKILL, then gets `RLIMIT_CPU` (at `cpu_ms`, rounded up to a second) and
-- `RLIMIT_CORE` 0. `job_started` follows only once `RLIMIT_CPU` is set: if it cannot be set the process is killed
-- and the job ends `failed` with no `job_started`.
++2. **Start**: the Node makes the job's cgroup and starts itself again (`/proc/self/exe`) as the **sandbox init**,
++ with its name as its only argument and its spec (the function's command and environment included) in a memfd, so
++ that no host user reads the environment in its `/proc/<pid>/cmdline` (the Node checks that it shows the name
++ alone), straight into that cgroup (`clone3` `CLONE_INTO_CGROUP`), in its own process group and in new user, mount,
++ network, PID, IPC, UTS and cgroup namespaces, with `PR_SET_PDEATHSIG` SIGKILL. The init builds a private root (see
++ below) and `pivot_root`s into it, detaching the host's; sets `RLIMIT_CPU` (at `cpu_ms`, rounded up to a second),
++ `RLIMIT_CORE` 0, `RLIMIT_NOFILE` 1024, `RLIMIT_NPROC`, `RLIMIT_FSIZE` and `RLIMIT_AS`; drops every capability; sets
++ `no_new_privs` and the seccomp filter; reports that it is ready and waits for the Node's go; and only then
++ executes the function, with only fds 0-3 open. The function
++ works in a fresh directory `/tmp/openvibe-job-*` of its own `/tmp`; its environment is built from scratch
++ (`PATH=/usr/local/bin:/usr/bin:/bin`, `HOME` and `TMPDIR` that directory, `LANG=C.UTF-8`, `OPENVIBE_JOB_ID`, and
++ the function's `env`; nothing inherited from the Node); `args` come as one line of JSON on stdin; stderr is
++ discarded. While the init waits (so it is sure to be alive), the Node checks `NoNewPrivs`, the seccomp filter and
++ that nothing but `/tmp`, `/proc` and the device nodes is mounted writable, and sets `RLIMIT_CPU` again from
++ outside; only then does it let the init execute the function. `job_started` follows once that `execve`
++ succeeded: otherwise the process is killed and the job ends `failed` with no `job_started`.
3. **Run**: stdout streams as `job_stdout` chunks of at most 16 KiB each as sent (invalid UTF-8 becomes U+FFFD before
it is cut, and a UTF-8 sequence is never split); a watchdog samples CPU time and resident memory of every process in the
job's PID namespace every 200 ms and sends `job_usage` for every second once it fully elapsed.
4. **End**: the function's result is what it writes to **fd 3** (JSON, at most 256 KiB), reported only when it exits
0. `job_exit` carries the reason, the exit code (null when it was killed) and the usage, and is resent until
-- `job_exit_ack`.
++ `job_exit_ack`. The job's cgroup is then killed (`cgroup.kill`) and removed.
Every kill is SIGKILL to the whole process group and to the leader, which is PID 1 of the job's PID namespace, so its
death also kills every process the job left behind. The watchdog kills at `ttl_ms` (from receipt), `wall_ms` (from
@@@ -80,33 +147,66 @@ them again. A Node shutting down kills its jobs (`stopped`); a job acked but not
## Isolation model and its limits
--In five lines:
++In six lines:
1. Only owner-declared executables run, matched by exact name and version; the server chooses which and with what JSON.
--2. Each job is a separate unprivileged process (`run_as`) in its own process group and user, network and PID namespaces.
--3. The network namespace has only a loopback that is down: no IP network, not even to the Node or `localhost`.
--4. Nothing of the Node's environment reaches it; its working directory is fresh and removed after the job.
--5. ttl, wall, CPU, memory and output are enforced by a per-job watchdog that SIGKILLs the whole group; `RLIMIT_CPU` backs the CPU cap in the kernel.
--
--What it does **not** protect against (no seccomp, no cgroups, no mount namespace yet):
--
--- **Files**: there is no filesystem isolation. A job reads and writes whatever its uid may: with `run_as`, every
-- world-readable file and world-writable directory; with `allow_same_user`, everything the Node can, including
-- `credential.json` and the local control socket (so it could clear the local stop).
--- **Unix sockets by path** (Docker's, D-Bus's, the Node's control socket) are reachable: a network namespace does not
-- cover filesystem sockets; only their file permissions do.
--- **The kernel**: no seccomp filter, so the whole syscall surface is open, including what user namespaces expose to
-- unprivileged code (a common source of local privilege escalations). Keep the kernel patched.
--- **Memory and CPU are sampled**, not enforced by cgroups: a burst between two 200 ms samples can exceed `mem_bytes`
-- (and meet the kernel's OOM killer first); the `RLIMIT_CPU` backstop is set just after start (before `job_started`)
-- and covers each process on its own, not their sum.
--- **No limit on processes, threads, disk or I/O**: a fork bomb is bounded only by the uid's `RLIMIT_NPROC`, disk use
-- in the working directory or `/tmp` only by free space; `/tmp` and `/dev/shm` are shared with the machine.
--- **`/proc` is the host's**: a job can list the machine's processes and their command lines (not signal them: its PID
-- namespace has no host PIDs).
++2. Each job is a separate unprivileged process (`run_as`), with no capability, `no_new_privs` and a seccomp
++ allowlist, in its own process group and user, mount, network, PID, IPC, UTS and cgroup namespaces.
++3. Its filesystem is a private, read-only root holding only the system paths, a copy of the function's
++ `artifact_dir` made as it starts and a size-capped `/tmp` of its own: none of the Node's files, its credential or
++ its control socket, nor the host's private keys.
++4. The network namespace has only a loopback that is down: no IP network, not even to the Node or `localhost`
++ (`egress` `none`; the other policies are not enforced yet, so they keep the worker off).
++5. Nothing of the Node's environment reaches it.
++6. CPU, memory, processes and disk I/O are enforced by its cgroup v2 and rlimits; ttl, wall, CPU and output also by a
++ per-job watchdog that SIGKILLs the whole group; `RLIMIT_CPU` backs the CPU cap per process.
++
++**The private root.** A 16 MiB tmpfs, read-only once built, holding: `/usr`, `/bin`, `/sbin`, `/lib*` (recursive
++binds made read-only, `nosuid` and `nodev` with every mount nested in them, by `mount_setattr` `AT_RECURSIVE`;
++or the same symlinks on merged-`/usr` systems); from `/etc` only `passwd`, `group`, `hosts`, `nsswitch.conf`,
++`localtime`, `mime.types`, `ld.so.*` and `alternatives` (a bind that is, holds or leads by a symlink to
++`/etc/ssl/private`, `/etc/pki/tls/private`, `/etc/shadow` or `/var` keeps the worker off); a copy of the public CA
++certificates of `/etc/ssl/certs`, `/etc/ssl/cert.pem`, `/etc/ssl/ca-bundle.pem`, `/etc/pki/tls/certs`,
++`/etc/pki/tls/cacert.pem`, `/etc/pki/ca-trust/extracted` and `/etc/ca-certificates/extracted` (those that exist),
++made for each job as its user and never bound: of each file only the PEM `CERTIFICATE` blocks that parse as X.509
++certificates, written again without anything else (a private key filed among them, in a subdirectory or a mount
++there included, never reaches a job); a file with none (a binary keystore, a `TRUSTED CERTIFICATE` bundle), one the
++job's user may not read, or one over 4 MiB is left out, and symlinks are kept as they are (they lead only to what the
++root holds); the copy of the
++function's `artifact_dir` (read-only); `/dev/null`, `zero`, `full`, `random`, `urandom` and the `fd`/`stdin`/`stdout`/`stderr` links; a fresh
++`/proc` of its own PID namespace; and `/tmp` (also `/dev/shm`), a tmpfs of `max_disk_bytes`, the only writable place.
++There is no `/home`, `/root`, `/var`, `/run`, `/srv`, `/mnt`, `/media`, `/sys`, rest of `/etc`, nor the Node's
++config or state directory (the Node passes both to the worker and the probe checks them).
++
++**The seccomp filter.** A system call of another architecture (or the x32 ABI) kills or fails; `clone` with a
++namespace flag fails, `clone3` fails `ENOSYS` (libc falls back to `clone`); `socket` is limited to Unix, IPv4, IPv6
++and netlink (all scoped by the job's network namespace); the calls on the allowlist (files, memory, signals, threads,
++time, sockets, polling, `execve`) pass; anything else fails `EPERM`: among them `mount`, `umount`, `pivot_root`,
++`chroot`, `unshare`, `setns`, `ptrace`, `process_vm_*`, `bpf`, `perf_event_open`, `keyctl`, `add_key`, `io_uring_*`,
++`userfaultfd`, `open_by_handle_at`, the kernel module and `kexec` calls, `reboot`, `swapon`, `quotactl`, the new mount
++API, `mbind`/`set_mempolicy` and the `pkey_*` calls.
++
++What it does **not** protect against:
++
++- **`allow_same_user`** (development only): the job keeps the Node user's supplementary groups. It still sees only
++ the private root, read-only, and none of the Node's directories.
++- **The kernel**: the seccomp allowlist still leaves the common surface (files, memory, sockets, `ioctl`,
++ `prctl`) and what an unprivileged user namespace allows; keep the kernel patched.
++- **`mem_bytes` and CPU time are also sampled** for the usage report and the `limit` reason; the cgroup enforces
++ memory in the kernel (an OOM kills the whole job and it ends `limit`).
++- **Sockets and FIFOs under the system paths**: the system paths are bound live, not copied; a standard system keeps
++ none under `/usr` or `/lib*` (they live in `/run`, `/tmp` and `/var`, which a job never sees).
++- **Every certificate in the CA directories is seen by every job**, CA or not (a host's own public certificate filed
++ in `/etc/ssl/certs`, say); its key is not. A certificate Go's X.509 parser refuses is left out of the copy.
++- **`io.max` limits block devices only**: I/O to a network filesystem mounted under the system paths is not limited.
++- **Zombies**: the function is PID 1 of its namespace; orphans it does not reap count towards `max_pids` until it
++ exits.
- **The `args` and the result are trusted to the function**: the worker passes the server's JSON unchanged; validating
it is the function's job.
++Not built yet (follow-ups): the `public` and `openvibe-only` egress policies (an egress proxy reached through a veth
++or a socket bound into the job's namespace, filtering by destination).
++
## Refusals and exit reasons
Refused with `nack` (keyed by the job id; the first matching row wins, after the checks every Node does, in
@@@ -127,6 +227,6 @@ Accepted jobs end with `job_exit.reason`:
| `exited` | the process ended by itself; `code` is its status (null if a signal the worker did not send ended it) |
| `cancelled` | `job_cancel`; a job cancelled before it started never starts |
| `ttl` | `ttl_ms` (clamped) ran out, before or while it ran |
--| `limit` | `wall_ms`, `cpu_ms` (sampled, or `RLIMIT_CPU`'s SIGXCPU) or `mem_bytes` exceeded, stdout past `max_output_bytes`, or a result over 256 KiB |
++| `limit` | `wall_ms`, `cpu_ms` (sampled, or `RLIMIT_CPU`'s SIGXCPU) or `mem_bytes` (sampled, or the cgroup's OOM kill) exceeded, stdout past `max_output_bytes`, or a result over 256 KiB |
| `stopped` | the e-stop or local stop latched, or the Node shut down |
--| `failed` | the job could not run isolated: namespaces not created, spawn error, working directory not made, CPU limit not set; never run with less isolation |
++| `failed` | the job could not run isolated: sandbox, cgroup or namespaces not set up, spawn error, working directory not made, CPU limit not set; never run with less isolation |
ddiff --git a/install/install.sh b/install/install.sh
iindex 56a7eb3..6af838e 100755
---- a/install/install.sh
++++ b/install/install.sh
@@@ -301,6 +301,9 @@ disable_stock_server
# ---- service ----
if [ "$SERVICE" = 1 ]; then
++ # An upgrade rewrites the service definition (this version's adds Delegate=yes, which the worker needs): the
++ # service manager refuses to install over an existing one. Config and credential are kept.
++ $SUDO "$BIN_DIR/openvibe-node" uninstall >/dev/null 2>&1 || true
if [ -n "$SVC_USER" ]; then
$SUDO "$BIN_DIR/openvibe-node" install --user "$SVC_USER" >/dev/null 2>&1 || $SUDO "$BIN_DIR/openvibe-node" install --user "$SVC_USER"
else
ddiff --git a/internal/config/config.go b/internal/config/config.go
iindex bd5ce13..8bee62a 100644
---- a/internal/config/config.go
++++ b/internal/config/config.go
@@@ -119,18 +119,45 @@ type WorkerConfig struct {
// RunAs is the dedicated unprivileged uid/gid jobs run as; it needs the Node to run as root. Without it the Node
// refuses to run jobs unless AllowSameUser is set.
RunAs *RunAs `json:"run_as,omitempty"`
-- // AllowSameUser runs jobs as the Node's own user (no run_as). A job can then read every file the Node can,
-- // including credential.json, and use the Node user's groups (gpio, dialout, video): for development only.
++ // AllowSameUser runs jobs as the Node's own user (no run_as), with its groups (gpio, dialout, video), on what the
++ // job's private root holds: for development only.
AllowSameUser bool `json:"allow_same_user,omitempty"`
Caps WorkerCaps `json:"caps,omitzero"`
++ // Egress is what a job may reach on the network: none (the default; a network namespace with only a loopback that
++ // is down), public or openvibe-only. Only none is enforced yet: with another value the worker stays off.
++ Egress string `json:"egress,omitempty"`
++ // NodeDirs are the Node's own directories (config, credential, state, control socket), which no job may see; the
++ // Node sets them, the config file does not.
++ NodeDirs []string `json:"-"`
}
++// The worker.egress policies.
++const (
++ EgressNone = "none"
++ EgressPublic = "public"
++ EgressOpenVibeOnly = "openvibe-only"
++)
++
// FunctionConfig is one function a job may run: Command is started with the job's args as JSON on stdin.
type FunctionConfig struct {
Name string `json:"name"`
Version string `json:"version"`
Command []string `json:"command"` // Command[0] is an absolute path
Env map[string]string `json:"env,omitempty"`
++ // ArtifactDir is the directory each job gets a read-only copy of, made as it starts, in its private root (default:
++ // the directory of Command[0]). It must not be / nor hold, or lie inside, the Node's own directories.
++ ArtifactDir string `json:"artifact_dir,omitempty"`
++}
++
++// Artifact is the directory a job of the function gets a read-only copy of.
++func (f FunctionConfig) Artifact() string {
++ if f.ArtifactDir != "" {
++ return filepath.Clean(f.ArtifactDir)
++ }
++ if len(f.Command) == 0 {
++ return ""
++ }
++ return filepath.Dir(f.Command[0])
}
// RunAs is a uid and gid on this machine.
@@@ -148,17 +175,31 @@ type WorkerCaps struct {
MaxMemBytes int64 `json:"max_mem_bytes,omitempty"`
MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` // stdout of one job, in bytes
MaxJobs int `json:"max_jobs,omitempty"` // jobs running at once
++ // The kernel's limits on every job (cgroup v2 and rlimits): CPU in thousandths of a core (cpu.max), processes and
++ // threads (pids.max and RLIMIT_NPROC), bytes in its /tmp (tmpfs size and RLIMIT_FSIZE), disk I/O per block
++ // device (io.max, bytes and operations per second), and address space (RLIMIT_AS, always set: a runtime that
++ // reserves a large address space up front, such as Node.js, the JVM or a race-instrumented binary, needs a large
++ // value, never none).
++ MaxCPUMillis int64 `json:"max_cpu_millis,omitempty"`
++ MaxPids int64 `json:"max_pids,omitempty"`
++ MaxDiskBytes int64 `json:"max_disk_bytes,omitempty"`
++ MaxIOBps int64 `json:"max_io_bps,omitempty"`
++ MaxIOPS int64 `json:"max_io_iops,omitempty"`
++ MaxVMBytes int64 `json:"max_vm_bytes,omitempty"`
}
// DefaultWorkerCaps are the caps for every field the config leaves at zero.
var DefaultWorkerCaps = WorkerCaps{MaxTTLMS: 600000, MaxWallMS: 300000, MaxCPUMS: 300000, MaxMemBytes: 512 << 20,
-- MaxOutputBytes: 1 << 20, MaxJobs: 1}
++ MaxOutputBytes: 1 << 20, MaxJobs: 1, MaxCPUMillis: 1000, MaxPids: 64, MaxDiskBytes: 64 << 20, MaxIOBps: 64 << 20,
++ MaxIOPS: 1000, MaxVMBytes: 8 << 30}
// WithDefaults fills the zero fields of c from DefaultWorkerCaps.
func (c WorkerCaps) WithDefaults() WorkerCaps {
d := DefaultWorkerCaps
for _, f := range []struct{ v, def *int64 }{{&c.MaxTTLMS, &d.MaxTTLMS}, {&c.MaxWallMS, &d.MaxWallMS},
-- {&c.MaxCPUMS, &d.MaxCPUMS}, {&c.MaxMemBytes, &d.MaxMemBytes}, {&c.MaxOutputBytes, &d.MaxOutputBytes}} {
++ {&c.MaxCPUMS, &d.MaxCPUMS}, {&c.MaxMemBytes, &d.MaxMemBytes}, {&c.MaxOutputBytes, &d.MaxOutputBytes},
++ {&c.MaxCPUMillis, &d.MaxCPUMillis}, {&c.MaxPids, &d.MaxPids}, {&c.MaxDiskBytes, &d.MaxDiskBytes},
++ {&c.MaxIOBps, &d.MaxIOBps}, {&c.MaxIOPS, &d.MaxIOPS}, {&c.MaxVMBytes, &d.MaxVMBytes}} {
if *f.v <= 0 {
*f.v = *f.def
}
@@@ -321,12 +362,21 @@ func (w WorkerConfig) validate() error {
if len(f.Command) == 0 || !filepath.IsAbs(f.Command[0]) {
return fmt.Errorf("config: worker function %s@%s needs a command starting with an absolute path", f.Name, f.Version)
}
++ if f.ArtifactDir != "" && (!filepath.IsAbs(f.ArtifactDir) || filepath.Clean(f.ArtifactDir) == "/") {
++ return fmt.Errorf("config: worker function %s@%s: artifact_dir must be an absolute path other than /", f.Name, f.Version)
++ }
++ }
++ switch w.Egress {
++ case "", EgressNone, EgressPublic, EgressOpenVibeOnly:
++ default:
++ return fmt.Errorf("config: worker.egress %q is not none, public or openvibe-only", w.Egress)
}
if w.RunAs != nil && (w.RunAs.UID == 0 || w.RunAs.GID == 0) {
return errors.New("config: worker.run_as must not be root (uid or gid 0)")
}
c := w.Caps
-- if c.MaxTTLMS < 0 || c.MaxWallMS < 0 || c.MaxCPUMS < 0 || c.MaxMemBytes < 0 || c.MaxOutputBytes < 0 || c.MaxJobs < 0 {
++ if c.MaxTTLMS < 0 || c.MaxWallMS < 0 || c.MaxCPUMS < 0 || c.MaxMemBytes < 0 || c.MaxOutputBytes < 0 || c.MaxJobs < 0 ||
++ c.MaxCPUMillis < 0 || c.MaxPids < 0 || c.MaxDiskBytes < 0 || c.MaxIOBps < 0 || c.MaxIOPS < 0 || c.MaxVMBytes < 0 {
return errors.New("config: worker.caps must not be negative")
}
return nil
ddiff --git a/internal/config/config_test.go b/internal/config/config_test.go
iindex 94592ca..5c143e8 100644
---- a/internal/config/config_test.go
++++ b/internal/config/config_test.go
@@@ -66,6 +66,18 @@ func TestPluginCommand(t *testing.T) {
}
}
++// TestWorkerVMLimitRequired: RLIMIT_AS is always set: max_vm_bytes has no "none" value and defaults when zero.
++func TestWorkerVMLimitRequired(t *testing.T) {
++ c := Default()
++ c.Worker.Caps.MaxVMBytes = -1
++ if err := c.Validate(); err == nil {
++ t.Fatal("max_vm_bytes -1 was accepted")
++ }
++ if v := (WorkerCaps{}).WithDefaults().MaxVMBytes; v != DefaultWorkerCaps.MaxVMBytes {
++ t.Fatalf("default max_vm_bytes %d", v)
++ }
++}
++
// TestWorkerNameReserved: status.capabilities.worker holds the job runtime classes, so no plugin may be named worker.
func TestWorkerNameReserved(t *testing.T) {
c := Default()
ddiff --git a/internal/node/node.go b/internal/node/node.go
iindex 77a31c2..d9a389c 100644
---- a/internal/node/node.go
++++ b/internal/node/node.go
@@@ -149,7 +149,10 @@ func New(opt Options) (*Node, error) {
}
}
if opt.Config.Worker.Enabled {
-- w := worker.New(opt.Config.Worker, n.sendJobFrame, opt.Log)
++ wcfg := opt.Config.Worker
++ // No job may see the Node's config, credential, state or control socket (they all lie in these).
++ wcfg.NodeDirs = []string{opt.Paths.ConfigDir, opt.Paths.StateDir}
++ w := worker.New(wcfg, n.sendJobFrame, opt.Log)
if err := w.Probe(); err != nil {
opt.Log.Error("worker off: jobs cannot run isolated here, every job is refused", "err", err)
} else {
ddiff --git a/internal/node/worker_test.go b/internal/node/worker_test.go
iindex 8f2c726..bf0142b 100644
---- a/internal/node/worker_test.go
++++ b/internal/node/worker_test.go
@@@ -5,8 +5,10 @@ package node
import (
"context"
"encoding/json"
++ "errors"
"fmt"
"io"
++ "io/fs"
"log/slog"
"os"
"path/filepath"
@@@ -25,7 +27,7 @@ import (
)
// TestWorkerHelperProcess is the job's process when this test binary runs as the `hello` function (`-- job`): it
--// appends a line to the marker file named in its args, prints, holds for hold_ms (1.5 s by default) and writes its
++// tries to append a line to the marker file named in its args (the sandbox must prevent it), prints, holds for hold_ms (1.5 s by default) and writes its
// result to fd 3.
func TestWorkerHelperProcess(t *testing.T) {
i := slices.Index(os.Args, "--")
@@@ -58,8 +60,15 @@ func startWorkerNode(t *testing.T) *env {
if err != nil {
t.Fatal(err)
}
-- cfg := config.WorkerConfig{Enabled: true, AllowSameUser: true, Caps: config.WorkerCaps{MaxMemBytes: 8 << 30},
-- Functions: []config.FunctionConfig{{Name: "hello", Version: "1.0.0", Command: []string{exe, "-test.run=^TestWorkerHelperProcess$", "--", "job"}}}}
++ // A race-instrumented binary maps terabytes of shadow memory up front: RLIMIT_AS 64 TiB leaves it room. The job
++ // gets a copy of its artifact directory, this test binary, in max_disk_bytes.
++ cfg := config.WorkerConfig{Enabled: true, AllowSameUser: true, Caps: config.WorkerCaps{MaxMemBytes: 8 << 30, MaxVMBytes: 1 << 46,
++ MaxDiskBytes: 256 << 20},
++ Functions: []config.FunctionConfig{{Name: "hello", Version: "1.0.0", Command: []string{exe, "-test.run=^TestWorkerHelperProcess$", "--", "job"},
++ Env: map[string]string{"GOMAXPROCS": "2"}}}}
++ if os.Geteuid() == 0 { // as root, jobs run as nobody: the test binary must then lie in a directory nobody can read
++ cfg.AllowSameUser, cfg.RunAs = false, &config.RunAs{UID: 65534, GID: 65534}
++ }
e, status := startWorkerNodeWith(t, cfg, true)
caps := cfg.Caps.WithDefaults()
want := map[string]any{"capabilities": []any{"worker:function"}, "runtime_classes": []any{protocol.ClassFunction}, "max_jobs": float64(caps.MaxJobs),
@@@ -240,9 +249,18 @@ func TestJobRunsOverTheLink(t *testing.T) {
t.Fatalf("resent job answered %+v, first job_exit %+v", again, ex)
}
}
-- b, err := os.ReadFile(marker)
-- if err != nil || string(b) != "started\n" {
-- t.Fatalf("the function ran %q (%v), want once", b, err)
++ started = 0
++ for _, f := range e.conn.Frames() {
++ if m, ok := f.Msg.(protocol.JobStarted); ok && m.ID == j.ID {
++ started++
++ }
++ }
++ if started != 1 {
++ t.Fatalf("the function started %d times, want once", started)
++ }
++ // The marker lies in the Node's state directory, which the job cannot see.
++ if _, err := os.Stat(marker); !errors.Is(err, fs.ErrNotExist) {
++ t.Fatalf("the job wrote into the Node's state directory: %v", err)
}
}
ddiff --git a/internal/service/service.go b/internal/service/service.go
iindex 076df26..763897e 100644
---- a/internal/service/service.go
++++ b/internal/service/service.go
@@@ -84,6 +84,7 @@ func config(opt Options) *ks.Config {
"OnFailure": "restart", // Windows
"OnFailureDelayDuration": "5s",
"DelayedAutoStart": false,
++ "SystemdScript": systemdScript,
},
}
if runtime.GOOS == "darwin" {
@@@ -93,6 +94,36 @@ func config(opt Options) *ks.Config {
return c
}
++// systemdScript is kardianos/service's systemd unit with Delegate=yes: the worker makes a cgroup per job inside the
++// Node's own (internal/worker), which systemd allows only in a delegated cgroup.
++const systemdScript = `[Unit]
++Description={{Description}}
++ConditionFileIsExecutable={{Path | cmdEscape}}
++{{range Dependencies}}{{.}}
++{{end}}
++[Service]
++StartLimitInterval=5
++StartLimitBurst=10
++ExecStart={{Path | cmdEscape}}{{range Arguments}} {{. | cmd}}{{end}}
++{{if ChRoot}}RootDirectory={{ChRoot | cmd}}
++{{end}}{{if WorkingDirectory}}WorkingDirectory={{WorkingDirectory | cmdEscape}}
++{{end}}{{if UserName}}User={{UserName}}
++{{end}}{{if ReloadSignal}}ExecReload=/bin/kill -{{ReloadSignal}} "$MAINPID"
++{{end}}{{if PIDFile}}PIDFile={{PIDFile | cmd}}
++{{end}}{{if OutputFileSupport}}StandardOutput=file:{{LogDirectory}}/{{Name}}.out
++StandardError=file:{{LogDirectory}}/{{Name}}.err
++{{end}}{{if LimitNOFILE}}LimitNOFILE={{LimitNOFILE}}
++{{end}}{{if Restart}}Restart={{Restart}}
++{{end}}{{if SuccessExitStatus}}SuccessExitStatus={{SuccessExitStatus}}
++{{end}}RestartSec=120
++Delegate=yes
++EnvironmentFile=-/etc/sysconfig/{{Name}}
++
++{{range EnvVars}}{{.}}
++{{end}}[Install]
++WantedBy=multi-user.target
++`
++
// Run runs program under the service manager, or in the foreground when started from a terminal.
func Run(opt Options, run Program) error {
p := &program{run: run}
ddiff --git a/internal/service/service_test.go b/internal/service/service_test.go
iindex 478394e..5539fc9 100644
---- a/internal/service/service_test.go
++++ b/internal/service/service_test.go
@@@ -2,6 +2,7 @@ package service
import (
"context"
++ "strings"
"testing"
"time"
)
@@@ -32,4 +33,7 @@ func TestConfig(t *testing.T) {
if c.Name != "openvibe-node" || c.Arguments[0] != "run" || c.Option["Restart"] != "always" {
t.Fatalf("%+v", c)
}
++ if u, _ := c.Option["SystemdScript"].(string); !strings.Contains(u, "\n[Service]\n") || !strings.Contains(u, "\nDelegate=yes\n") {
++ t.Fatalf("the systemd unit must delegate the Node's cgroup: %q", u)
++ }
}
ddiff --git a/internal/worker/cgroup_linux.go b/internal/worker/cgroup_linux.go
nnew file mode 100644
iindex 0000000..1ede93b
---- /dev/null
++++ b/internal/worker/cgroup_linux.go
@@@ -0,0 +1,367 @@
++//go:build linux
++
++package worker
++
++import (
++ "errors"
++ "fmt"
++ "io/fs"
++ "os"
++ "path/filepath"
++ "slices"
++ "strconv"
++ "strings"
++ "sync"
++ "sync/atomic"
++ "syscall"
++ "time"
++
++ "github.com/OpenVibers/OpenVibe.Node/internal/config"
++ "github.com/OpenVibers/OpenVibe.Node/internal/protocol"
++)
++
++const (
++ cgroupFS = "/sys/fs/cgroup"
++ cgroup2 = 0x63677270 // CGROUP2_SUPER_MAGIC
++ nodeLeaf = "openvibe-node"
++ cpuPeriodUS = 100000
++)
++
++// cgroupControllers are the cgroup v2 controllers every job is limited with; without any of them no job runs.
++var cgroupControllers = []string{"cpu", "memory", "pids", "io"}
++
++var (
++ cgroupOnce sync.Once
++ cgroupBase string
++ cgroupErr error
++ cgroupSeq atomic.Uint64
++)
++
++// jobCgroupBase is the cgroup v2 directory the job cgroups are made in: the Node's own cgroup, set up once. It needs
++// that cgroup delegated to the Node (systemd Delegate=yes, or a Node running as root).
++func jobCgroupBase() (string, error) {
++ cgroupOnce.Do(func() { cgroupBase, cgroupErr = setupCgroupBase() })
++ return cgroupBase, cgroupErr
++}
++
++// setupCgroupBase enables the controllers for the children of the Node's own cgroup. cgroup v2 lets a cgroup other
++// than the root do that only while it holds no process, so the Node's processes move to a leaf, openvibe-node, first
++// (a Node already in that leaf uses its parent). Nothing moves unless every controller is available.
++func setupCgroupBase() (string, error) {
++ var st syscall.Statfs_t
++ if err := syscall.Statfs(cgroupFS, &st); err != nil || st.Type != cgroup2 {
++ return "", errors.New("cgroup v2 is not mounted at " + cgroupFS)
++ }
++ b, err := os.ReadFile("/proc/self/cgroup")
++ if err != nil {
++ return "", err
++ }
++ rel := ""
++ for _, l := range strings.Split(string(b), "\n") {
++ if r, ok := strings.CutPrefix(l, "0::"); ok {
++ rel = r
++ }
++ }
++ if rel == "" {
++ return "", errors.New("the Node is not in a cgroup v2 hierarchy")
++ }
++ base := filepath.Join(cgroupFS, rel)
++ if filepath.Base(base) == nodeLeaf {
++ base = filepath.Dir(base)
++ }
++ avail, err := os.ReadFile(filepath.Join(base, "cgroup.controllers"))
++ if err != nil {
++ return "", err
++ }
++ enabled, err := os.ReadFile(filepath.Join(base, "cgroup.subtree_control"))
++ if err != nil {
++ return "", err
++ }
++ var missing []string
++ for _, c := range cgroupControllers {
++ if !hasWord(string(avail), c) {
++ return "", fmt.Errorf("the cgroup v2 controller %s is not available in %s (delegate the Node's cgroup to it: systemd Delegate=yes)", c, base)
++ }
++ if !hasWord(string(enabled), c) {
++ missing = append(missing, c)
++ }
++ }
++ if len(missing) == 0 {
++ return base, nil
++ }
++ if base != cgroupFS { // the root cgroup may hold processes and enable controllers all the same
++ if err := os.Mkdir(filepath.Join(base, nodeLeaf), 0o755); err != nil && !errors.Is(err, os.ErrExist) {
++ return "", err
++ }
++ }
++ for try := 0; ; try++ {
++ err := moveProcs(base)
++ for _, c := range missing {
++ if err == nil {
++ err = os.WriteFile(filepath.Join(base, "cgroup.subtree_control"), []byte("+"+c), 0)
++ }
++ }
++ if err == nil {
++ return base, nil
++ }
++ if try == 2 || !errors.Is(err, syscall.EBUSY) { // EBUSY: a process appeared in base meanwhile
++ return "", fmt.Errorf("enabling the cgroup v2 controllers in %s: %w", base, err)
++ }
++ }
++}
++
++// moveProcs moves every process of base into its openvibe-node leaf (none from the root cgroup). They must all be the
++// Node and its descendants: a cgroup holding other processes (a login session's scope, the parent of a Node that
++// was moved into a leaf it does not own) is not the Node's to rearrange, and nothing moves.
++func moveProcs(base string) error {
++ if base == cgroupFS {
++ return nil
++ }
++ b, err := os.ReadFile(filepath.Join(base, "cgroup.procs"))
++ if err != nil {
++ return err
++ }
++ pids := strings.Fields(string(b))
++ for _, pid := range pids {
++ if n, _ := strconv.Atoi(pid); !ownProcess(n) {
++ return fmt.Errorf("%s holds process %s, which is not the Node's: run the Node as its own systemd service "+
++ "with Delegate=yes", base, pid)
++ }
++ }
++ for _, pid := range pids {
++ err := os.WriteFile(filepath.Join(base, nodeLeaf, "cgroup.procs"), []byte(pid), 0)
++ if err != nil && !errors.Is(err, syscall.ESRCH) {
++ return err
++ }
++ }
++ return nil
++}
++
++// ownProcess reports whether pid is the Node or one of its descendants. A process that has exited counts as the
++// Node's: moving it fails ESRCH, which moveProcs ignores.
++func ownProcess(pid int) bool {
++ self := os.Getpid()
++ for hops := 0; pid > 1 && hops < 4096; hops++ {
++ if pid == self {
++ return true
++ }
++ b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
++ if err != nil {
++ return errors.Is(err, fs.ErrNotExist) && hops == 0
++ }
++ f := strings.Fields(string(b[strings.LastIndexByte(string(b), ')')+1:])) // after the comm, which may hold spaces
++ if len(f) < 2 {
++ return false
++ }
++ pid, _ = strconv.Atoi(f[1]) // ppid
++ }
++ return false
++}
++
++func hasWord(s, w string) bool {
++ for _, f := range strings.Fields(s) {
++ if f == w {
++ return true
++ }
++ }
++ return false
++}
++
++// jobCgroup is one job's cgroup: the job starts in it (clone3 CLONE_INTO_CGROUP) and never leaves it.
++type jobCgroup struct {
++ dir string
++ fd int
++ io string // the io.max limits every block device must have: "rbps=… wbps=… riops=… wiops=…"
++}
++
++// newJobCgroup makes a job's cgroup with its limits: memory.max at the job's mem_bytes (swap off, and an OOM kills
++// the whole job), pids.max, cpu.max and io.max on every block device.
++func newJobCgroup(caps config.WorkerCaps, lim protocol.JobLimits) (*jobCgroup, error) {
++ base, err := jobCgroupBase()
++ if err != nil {
++ return nil, err
++ }
++ dir := filepath.Join(base, fmt.Sprintf("openvibe-job-%d-%d", os.Getpid(), cgroupSeq.Add(1)))
++ if err := os.Mkdir(dir, 0o755); err != nil {
++ return nil, err
++ }
++ cg := &jobCgroup{dir: dir, fd: -1}
++ if err := cg.limit(caps, lim); err != nil {
++ cg.remove()
++ return nil, err
++ }
++ fd, err := syscall.Open(dir, syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_CLOEXEC, 0)
++ if err != nil {
++ cg.remove()
++ return nil, err
++ }
++ cg.fd = fd
++ return cg, nil
++}
++
++func (c *jobCgroup) limit(caps config.WorkerCaps, lim protocol.JobLimits) error {
++ set := func(file, v string) error {
++ if err := os.WriteFile(filepath.Join(c.dir, file), []byte(v), 0); err != nil {
++ return fmt.Errorf("cgroup %s: %w", file, err)
++ }
++ return nil
++ }
++ for _, s := range []struct{ file, v string }{
++ {"memory.max", strconv.FormatInt(lim.MemBytes, 10)},
++ {"memory.oom.group", "1"},
++ {"pids.max", strconv.FormatInt(caps.MaxPids, 10)},
++ {"cpu.max", fmt.Sprintf("%d %d", caps.MaxCPUMillis*cpuPeriodUS/1000, cpuPeriodUS)},
++ } {
++ if err := set(s.file, s.v); err != nil {
++ return err
++ }
++ }
++ if err := limitSwap(c.dir, "/proc/swaps"); err != nil {
++ return err
++ }
++ c.io = fmt.Sprintf("rbps=%d wbps=%d riops=%d wiops=%d", caps.MaxIOBps, caps.MaxIOBps, caps.MaxIOPS, caps.MaxIOPS)
++ devs, err := blockDevices()
++ if err != nil {
++ return err
++ }
++ for _, d := range devs {
++ if err := set("io.max", d+" "+c.io); err != nil {
++ // ENODEV for a disk that is still there is a disk the job's I/O would not be limited on: no job runs.
++ if _, serr := os.Stat("/sys/dev/block/" + d); errors.Is(err, syscall.ENODEV) && errors.Is(serr, os.ErrNotExist) {
++ continue // unplugged meanwhile: nothing to limit
++ }
++ return fmt.Errorf("limiting the I/O on block device %s: %w", d, err)
++ }
++ }
++ return c.checkIO()
++}
++
++// limitSwap sets the job cgroup dir's memory.swap.max to 0 and reads it back. The control exists only with swap
++// accounting: without it a job could swap past memory.max, which is accepted only while the host has no swap at all
++// (procSwaps, /proc/swaps, lists none).
++func limitSwap(dir, procSwaps string) error {
++ file := filepath.Join(dir, "memory.swap.max")
++ f, err := os.OpenFile(file, os.O_WRONLY|os.O_TRUNC, 0) // no O_CREATE: a missing control must show as missing
++ if errors.Is(err, os.ErrNotExist) {
++ b, err := os.ReadFile(procSwaps)
++ if err != nil {
++ return fmt.Errorf("cgroup memory.swap.max is missing and the host's swap is unknown: %w", err)
++ }
++ if lines := strings.Split(strings.TrimSpace(string(b)), "\n"); len(lines) > 1 {
++ return errors.New("cgroup memory.swap.max is missing (no swap accounting) while the host has swap: a job " +
++ "could swap past memory.max (turn swap accounting on, or swap off)")
++ }
++ return nil
++ }
++ if err != nil {
++ return fmt.Errorf("cgroup memory.swap.max: %w", err)
++ }
++ _, err = f.WriteString("0")
++ if cerr := f.Close(); err == nil {
++ err = cerr
++ }
++ if err != nil {
++ return fmt.Errorf("cgroup memory.swap.max: %w", err)
++ }
++ if b, err := os.ReadFile(file); err != nil || strings.TrimSpace(string(b)) != "0" {
++ return fmt.Errorf("cgroup memory.swap.max reads %q, not 0 (%v)", b, err)
++ }
++ return nil
++}
++
++// checkIO reads io.max back and checks that every whole block device with a medium has the job's limits: the kernel
++// takes a write it does not apply in full (a value of 0 or 1 leaves that limit off) without failing it.
++func (c *jobCgroup) checkIO() error {
++ devs, err := blockDevices()
++ if err != nil {
++ return err
++ }
++ b, err := os.ReadFile(filepath.Join(c.dir, "io.max"))
++ if err != nil {
++ return fmt.Errorf("cgroup io.max: %w", err)
++ }
++ got := map[string][]string{} // "8:0 rbps=… wbps=… riops=… wiops=…", one line per limited device
++ for _, l := range strings.Split(string(b), "\n") {
++ if f := strings.Fields(l); len(f) > 0 {
++ got[f[0]] = f[1:]
++ }
++ }
++ for _, d := range devs {
++ for _, kv := range strings.Fields(c.io) {
++ if !slices.Contains(got[d], kv) {
++ return fmt.Errorf("block device %s is not limited to %s in cgroup io.max (%q)", d, c.io, strings.Join(got[d], " "))
++ }
++ }
++ }
++ return nil
++}
++
++// blockDevices lists the major:minor of every whole block device with a medium (io.max takes whole devices only).
++// A hidden disk (a path of an NVMe multipath disk) is not one: it has no block device of its own (io.max refuses it
++// ENODEV) and is reached only through the visible disk over it, which is limited.
++func blockDevices() ([]string, error) { return blockDevicesIn("/sys/block") }
++
++// blockDevicesIn is blockDevices on the sysfs directory sys. A disk whose dev or size cannot be read is one whose
++// io.max would go unset and unchecked: it is an error, unless the disk is gone meanwhile (unplugged).
++func blockDevicesIn(sys string) ([]string, error) {
++ ents, err := os.ReadDir(sys)
++ if err != nil {
++ return nil, err
++ }
++ var out []string
++ for _, e := range ents {
++ attr := func(name string, optional bool) (string, error) {
++ b, err := os.ReadFile(filepath.Join(sys, e.Name(), name))
++ if errors.Is(err, fs.ErrNotExist) && optional {
++ return "", nil
++ }
++ return strings.TrimSpace(string(b)), err
++ }
++ dev, err := attr("dev", false)
++ size, serr := attr("size", false)
++ hidden, herr := attr("hidden", true) // older kernels have no hidden attribute and no hidden disks
++ if err = errors.Join(err, serr, herr); err != nil {
++ if _, gone := os.Stat(filepath.Join(sys, e.Name())); errors.Is(gone, fs.ErrNotExist) {
++ continue
++ }
++ return nil, fmt.Errorf("block device %s: %w", e.Name(), err)
++ }
++ if dev == "" {
++ return nil, fmt.Errorf("block device %s has no major:minor", e.Name())
++ }
++ if size == "0" || hidden == "1" {
++ continue
++ }
++ out = append(out, dev)
++ }
++ return out, nil
++}
++
++// oomKilled reports whether the kernel's OOM killer ended a process of the job (memory.max reached).
++func (c *jobCgroup) oomKilled() bool {
++ b, err := os.ReadFile(filepath.Join(c.dir, "memory.events"))
++ if err != nil {
++ return false
++ }
++ for _, l := range strings.Split(string(b), "\n") {
++ if n, ok := strings.CutPrefix(l, "oom_kill "); ok && n != "0" {
++ return true
++ }
++ }
++ return false
++}
++
++// remove kills whatever is left in the cgroup and removes it (its last processes may take a moment to be reaped).
++func (c *jobCgroup) remove() {
++ if c.fd >= 0 {
++ syscall.Close(c.fd)
++ c.fd = -1
++ }
++ _ = os.WriteFile(filepath.Join(c.dir, "cgroup.kill"), []byte("1"), 0)
++ for i := 0; i < 100; i++ {
++ if err := syscall.Rmdir(c.dir); err == nil || errors.Is(err, syscall.ENOENT) {
++ return
++ }
++ time.Sleep(20 * time.Millisecond)
++ }
++}
ddiff --git a/internal/worker/run_linux_test.go b/internal/worker/run_linux_test.go
iindex 8890bf4..40043d8 100644
---- a/internal/worker/run_linux_test.go
++++ b/internal/worker/run_linux_test.go
@@@ -3,17 +3,31 @@
package worker
import (
++ "bufio"
"bytes"
++ "crypto/ecdsa"
++ "crypto/elliptic"
++ "crypto/rand"
++ "crypto/x509"
++ "crypto/x509/pkix"
"encoding/json"
++ "encoding/pem"
"errors"
"fmt"
"io"
"io/fs"
++ "log/slog"
++ "maps"
++ "math/big"
"net"
"os"
++ "os/exec"
"path/filepath"
"slices"
++ "strconv"
"strings"
++ "sync"
++ "syscall"
"testing"
"time"
"unicode/utf8"
@@@ -61,6 +75,127 @@ func TestHelperProcess(t *testing.T) {
_, derr := net.DialTimeout("tcp", "192.0.2.1:9", time.Second)
_ = json.NewEncoder(result).Encode(map[string]any{"env": os.Environ(), "cwd": cwd, "pid": os.Getpid(),
"uid": os.Getuid(), "loopback": fmt.Sprint(lerr), "dial": fmt.Sprint(derr)})
++ case "escape": // every attempt must fail but the write to /tmp; the result is each attempt's error
++ node := os.Getenv("OPENVIBE_TEST_NODE_DIR")
++ try := map[string]error{}
++ _, try["read credential"] = os.ReadFile(node + "/credential.json")
++ _, try["list node dir"] = os.ReadDir(node)
++ if c, err := net.DialTimeout("unix", node+"/node.sock", time.Second); err == nil {
++ c.Close()
++ } else {
++ try["dial node socket"] = err
++ }
++ for _, p := range []string{"/var", "/home", "/root", "/sys", "/etc/shadow", "/etc/ssl/private", "/run"} {
++ _, try["stat "+p] = os.Stat(p)
++ }
++ try["write /"] = os.WriteFile("/x", nil, 0o644)
++ try["write /usr"] = os.WriteFile("/usr/x", nil, 0o644)
++ try["write artifact"] = os.WriteFile(filepath.Join(filepath.Dir(os.Args[0]), "x"), nil, 0o644)
++ try["mount"] = syscall.Mount("tmpfs", "/tmp", "tmpfs", 0, "")
++ try["unshare"] = syscall.Unshare(syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET)
++ try["chroot"] = syscall.Chroot("/tmp")
++ if fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, 0); err == nil {
++ syscall.Close(fd)
++ } else {
++ try["packet socket"] = err
++ }
++ try["write /tmp"] = os.WriteFile("/tmp/x", []byte("ok"), 0o600)
++ st, _ := os.ReadFile("/proc/self/status")
++ var keys []string // every file of its /etc that holds a PEM private key (symlinks lead to /usr: not followed)
++ _ = filepath.WalkDir("/etc", func(p string, d fs.DirEntry, err error) error {
++ if err != nil || !d.Type().IsRegular() {
++ return nil
++ }
++ if b, err := os.ReadFile(p); err == nil && bytes.Contains(b, []byte("PRIVATE KEY-----")) {
++ keys = append(keys, p)
++ }
++ return nil
++ })
++ out := map[string]string{"status": string(st), "private keys": strings.Join(keys, " ")}
++ for k, err := range try {
++ out[k] = fmt.Sprint(err)
++ }
++ _ = json.NewEncoder(result).Encode(out)
++ case "quick":
++ fmt.Fprint(result, `{"ok":true}`)
++ case "snapshot": // looks for what TestArtifactSnapshot adds to its artifact directory on the host once it started
++ dir := filepath.Dir(os.Args[0])
++ for end := time.Now().Add(2 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) {
++ if _, err := os.Lstat(filepath.Join(dir, "late")); err == nil {
++ break
++ }
++ }
++ try := map[string]error{}
++ _, try["late file"] = os.Lstat(filepath.Join(dir, "late"))
++ _, try["read through link"] = os.ReadFile(filepath.Join(dir, "link"))
++ _, try["stat unreadable"] = os.Lstat(filepath.Join(dir, "unreadable"))
++ if c, err := net.DialTimeout("unix", filepath.Join(dir, "s"), time.Second); err == nil {
++ c.Close()
++ } else {
++ try["dial socket"] = err
++ }
++ if f, err := os.OpenFile(filepath.Join(dir, "f"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil {
++ f.Close()
++ } else {
++ try["open FIFO"] = err
++ }
++ try["write artifact"] = os.WriteFile(filepath.Join(dir, "x"), nil, 0o644)
++ out := map[string]string{}
++ for k, err := range try {
++ out[k] = fmt.Sprint(err)
++ }
++ _ = json.NewEncoder(result).Encode(out)
++ case "stage": // not a job: TestStageArtifactInSystemPath's private root, in a user and mount namespace of its own
++ r, sys, dir := os.Args[i+2], os.Args[i+3], os.Args[i+4]
++ out := map[string]string{}
++ err := syscall.Mount("", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")
++ if err == nil {
++ err = syscall.Mount("tmpfs", r, "tmpfs", 0, "size=1m")
++ }
++ if err == nil {
++ err = bind(r, bindSpec{Path: sys})
++ }
++ var dev uint64
++ if err == nil {
++ dev, err = stageArtifact(r, dir, 1<<20)
++ }
++ if err == nil {
++ err = os.Symlink(sys, r+"/abs")
++ }
++ if err != nil {
++ fmt.Fprintf(result, "setup: %v\n", err)
++ os.Exit(0)
++ }
++ out["in root"], _ = inRoot(r, "/abs/fn")
++ fmt.Fprintln(result, "staged")
++ _, _ = io.ReadAll(os.Stdin) // the test adds a socket, a FIFO and a file to the host's directories
++ var st syscall.Stat_t
++ if err := syscall.Stat(r+dir, &st); err != nil || st.Dev != dev {
++ out["copy"] = fmt.Sprintf("the artifact path does not lead to the copy (%v)", err)
++ }
++ if c, err := net.DialTimeout("unix", r+dir+"/s", time.Second); err == nil {
++ c.Close()
++ out["dial socket"] = "connected"
++ } else {
++ out["dial socket"] = err.Error()
++ }
++ if f, err := os.OpenFile(r+dir+"/f", os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil {
++ f.Close()
++ out["open FIFO"] = "opened"
++ } else {
++ out["open FIFO"] = err.Error()
++ }
++ _, err = os.Lstat(r + dir + "/a")
++ out["artifact file"] = fmt.Sprint(err)
++ _, err = os.Lstat(r + sys + "/late")
++ out["system path file"] = fmt.Sprint(err)
++ _ = json.NewEncoder(result).Encode(out)
++ case "forkbomb": // starts processes until the kernel refuses one
++ n, err := 0, error(nil)
++ for ; n < 1000 && err == nil; n++ {
++ err = exec.Command("/bin/sleep", "60").Start()
++ }
++ _ = json.NewEncoder(result).Encode(map[string]any{"started": n - 1, "err": fmt.Sprint(err)})
}
os.Exit(0)
}
@@@ -72,8 +207,13 @@ func newRunWorker(t *testing.T, caps config.WorkerCaps, modes ...string) (*Worke
if caps.MaxMemBytes == 0 {
caps.MaxMemBytes = 8 << 30
}
++ return newRunWorkerConfig(t, helperConfig(t, caps, modes...))
++}
++
++func newRunWorkerConfig(t *testing.T, cfg config.WorkerConfig) (*Worker, *sink) {
++ t.Helper()
s := &sink{}
-- w := New(helperConfig(t, caps, modes...), s.send, quiet())
++ w := New(cfg, s.send, quiet())
if err := w.Probe(); err != nil {
if os.Getenv("OPENVIBE_WORKER_TESTS") == "require" {
t.Fatalf("job isolation unavailable: %v", err)
@@@ -169,8 +309,12 @@ func TestIsolation(t *testing.T) {
if err := json.Unmarshal(ex.Result, &r); err != nil || ex.Reason != protocol.ExitExited {
t.Fatalf("%+v %s: %v", ex, ex.Result, err)
}
-- if r.PID != 1 || r.UID != os.Getuid() {
-- t.Fatalf("pid %d uid %d, want 1 and %d", r.PID, r.UID, os.Getuid())
++ uid := os.Getuid()
++ if uid == 0 {
++ uid = 65534 // helperConfig's run_as
++ }
++ if r.PID != 1 || r.UID != uid {
++ t.Fatalf("pid %d uid %d, want 1 and %d", r.PID, r.UID, uid)
}
if r.Loopback == "<nil>" || r.Dial == "<nil>" {
t.Fatalf("network reachable: loopback %s, dial %s", r.Loopback, r.Dial)
@@@ -311,3 +455,1064 @@ func TestLatchKillsAndRefuses(t *testing.T) {
t.Fatalf("%+v", ex)
}
}
++
++// TestEscape: the job cannot see the Node's directories (a secret file, the control socket), nor /var, /home, /root,
++// /sys, /run or the rest of /etc (/etc/ssl/private, the host's TLS keys, included); it cannot write / nor its
++// artifact; mount, unshare, chroot and packet sockets are refused; it runs with no_new_privs and the seccomp filter,
++// and only its own /tmp is writable.
++func TestEscape(t *testing.T) {
++ node := t.TempDir()
++ if err := os.WriteFile(filepath.Join(node, "credential.json"), []byte(`{"secret":"x"}`), 0o600); err != nil {
++ t.Fatal(err)
++ }
++ l, err := net.Listen("unix", filepath.Join(node, "node.sock"))
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer l.Close()
++ cfg := helperConfig(t, config.WorkerCaps{MaxMemBytes: 8 << 30}, "escape")
++ cfg.NodeDirs = []string{node}
++ cfg.Functions[0].Env["OPENVIBE_TEST_NODE_DIR"] = node
++ w, s := newRunWorkerConfig(t, cfg)
++ j := testJob(1, "escape")
++ run(t, w, j)
++ ex := s.exit(t, j.ID)
++ var r map[string]string
++ if err := json.Unmarshal(ex.Result, &r); err != nil || ex.Reason != protocol.ExitExited {
++ t.Fatalf("%+v %s: %v", ex, ex.Result, err)
++ }
++ if !strings.Contains(r["status"], "\nNoNewPrivs:\t1\n") || !strings.Contains(r["status"], "\nSeccomp:\t2\n") {
++ t.Fatalf("not confined: %s", r["status"])
++ }
++ if r["write /tmp"] != "<nil>" {
++ t.Fatalf("its /tmp is not writable: %s", r["write /tmp"])
++ }
++ if r["private keys"] != "" {
++ t.Errorf("its /etc holds private keys: %s", r["private keys"])
++ }
++ delete(r, "write /tmp")
++ delete(r, "status")
++ delete(r, "private keys")
++ if len(r) != 17 {
++ t.Fatalf("%d attempts reported: %v", len(r), r)
++ }
++ for k, v := range r {
++ if v == "<nil>" {
++ t.Errorf("%s succeeded", k)
++ }
++ if strings.Contains(k, "node") || strings.Contains(k, "credential") || strings.HasPrefix(k, "stat ") {
++ if !strings.Contains(v, "no such file or directory") {
++ t.Errorf("%s: %s, want no such file or directory", k, v)
++ }
++ }
++ }
++ if _, err := os.Stat(filepath.Join(filepath.Dir(cfg.Functions[0].Command[0]), "x")); err == nil {
++ t.Error("the job wrote into its artifact directory")
++ }
++}
++
++// TestArtifactNestedMount: a mount nested in the artifact directory is not copied, be it another filesystem or a bind
++// mount of the artifact's own (a host directory with a secret, here): the job fails without running (as root only:
++// the test mounts on the host).
++func TestArtifactNestedMount(t *testing.T) {
++ if os.Geteuid() != 0 {
++ t.Skip("mounting on the host needs root")
++ }
++ secret := filepath.Join(t.TempDir(), "secret")
++ if err := os.Mkdir(secret, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ for i, m := range []struct{ src, fstype string }{{"tmpfs", "tmpfs"}, {filepath.Dir(secret), ""}} {
++ cfg := helperConfig(t, config.WorkerCaps{MaxMemBytes: 8 << 30}, "quick")
++ nested := filepath.Join(cfg.Functions[0].Artifact(), "nested")
++ if err := os.Mkdir(nested, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ flags := uintptr(0)
++ if m.fstype == "" {
++ flags = syscall.MS_BIND
++ }
++ if err := syscall.Mount(m.src, nested, m.fstype, flags, "mode=0755,size=1m"); err != nil {
++ os.Remove(nested)
++ t.Fatal(err)
++ }
++ w, s := newRunWorkerConfig(t, cfg)
++ log := &logBuf{}
++ w.log = slog.New(slog.NewTextHandler(log, nil))
++ j := testJob(i+1, "quick")
++ run(t, w, j)
++ ex := s.exit(t, j.ID)
++ _ = syscall.Unmount(nested, syscall.MNT_DETACH)
++ os.Remove(nested)
++ if ex.Reason != protocol.ExitFailed || ex.Usage.StartedMS != nil || s.count(protocol.TypeJobStarted, j.ID) != 0 {
++ t.Fatalf("%s mounted in the artifact directory: %+v", m.src, ex)
++ }
++ if !strings.Contains(log.String(), "is a mount point") {
++ t.Fatalf("%s mounted in the artifact directory: the job failed for another reason:\n%s", m.src, log.String())
++ }
++ }
++}
++
++// logBuf is a worker's log, written while its jobs run.
++type logBuf struct {
++ mu sync.Mutex
++ b bytes.Buffer
++}
++
++func (l *logBuf) Write(p []byte) (int, error) {
++ l.mu.Lock()
++ defer l.mu.Unlock()
++ return l.b.Write(p)
++}
++
++func (l *logBuf) String() string {
++ l.mu.Lock()
++ defer l.mu.Unlock()
++ return l.b.String()
++}
++
++// TestArtifactSnapshot: a job runs on a copy of its artifact directory made as it started. A file, a Unix socket and a
++// FIFO the host adds to the directory once the job started never appear in it, so a socket or FIFO there cannot reach
++// a host process however late it appears; the copy is read-only and the host's directory untouched. A symlink is
++// copied as a symlink (one to a host file leads nowhere in the job's root) and a file the job's user may not read is
++// left out. That holds for an artifact directory in a system path too (as /usr/local/bin, the default for a command
++// there): the copy covers what the bind of that path shows of the host's directory.
++func TestArtifactSnapshot(t *testing.T) {
++ t.Run("own directory", func(t *testing.T) { artifactSnapshot(t, t.TempDir()) })
++ t.Run("in a system path", func(t *testing.T) {
++ sys := t.TempDir() // bound in every private root as /usr is
++ saved := systemPaths
++ systemPaths = append(slices.Clip(systemPaths), sys)
++ t.Cleanup(func() { systemPaths = saved })
++ dir := filepath.Join(sys, "fn")
++ if err := os.Mkdir(dir, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Chmod(filepath.Dir(sys), 0o755); err != nil {
++ t.Fatal(err)
++ }
++ artifactSnapshot(t, dir)
++ })
++}
++
++func artifactSnapshot(t *testing.T, dir string) {
++ cfg := helperConfig(t, config.WorkerCaps{MaxMemBytes: 8 << 30}, "snapshot")
++ for _, d := range []string{filepath.Dir(dir), dir} { // as root, jobs run as nobody and reach their artifact as nobody
++ if err := os.Chmod(d, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ }
++ moveHelper(t, &cfg, dir)
++ host := filepath.Join(t.TempDir(), "host-secret")
++ if err := os.WriteFile(host, []byte("x"), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Symlink(host, filepath.Join(dir, "link")); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.WriteFile(filepath.Join(dir, "unreadable"), []byte("x"), 0); err != nil {
++ t.Fatal(err)
++ }
++ w, s := newRunWorkerConfig(t, cfg)
++ j := testJob(1, "snapshot")
++ run(t, w, j)
++ s.wait(t, protocol.TypeJobStarted, j.ID, 1, 10*time.Second)
++ l, err := net.Listen("unix", filepath.Join(dir, "s"))
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer l.Close()
++ if err := syscall.Mkfifo(filepath.Join(dir, "f"), 0o666); err != nil {
++ t.Fatal(err)
++ }
++ for _, p := range []string{"s", "f"} { // a job that could see them could use them
++ if err := os.Chmod(filepath.Join(dir, p), 0o777); err != nil {
++ t.Fatal(err)
++ }
++ }
++ fifo, err := os.OpenFile(filepath.Join(dir, "f"), os.O_RDONLY|syscall.O_NONBLOCK, 0) // a reader, so a writer could open it
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer fifo.Close()
++ if err := os.WriteFile(filepath.Join(dir, "late"), nil, 0o644); err != nil {
++ t.Fatal(err)
++ }
++ ex := s.exit(t, j.ID)
++ var r map[string]string
++ if err := json.Unmarshal(ex.Result, &r); err != nil || ex.Reason != protocol.ExitExited {
++ t.Fatalf("%+v %s: %v", ex, ex.Result, err)
++ }
++ for _, k := range []string{"late file", "dial socket", "open FIFO", "read through link", "stat unreadable"} {
++ if !strings.Contains(r[k], "no such file or directory") {
++ t.Errorf("%s: %s, want no such file or directory", k, r[k])
++ }
++ }
++ if !strings.Contains(r["write artifact"], "read-only file system") {
++ t.Errorf("write artifact: %s, want read-only file system", r["write artifact"])
++ }
++ if _, err := os.Stat(filepath.Join(dir, "x")); err == nil {
++ t.Error("the job wrote into its artifact directory")
++ }
++}
++
++// TestStageArtifactInSystemPath: an artifact directory a bound system path holds is copied over what the bind shows
++// there, at the path it leads to in the private root (through an absolute symlink too), so a socket or FIFO the host
++// adds to it once the copy is made is out of reach, while the rest of the bound path still shows the host's directory.
++// It needs only a user and a mount namespace, no cgroup: it runs where the sandboxed jobs cannot.
++func TestStageArtifactInSystemPath(t *testing.T) {
++ r, sys := t.TempDir(), t.TempDir()
++ dir := filepath.Join(sys, "fn")
++ if err := os.Mkdir(dir, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.WriteFile(filepath.Join(dir, "a"), []byte("x"), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ art, err := os.Open(dir)
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer art.Close()
++ rr, rw, err := os.Pipe()
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer rr.Close()
++ cmd := exec.Command(os.Args[0], "-test.run=^TestHelperProcess$", "--", "stage", r, sys, dir)
++ cmd.ExtraFiles = []*os.File{rw, nil, nil, nil, art} // fd 3, the result; fd 7, artifactFD
++ cmd.Stderr = os.Stderr
++ cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNS,
++ UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Geteuid(), Size: 1}},
++ GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getegid(), Size: 1}}}
++ in, err := cmd.StdinPipe()
++ if err != nil {
++ t.Fatal(err)
++ }
++ if err := cmd.Start(); err != nil {
++ rw.Close()
++ if os.Getenv("OPENVIBE_WORKER_TESTS") == "require" {
++ t.Fatal(err)
++ }
++ t.Skipf("no user and mount namespace here: %v", err)
++ }
++ rw.Close()
++ defer func() { in.Close(); _ = cmd.Wait() }()
++ res := bufio.NewReader(rr)
++ if line, err := res.ReadString('\n'); line != "staged\n" {
++ t.Fatalf("staging: %q %v", line, err)
++ }
++ l, err := net.Listen("unix", filepath.Join(dir, "s"))
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer l.Close()
++ if err := syscall.Mkfifo(filepath.Join(dir, "f"), 0o666); err != nil {
++ t.Fatal(err)
++ }
++ fifo, err := os.OpenFile(filepath.Join(dir, "f"), os.O_RDONLY|syscall.O_NONBLOCK, 0) // a reader, so a writer could open it
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer fifo.Close()
++ if err := os.WriteFile(filepath.Join(sys, "late"), nil, 0o644); err != nil {
++ t.Fatal(err)
++ }
++ in.Close()
++ var out map[string]string
++ if err := json.NewDecoder(res).Decode(&out); err != nil {
++ t.Fatal(err)
++ }
++ if out["copy"] != "" {
++ t.Fatal(out["copy"])
++ }
++ if want := r + dir; out["in root"] != want {
++ t.Errorf("/abs/fn, a symlink to %s/fn, leads to %q in the root, want %s", sys, out["in root"], want)
++ }
++ for _, k := range []string{"dial socket", "open FIFO"} {
++ if !strings.Contains(out[k], "no such file or directory") {
++ t.Errorf("%s: %s, want no such file or directory", k, out[k])
++ }
++ }
++ if out["artifact file"] != "<nil>" || out["system path file"] != "<nil>" {
++ t.Errorf("the copy lacks the artifact's file (%s), or the bound system path does not show the host's (%s)",
++ out["artifact file"], out["system path file"])
++ }
++}
++
++// TestOwnProcess: the Node moves only itself and its descendants out of its cgroup (moveProcs).
++func TestOwnProcess(t *testing.T) {
++ cmd := exec.Command("sleep", "5")
++ if err := cmd.Start(); err != nil {
++ t.Fatal(err)
++ }
++ defer cmd.Wait()
++ defer cmd.Process.Kill()
++ if !ownProcess(os.Getpid()) || !ownProcess(cmd.Process.Pid) {
++ t.Error("the Node or its child is not the Node's")
++ }
++ if ownProcess(1) || ownProcess(os.Getppid()) {
++ t.Error("init or the Node's parent is the Node's")
++ }
++}
++
++// accounts points trustedGroup at these passwd and group files for the rest of the test.
++func accounts(t *testing.T, passwd, group string) {
++ dir := t.TempDir()
++ p, g := filepath.Join(dir, "passwd"), filepath.Join(dir, "group")
++ if err := os.WriteFile(p, []byte(passwd), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.WriteFile(g, []byte(group), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ op, og := passwdFile, groupFile
++ passwdFile, groupFile = p, g
++ t.Cleanup(func() { passwdFile, groupFile = op, og })
++}
++
++// TestTrustedGroup: a group is trusted only when every member, listed or by primary group, is a trusted user.
++func TestTrustedGroup(t *testing.T) {
++ passwd := "root:x:0:0::/root:/bin/sh\nnode:x:1000:1000::/:/bin/sh\nops:x:1001:0::/:/bin/sh\nbob:x:1002:1002::/:/bin/sh\n"
++ accounts(t, passwd, "root:x:0:\nnode:x:1000:\nwheel:x:10:root,bob\nstaff:x:50:node,ghost\nbob:x:1002:\n")
++ for _, c := range []struct {
++ gid uint32
++ want bool
++ }{
++ {0, false}, // ops (1001) has root's group as its primary group
++ {1000, true}, // the Node's own
++ {10, false}, // bob is listed
++ {50, false}, // ghost is no known user
++ {1002, false}, // bob's primary group
++ {77, false}, // not listed
++ } {
++ if got := trustedGroup(c.gid, []uint32{0, 1000}); got != c.want {
++ t.Errorf("trustedGroup(%d) = %v, want %v", c.gid, got, c.want)
++ }
++ }
++ if !trustedGroup(0, []uint32{0, 1000, 1001}) {
++ t.Error("root's group with trusted members only: not trusted")
++ }
++}
++
++// TestArtifactRoute: an artifact directory whose path crosses a directory of a bound system path that a host user
++// other than root or the Node's own could change is refused, as the Node opens it for a job and at boot: in such a
++// directory a host writer could, after every check, switch a symlink on the path (or rename a directory on it) so that
++// the job's path leads to a live host directory holding a socket or FIFO. A path that leaves the bound paths is the
++// private root's own, writable by anyone on the host or not.
++func TestArtifactRoute(t *testing.T) {
++ sys := t.TempDir() // bound in every private root as /usr is
++ for _, d := range []string{"real", "pub"} {
++ if err := os.Mkdir(filepath.Join(sys, d), 0o755); err != nil {
++ t.Fatal(err)
++ }
++ }
++ if err := os.Chmod(sys, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Symlink("real", filepath.Join(sys, "fn")); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Symlink("../real", filepath.Join(sys, "pub", "fn")); err != nil {
++ t.Fatal(err)
++ }
++ binds, links := []hostBind{{path: sys}}, []linkSpec{{"/abs", sys}}
++ for _, a := range []string{sys + "/real", sys + "/fn", "/abs/fn", "/abs/pub/../real", sys + "/pub/fn"} {
++ if err := checkRoute(binds, links, a, trustedUIDs()); err != nil {
++ t.Errorf("artifact %s on a path only its owner may change: %v", a, err)
++ }
++ }
++ for _, a := range []string{sys + "/missing", sys + "/real/missing/deeper"} {
++ if err := checkRoute(binds, links, a, trustedUIDs()); err != nil {
++ t.Errorf("missing artifact %s (the job fails, the probe does not): %v", a, err)
++ }
++ }
++ if err := checkRoute(binds, links, sys+"/real", []uint32{uint32(os.Geteuid()) + 1}); err == nil ||
++ !strings.Contains(err.Error(), "crosses "+sys+",") {
++ t.Errorf("artifact in a directory another user owns: %v, want refused", err)
++ }
++ // A group-writable directory: refused while a member of its group is not trusted, root's group included.
++ if err := os.Chmod(sys, 0o775); err != nil {
++ t.Fatal(err)
++ }
++ var st syscall.Stat_t
++ if err := syscall.Stat(sys, &st); err != nil {
++ t.Fatal(err)
++ }
++ gid, me := int(st.Gid), os.Geteuid()
++ accounts(t, fmt.Sprintf("root:x:0:0::/root:/bin/sh\nnode:x:%d:%d::/:/bin/sh\nmallory:x:%d:100::/:/bin/sh\n", me, gid, me+1),
++ fmt.Sprintf("root:x:0:\nnodes:x:%d:node,mallory\n", gid))
++ if err := checkRoute(binds, links, sys+"/real", trustedUIDs()); err == nil || !strings.Contains(err.Error(), "crosses "+sys+",") {
++ t.Errorf("artifact in a directory a group with an untrusted member may write: %v, want refused", err)
++ }
++ accounts(t, fmt.Sprintf("root:x:0:0::/root:/bin/sh\nnode:x:%d:%d::/:/bin/sh\n", me, gid), fmt.Sprintf("root:x:0:\nnodes:x:%d:node\n", gid))
++ if err := checkRoute(binds, links, sys+"/real", trustedUIDs()); err != nil {
++ t.Errorf("artifact in a directory only trusted users' group may write: %v", err)
++ }
++ if err := os.Chmod(sys, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Chmod(filepath.Join(sys, "pub"), 0o777); err != nil {
++ t.Fatal(err)
++ }
++ for _, a := range []string{sys + "/pub/fn", "/abs/pub/fn", sys + "/pub/fn/sub"} {
++ if err := checkRoute(binds, links, a, trustedUIDs()); err == nil || !strings.Contains(err.Error(), "crosses "+sys+"/pub,") {
++ t.Errorf("artifact %s through a symlink in a world-writable directory: %v, want refused", a, err)
++ }
++ }
++ if f, err := openArtifact(nil, binds, links, sys+"/pub/fn"); err == nil || !strings.Contains(err.Error(), "could change") {
++ if f != nil {
++ f.Close()
++ }
++ t.Errorf("openArtifact %s/pub/fn: %v, want refused", sys, err)
++ }
++ priv := t.TempDir()
++ if err := os.Chmod(priv, 0o777); err != nil {
++ t.Fatal(err)
++ }
++ if err := checkRoute(binds, links, priv+"/fn", trustedUIDs()); err != nil {
++ t.Errorf("artifact outside the bound paths: %v", err)
++ }
++ binds, links, err := rootPlan(nil)
++ if err != nil {
++ t.Fatal(err)
++ }
++ if err := checkRoute(binds, links, "/usr/bin", trustedUIDs()); usrTrusted(t) != (err == nil) {
++ t.Errorf("/usr/bin: %v", err)
++ }
++ cfg := helperConfig(t, config.WorkerCaps{}, "env")
++ cfg.Functions[0].ArtifactDir = sys + "/pub/fn"
++ saved := systemPaths
++ systemPaths = append(slices.Clip(systemPaths), sys)
++ t.Cleanup(func() { systemPaths = saved })
++ w := New(cfg, (&sink{}).send, quiet())
++ t.Cleanup(w.Close)
++ if err := w.Probe(); err == nil || !strings.Contains(err.Error(), "could change") {
++ t.Fatalf("probe with artifact_dir %s/pub/fn: %v, want refused", sys, err)
++ }
++}
++
++// TestArtifactRouteSwitched: a directory on a job's path to its artifact, in a bound system path, became
++// world-writable after boot (the probe passed): the job is refused before it starts, so when the host then switches
++// the symlink there to a host directory holding a socket and a FIFO, nothing reaches them. A switch while a job runs
++// meets its private copy instead (TestArtifactSnapshot, TestStageArtifactInSystemPath).
++func TestArtifactRouteSwitched(t *testing.T) {
++ sys := t.TempDir() // bound in every private root as /usr is
++ saved := systemPaths
++ systemPaths = append(slices.Clip(systemPaths), sys)
++ t.Cleanup(func() { systemPaths = saved })
++ cfg := helperConfig(t, config.WorkerCaps{MaxMemBytes: 8 << 30}, "snapshot")
++ real, other, pub := filepath.Join(sys, "real"), filepath.Join(sys, "other"), filepath.Join(sys, "pub")
++ for _, d := range []string{real, other, pub} {
++ if err := os.Mkdir(d, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ }
++ for _, d := range []string{filepath.Dir(sys), sys} {
++ if err := os.Chmod(d, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ }
++ moveHelper(t, &cfg, real)
++ if err := os.Link(cfg.Functions[0].Command[0], filepath.Join(other, "worker.test")); err != nil {
++ t.Fatal(err)
++ }
++ link := filepath.Join(pub, "fn")
++ if err := os.Symlink("../real", link); err != nil {
++ t.Fatal(err)
++ }
++ cfg.Functions[0].ArtifactDir, cfg.Functions[0].Command[0] = link, filepath.Join(link, "worker.test")
++ w, s := newRunWorkerConfig(t, cfg) // the probe passes: only the Node's user may change the path yet
++ if err := os.Chmod(pub, 0o777); err != nil {
++ t.Fatal(err)
++ }
++ log := &logBuf{}
++ w.log = slog.New(slog.NewTextHandler(log, nil))
++ j := testJob(1, "snapshot")
++ run(t, w, j)
++ l, err := net.Listen("unix", filepath.Join(other, "s"))
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer l.Close()
++ if err := syscall.Mkfifo(filepath.Join(other, "f"), 0o666); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Chmod(filepath.Join(other, "s"), 0o777); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Symlink("../other", link+".new"); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Rename(link+".new", link); err != nil {
++ t.Fatal(err)
++ }
++ ex := s.exit(t, j.ID)
++ if ex.Reason != protocol.ExitFailed || ex.Usage.StartedMS != nil || s.count(protocol.TypeJobStarted, j.ID) != 0 {
++ t.Fatalf("a job whose artifact path a host user could switch: %+v %s", ex, ex.Result)
++ }
++ if !strings.Contains(log.String(), "could change") {
++ t.Fatalf("the job failed for another reason:\n%s", log.String())
++ }
++}
++
++// moveHelper makes the directory dir the artifact directory of cfg's function: the test binary is linked (or copied)
++// there and run from there.
++func moveHelper(t *testing.T, cfg *config.WorkerConfig, dir string) {
++ t.Helper()
++ exe := filepath.Join(dir, "worker.test")
++ if err := os.Link(cfg.Functions[0].Command[0], exe); err != nil {
++ b, err := os.ReadFile(cfg.Functions[0].Command[0])
++ if err != nil {
++ t.Fatal(err)
++ }
++ if err := os.WriteFile(exe, b, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ }
++ cfg.Functions[0].Command[0] = exe
++}
++
++// TestArtifactUnreachable: a job's user must reach its artifact directory by its path: one under a directory it may
++// not search fails the job without running, though the Node (root) opens it; once that directory may be searched, the
++// job runs (as root only: otherwise jobs run as the Node's own user, who could not open it either).
++func TestArtifactUnreachable(t *testing.T) {
++ if os.Geteuid() != 0 {
++ t.Skip("jobs run as the Node's own user unless it is root")
++ }
++ cfg := helperConfig(t, config.WorkerCaps{MaxMemBytes: 8 << 30}, "quick")
++ top := t.TempDir()
++ if err := os.Chmod(filepath.Dir(top), 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.Chmod(top, 0o700); err != nil {
++ t.Fatal(err)
++ }
++ dir := filepath.Join(top, "fn")
++ if err := os.Mkdir(dir, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ moveHelper(t, &cfg, dir)
++ w, s := newRunWorkerConfig(t, cfg)
++ log := &logBuf{}
++ w.log = slog.New(slog.NewTextHandler(log, nil))
++ j := testJob(1, "quick")
++ run(t, w, j)
++ ex := s.exit(t, j.ID)
++ if ex.Reason != protocol.ExitFailed || ex.Usage.StartedMS != nil || s.count(protocol.TypeJobStarted, j.ID) != 0 {
++ t.Fatalf("an artifact directory its user may not reach: %+v", ex)
++ }
++ if !strings.Contains(log.String(), "permission denied") {
++ t.Fatalf("an artifact directory its user may not reach: the job failed for another reason:\n%s", log.String())
++ }
++ if err := os.Chmod(top, 0o711); err != nil {
++ t.Fatal(err)
++ }
++ j = testJob(2, "quick")
++ run(t, w, j)
++ if ex := s.exit(t, j.ID); ex.Reason != protocol.ExitExited || string(ex.Result) != `{"ok":true}` {
++ t.Fatalf("an artifact directory its user may reach: %+v %s", ex, ex.Result)
++ }
++}
++
++// TestIOMaxVerified: a job's cgroup holds io.max for every whole block device with a medium, read back from the
++// kernel, and a device whose io.max is not the job's (as when the kernel ignores a value) keeps the job from running.
++func TestIOMaxVerified(t *testing.T) {
++ newRunWorker(t, config.WorkerCaps{}, "quick") // skips where jobs cannot run sandboxed
++ caps := config.WorkerCaps{MaxIOBps: 3 << 20, MaxIOPS: 300}.WithDefaults()
++ cg, err := newJobCgroup(caps, protocol.JobLimits{MemBytes: 64 << 20})
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer cg.remove()
++ devs, err := blockDevices()
++ if err != nil {
++ t.Fatal(err)
++ }
++ b, err := os.ReadFile(filepath.Join(cg.dir, "io.max"))
++ if err != nil {
++ t.Fatal(err)
++ }
++ for _, d := range devs {
++ if !strings.Contains(string(b), d+" rbps=3145728 wbps=3145728 riops=300 wiops=300\n") {
++ t.Errorf("block device %s is not limited in io.max:\n%s", d, b)
++ }
++ }
++ if len(devs) == 0 {
++ t.Skip("no block device to limit")
++ }
++ cg.io = "rbps=3145728 wbps=3145728 riops=301 wiops=300"
++ if err := cg.checkIO(); err == nil || !strings.Contains(err.Error(), devs[0]) {
++ t.Fatalf("io.max that is not the job's: %v, want refused", err)
++ }
++}
++
++// TestBlockDevicesUnreadable: a disk whose dev (or size) cannot be read is an error, never a disk left without io.max;
++// an empty disk and a hidden one are not limited; a disk without a hidden attribute (an older kernel) is.
++func TestBlockDevicesUnreadable(t *testing.T) {
++ sys := t.TempDir()
++ disk := func(name string, attrs map[string]string) {
++ if err := os.Mkdir(filepath.Join(sys, name), 0o755); err != nil {
++ t.Fatal(err)
++ }
++ for k, v := range attrs {
++ if err := os.WriteFile(filepath.Join(sys, name, k), []byte(v+"\n"), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ }
++ }
++ disk("sda", map[string]string{"dev": "8:0", "size": "100", "hidden": "0"})
++ disk("sr0", map[string]string{"dev": "11:0", "size": "0", "hidden": "0"})
++ disk("nvme0c0n1", map[string]string{"dev": "259:1", "size": "100", "hidden": "1"})
++ disk("vda", map[string]string{"dev": "252:0", "size": "100"})
++ devs, err := blockDevicesIn(sys)
++ if err != nil || !slices.Equal(devs, []string{"8:0", "252:0"}) {
++ t.Fatalf("block devices %v, %v; want [8:0 252:0]", devs, err)
++ }
++ // dev a directory: unreadable even by root.
++ disk("sdb", map[string]string{"size": "100", "hidden": "0"})
++ if err := os.Mkdir(filepath.Join(sys, "sdb", "dev"), 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if devs, err := blockDevicesIn(sys); err == nil || !strings.Contains(err.Error(), "sdb") {
++ t.Fatalf("a disk whose dev cannot be read: %v, %v; want an error", devs, err)
++ }
++ if err := os.Remove(filepath.Join(sys, "sdb", "dev")); err != nil {
++ t.Fatal(err)
++ }
++ if devs, err := blockDevicesIn(sys); err == nil || !strings.Contains(err.Error(), "sdb") {
++ t.Fatalf("a disk without dev: %v, %v; want an error", devs, err)
++ }
++}
++
++// TestCAStaged: of the CA directories, a job's root gets only the X.509 certificates, each file rewritten with its
++// CERTIFICATE blocks alone: a private key filed with them (alone, in a bundle, in a subdirectory or in a mount nested
++// there, as root) and a file holding no certificate never reach it; symlinks are kept as they are.
++func TestCAStaged(t *testing.T) {
++ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
++ if err != nil {
++ t.Fatal(err)
++ }
++ tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test CA"},
++ NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
++ der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
++ if err != nil {
++ t.Fatal(err)
++ }
++ kder, err := x509.MarshalPKCS8PrivateKey(key)
++ if err != nil {
++ t.Fatal(err)
++ }
++ cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
++ priv := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: kder})
++ fake := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: kder}) // a key labelled a certificate
++ src := t.TempDir()
++ certs := filepath.Join(src, "certs")
++ write := func(p string, b []byte) {
++ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.WriteFile(p, b, 0o644); err != nil {
++ t.Fatal(err)
++ }
++ }
++ write(filepath.Join(certs, "ca.pem"), cert)
++ write(filepath.Join(certs, "bundle.crt"), slices.Concat([]byte("# a comment\n"), cert, priv, fake, []byte("MIIB...raw\n"), cert))
++ write(filepath.Join(certs, "server.key"), priv)
++ write(filepath.Join(certs, "java", "cacerts"), kder)
++ write(filepath.Join(certs, "sub", "ca.pem"), cert)
++ write(filepath.Join(certs, "sub", "server.key"), slices.Concat(priv, fake))
++ if err := os.Symlink("ca.pem", filepath.Join(certs, "1a2b3c4d.0")); err != nil {
++ t.Fatal(err)
++ }
++ if os.Geteuid() == 0 {
++ mnt := filepath.Join(certs, "mnt")
++ if err := os.Mkdir(mnt, 0o755); err != nil {
++ t.Fatal(err)
++ }
++ if err := syscall.Mount("tmpfs", mnt, "tmpfs", 0, "mode=0755,size=1m"); err != nil {
++ t.Fatal(err)
++ }
++ defer syscall.Unmount(mnt, syscall.MNT_DETACH)
++ write(filepath.Join(mnt, "server.key"), priv)
++ write(filepath.Join(mnt, "server.pem"), slices.Concat(cert, priv))
++ }
++ r := t.TempDir()
++ if err := stageCA(r, []string{certs, filepath.Join(src, "absent.pem")}); err != nil {
++ t.Fatal(err)
++ }
++ got := map[string]string{}
++ err = filepath.WalkDir(r, func(p string, d fs.DirEntry, err error) error {
++ if err != nil || d.IsDir() {
++ return err
++ }
++ rel, _ := filepath.Rel(r+certs, p)
++ if d.Type()&fs.ModeSymlink != 0 {
++ l, err := os.Readlink(p)
++ got[rel] = "-> " + l
++ return err
++ }
++ b, err := os.ReadFile(p)
++ got[rel] = string(b)
++ return err
++ })
++ if err != nil {
++ t.Fatal(err)
++ }
++ want := map[string]string{"ca.pem": string(cert), "bundle.crt": string(cert) + string(cert),
++ "sub/ca.pem": string(cert), "1a2b3c4d.0": "-> ca.pem"}
++ if os.Geteuid() == 0 {
++ want["mnt/server.pem"] = string(cert)
++ }
++ if !maps.Equal(got, want) {
++ t.Fatalf("staged %v\nwant %v", got, want)
++ }
++}
++
++// TestShortLivedJob: a function that exits at once still starts, and its result and code arrive: the Node checks its
++// confinement and sets RLIMIT_CPU while the sandbox init waits to execute it, never once it may have ended.
++func TestShortLivedJob(t *testing.T) {
++ w, s := newRunWorker(t, config.WorkerCaps{}, "quick")
++ for i := 1; i <= 10; i++ {
++ j := testJob(i, "quick")
++ run(t, w, j)
++ ex := s.exit(t, j.ID)
++ if ex.Reason != protocol.ExitExited || ex.Code == nil || *ex.Code != 0 || string(ex.Result) != `{"ok":true}` ||
++ s.count(protocol.TypeJobStarted, j.ID) != 1 {
++ t.Fatalf("%s: %+v %s", j.ID, ex, ex.Result)
++ }
++ }
++}
++
++// TestForkBomb: pids.max and RLIMIT_NPROC stop a job that starts processes without end; the Node is untouched and the
++// job ends when its first process exits, taking the others with it.
++func TestForkBomb(t *testing.T) {
++ const pids = 32
++ w, s := newRunWorker(t, config.WorkerCaps{MaxPids: pids}, "forkbomb")
++ j := testJob(1, "forkbomb")
++ begin := time.Now()
++ run(t, w, j)
++ ex := s.exit(t, j.ID)
++ var r struct {
++ Started int
++ Err string
++ }
++ if err := json.Unmarshal(ex.Result, &r); err != nil || ex.Reason != protocol.ExitExited {
++ t.Fatalf("%+v %s: %v", ex, ex.Result, err)
++ }
++ if r.Err == "<nil>" || r.Started >= pids {
++ t.Fatalf("started %d processes (max_pids %d), last error %s", r.Started, pids, r.Err)
++ }
++ if d := time.Since(begin); d > 30*time.Second {
++ t.Fatalf("the job's processes outlived it: it ended after %s", d)
++ }
++}
++
++// TestEgressPolicy: only egress none is enforced; with another policy the probe fails (the class is not advertised)
++// and a job fails without running.
++func TestEgressPolicy(t *testing.T) {
++ for _, e := range []string{config.EgressPublic, config.EgressOpenVibeOnly} {
++ s := &sink{}
++ cfg := helperConfig(t, config.WorkerCaps{}, "env")
++ cfg.Egress = e
++ w := New(cfg, s.send, quiet())
++ t.Cleanup(w.Close)
++ if err := w.Probe(); err == nil || !strings.Contains(err.Error(), "egress") {
++ t.Fatalf("%s: probe %v, want refused", e, err)
++ }
++ j := testJob(1, "env")
++ run(t, w, j)
++ if ex := s.exit(t, j.ID); ex.Reason != protocol.ExitFailed || ex.Usage.StartedMS != nil || s.count(protocol.TypeJobStarted, j.ID) != 0 {
++ t.Fatalf("%s: %+v", e, ex)
++ }
++ }
++}
++
++// TestRootPlan: the private root holds the device nodes; no bind is, holds or lies in /etc/ssl, /etc/pki (whose CA
++// certificates are copied, not bound), the Node's directories nor /var; a system path holding a Node directory is
++// refused.
++func TestRootPlan(t *testing.T) {
++ if _, _, err := rootPlan([]string{"/usr/local/openvibe-node"}); err == nil {
++ t.Error("a Node directory in /usr was not refused")
++ }
++ binds, _, err := rootPlan([]string{"/var/lib/openvibe-node", "/etc/openvibe-node"})
++ if err != nil {
++ t.Fatal(err)
++ }
++ for _, b := range binds {
++ if _, ok := overlaps(b.path, []string{"/etc/ssl", "/etc/pki", "/etc/ca-certificates", "/etc/shadow"}); ok ||
++ under(b.path, []string{"/var", "/etc/openvibe-node", "/home", "/root", "/run"}) {
++ t.Errorf("%s is bound", b.path)
++ }
++ }
++ if !slices.ContainsFunc(binds, func(b hostBind) bool { return b.path == "/dev/null" && b.dev }) {
++ t.Error("/dev/null is not bound")
++ }
++}
++
++// TestArtifactRefused: an artifact directory that is, holds or lies in the host's kernel interfaces, runtime sockets,
++// configuration or the Node's directories, or that is or holds a shared directory, is refused; one whose tree holds a
++// Unix socket or a FIFO is refused at boot; a function with one keeps the probe from passing; and the directory a job
++// opens is checked where its path leads at that moment.
++func TestArtifactRefused(t *testing.T) {
++ for _, a := range []string{"/", "/run", "/run/user", "/var/run", "/var", "/proc/1/root", "/sys", "/dev", "/etc",
++ "/tmp", "/var/tmp", "/home"} {
++ if err := checkArtifact(nil, a); err == nil {
++ t.Errorf("artifact %s was not refused", a)
++ }
++ }
++ for _, c := range []struct{ node, artifact string }{{"/srv/ov/state", "/srv/ov"}, {"/srv/ov", "/srv/ov/bin"},
++ {"/srv/ov", "/srv/ov"}} {
++ if err := checkArtifact([]string{c.node}, c.artifact); err == nil {
++ t.Errorf("artifact %s with the Node in %s was not refused", c.artifact, c.node)
++ }
++ }
++ dir := t.TempDir()
++ if err := checkArtifact(nil, dir); err != nil {
++ t.Fatalf("artifact %s: %v", dir, err)
++ }
++ if err := scanArtifact(dir); err != nil {
++ t.Fatalf("artifact %s: %v", dir, err)
++ }
++ if err := os.Mkdir(filepath.Join(dir, "sub"), 0o755); err != nil {
++ t.Fatal(err)
++ }
++ l, err := net.Listen("unix", filepath.Join(dir, "sub", "s"))
++ if err != nil {
++ t.Fatal(err)
++ }
++ if err := scanArtifact(dir); err == nil || !strings.Contains(err.Error(), "socket") {
++ t.Errorf("artifact holding a Unix socket: %v", err)
++ }
++ l.Close()
++ if err := syscall.Mkfifo(filepath.Join(dir, "f"), 0o600); err != nil {
++ t.Fatal(err)
++ }
++ if err := scanArtifact(dir); err == nil || !strings.Contains(err.Error(), "FIFO") {
++ t.Errorf("artifact holding a FIFO: %v", err)
++ }
++ // A path swapped for a symlink to a host-only directory after boot: the directory opened is refused.
++ link := filepath.Join(t.TempDir(), "fn")
++ if err := os.Symlink("/run", link); err != nil {
++ t.Fatal(err)
++ }
++ if f, err := openArtifact(nil, nil, nil, link); err == nil || !strings.Contains(err.Error(), "/run") {
++ if f != nil {
++ f.Close()
++ }
++ t.Errorf("artifact %s leading to /run: %v, want refused", link, err)
++ }
++ binds, links, err := rootPlan(nil)
++ if err != nil {
++ t.Fatal(err)
++ }
++ if f, err := openArtifact(nil, binds, links, "/usr/bin"); !usrTrusted(t) {
++ if err == nil || !strings.Contains(err.Error(), "could change") {
++ t.Errorf("artifact in a system path whose owner is not mapped here: %v, want refused", err)
++ }
++ } else if f == nil || err != nil {
++ t.Errorf("artifact in a system path: %v %v, want it copied as any other", f, err)
++ } else {
++ f.Close()
++ }
++ cfg := helperConfig(t, config.WorkerCaps{}, "env")
++ cfg.Functions[0].ArtifactDir = "/run"
++ w := New(cfg, (&sink{}).send, quiet())
++ t.Cleanup(w.Close)
++ if err := w.Probe(); err == nil || !strings.Contains(err.Error(), "/run") {
++ t.Fatalf("probe with artifact_dir /run: %v, want refused", err)
++ }
++}
++
++// TestUsageSecondsContiguous: a function that holds for several whole wall-clock seconds after job_started sends
++// exactly one job_usage for each fully elapsed second, `second` 0 then 1 then ... with no gap and no duplicate, and
++// its job_exit's usage is never smaller than the seconds already sent (wall_ms >= (n+1)*1000 for every second n).
++func TestUsageSecondsContiguous(t *testing.T) {
++ // A job runs only sandboxed (there is no unconfined mode, even for a test): this runs where the sandbox can.
++ cfg := helperConfig(t, config.WorkerCaps{MaxMemBytes: 8 << 30})
++ exe, err := os.Executable()
++ if err != nil {
++ t.Fatal(err)
++ }
++ cfg.Functions = []config.FunctionConfig{{Name: "meter", Version: "1.0.0",
++ Command: []string{exe, "-test.run=^TestUsageHelperProcess$", "--", "meter"}, Env: map[string]string{"GOMAXPROCS": "2"}}}
++ w, s := newRunWorkerConfig(t, cfg)
++
++ const holdMS = 2500 // two whole seconds, so seconds 0 and 1 are sent before the exit
++ j := testJob(1, "meter")
++ j.Args = json.RawMessage(fmt.Sprintf(`{"hold_ms":%d}`, holdMS))
++ if f, r := w.Admit(j); f != "" {
++ t.Fatalf("%s %s", f, r)
++ }
++ w.Launch(j.ID)
++ ex := s.exit(t, j.ID)
++ if ex.Reason != protocol.ExitExited || ex.Code == nil || *ex.Code != 0 || ex.Usage.StartedMS == nil {
++ t.Fatalf("%+v", ex)
++ }
++ if n := s.count(protocol.TypeJobStarted, j.ID); n != 1 {
++ t.Fatalf("job_started sent %d times, want 1", n)
++ }
++ started := s.wait(t, protocol.TypeJobStarted, j.ID, 1, 5*time.Second).(protocol.JobStarted)
++ if started.StartedMS != *ex.Usage.StartedMS {
++ t.Fatalf("job_started at %d, job_exit usage at %d", started.StartedMS, *ex.Usage.StartedMS)
++ }
++ var seconds []int64
++ for _, m := range s.all() {
++ u, ok := m.(protocol.JobUsage)
++ if !ok || u.ID != j.ID {
++ continue
++ }
++ if u.StartedMS != *ex.Usage.StartedMS || u.CPUMS == nil {
++ t.Fatalf("usage %+v", u)
++ }
++ seconds = append(seconds, u.Second)
++ }
++ if len(seconds) < 2 {
++ t.Fatalf("usage seconds %v, want 0..n-1 for a job that held %d ms", seconds, holdMS)
++ }
++ for n, second := range seconds {
++ if second != int64(n) {
++ t.Fatalf("usage seconds %v, want 0..%d with no gap or duplicate", seconds, len(seconds)-1)
++ }
++ if ex.Usage.WallMS < (second+1)*1000 {
++ t.Fatalf("usage second %d sent but wall_ms is %d, want at least %d", second, ex.Usage.WallMS, (second+1)*1000)
++ }
++ }
++}
++
++// TestSeccompFilter: the program fits the kernel's limit and every jump lands inside it.
++func TestSeccompFilter(t *testing.T) {
++ p, err := seccompFilter()
++ if err != nil {
++ t.Skip(err)
++ }
++ if len(p) > 4096 {
++ t.Fatalf("%d instructions", len(p))
++ }
++ for i, ins := range p {
++ if ins.Code&0x07 == 0x05 && (i+1+int(ins.Jt) >= len(p) || i+1+int(ins.Jf) >= len(p)) {
++ t.Fatalf("instruction %d jumps out of the program: %+v", i, ins)
++ }
++ }
++ if last := p[len(p)-1]; last.Code != 0x06 {
++ t.Fatalf("the program does not end in a return: %+v", last)
++ }
++}
++
++// usrTrusted reports whether /usr is root's (or this user's) as seen here. In a user namespace that does not map
++// the host's root, as some test sandboxes run, /usr shows the overflow owner (65534) and checkRoute refuses a path
++// through it: it cannot tell root from any other unmapped host user.
++func usrTrusted(t *testing.T) bool {
++ t.Helper()
++ var st syscall.Stat_t
++ if err := syscall.Stat("/usr", &st); err != nil {
++ t.Fatal(err)
++ }
++ return slices.Contains(trustedUIDs(), st.Uid)
++}
++
++// TestLimitSwap: memory.swap.max is set to 0 and read back; a missing control (no swap accounting) is refused while
++// the host has swap, or when whether it has some is unknown, and never created.
++func TestLimitSwap(t *testing.T) {
++ dir, cg := t.TempDir(), t.TempDir()
++ const header = "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"
++ none, some := filepath.Join(dir, "none"), filepath.Join(dir, "some")
++ if err := os.WriteFile(none, []byte(header), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ if err := os.WriteFile(some, []byte(header+"/swap.img\tfile\t\t2097148\t\t0\t\t-2\n"), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ if err := limitSwap(cg, some); err == nil || !strings.Contains(err.Error(), "while the host has swap") {
++ t.Errorf("no memory.swap.max, host swap on: %v, want refused", err)
++ }
++ if err := limitSwap(cg, filepath.Join(dir, "missing")); err == nil || !strings.Contains(err.Error(), "unknown") {
++ t.Errorf("no memory.swap.max, host swap unknown: %v, want refused", err)
++ }
++ if err := limitSwap(cg, none); err != nil {
++ t.Errorf("no memory.swap.max, no host swap: %v", err)
++ }
++ ctl := filepath.Join(cg, "memory.swap.max")
++ if _, err := os.Stat(ctl); !errors.Is(err, os.ErrNotExist) {
++ t.Fatalf("a missing memory.swap.max was created: %v", err)
++ }
++ if err := os.WriteFile(ctl, []byte("max\n"), 0o644); err != nil {
++ t.Fatal(err)
++ }
++ if err := limitSwap(cg, some); err != nil {
++ t.Errorf("memory.swap.max there: %v", err)
++ }
++ if b, _ := os.ReadFile(ctl); string(b) != "0" {
++ t.Errorf("memory.swap.max = %q, want 0", b)
++ }
++ if os.Geteuid() != 0 {
++ if err := os.Chmod(ctl, 0o444); err != nil {
++ t.Fatal(err)
++ }
++ if err := limitSwap(cg, none); err == nil {
++ t.Error("memory.swap.max not writable: no error")
++ }
++ }
++}
++
++// TestSpecNotInCmdline: the sandbox init gets its spec, the function's environment included, on specFD and never in
++// its argv: while it waits for the Node's go, its /proc/<pid>/cmdline is its name alone. It needs only user, mount and
++// PID namespaces, no cgroup: it runs where the sandboxed jobs cannot.
++func TestSpecNotInCmdline(t *testing.T) {
++ const envMarker = "marker-from-the-environment"
++ sf, err := specFile(sandboxSpec{Root: t.TempDir(), Work: "/tmp/w", Disk: 1 << 20, Argv: []string{"/bin/true"},
++ Env: []string{"OPENVIBE_MARKER=" + envMarker}})
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer sf.Close()
++ sr, sw, err := os.Pipe()
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer sr.Close()
++ gr, gw, err := os.Pipe()
++ if err != nil {
++ t.Fatal(err)
++ }
++ defer gw.Close()
++ cmd := &exec.Cmd{Path: "/proc/self/exe", Args: []string{sandboxArg0}, Env: []string{}, Dir: "/", Stderr: os.Stderr,
++ ExtraFiles: []*os.File{nil, sw, gr, sf}, // statusFD, goFD, specFD
++ SysProcAttr: &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL,
++ Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNS | syscall.CLONE_NEWPID,
++ UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Geteuid(), Size: 1}},
++ GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getegid(), Size: 1}}}}
++ err = cmd.Start()
++ sw.Close()
++ gr.Close()
++ if err != nil {
++ if os.Getenv("OPENVIBE_WORKER_TESTS") == "require" {
++ t.Fatal(err)
++ }
++ t.Skipf("no user, mount and PID namespace here: %v", err)
++ }
++ defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }()
++ _ = sr.SetReadDeadline(time.Now().Add(10 * time.Second))
++ var b [1]byte
++ if _, err := io.ReadFull(sr, b[:]); err != nil {
++ t.Fatalf("the sandbox init did not report: %v", err)
++ }
++ if b[0] != 0 {
++ msg, _ := io.ReadAll(sr)
++ if strings.Contains(string(msg), "spec") || os.Getenv("OPENVIBE_WORKER_TESTS") == "require" {
++ t.Fatalf("the sandbox init failed: %s%s", b[:], msg)
++ }
++ t.Skipf("the sandbox init cannot stand here: %s%s", b[:], msg)
++ }
++ c, err := os.ReadFile("/proc/" + strconv.Itoa(cmd.Process.Pid) + "/cmdline")
++ if err != nil {
++ t.Fatal(err)
++ }
++ if string(c) != sandboxArg0+"\x00" || strings.Contains(string(c), envMarker) {
++ t.Errorf("the sandbox init's command line is %q, want its name alone", c)
++ }
++ gw.Close() // no go: the init exits instead of executing the function
++ if msg, _ := io.ReadAll(sr); !strings.Contains(string(msg), "did not let the function start") {
++ t.Errorf("the init after no go: %q", msg)
++ }
++}
ddiff --git a/internal/worker/sandbox_linux.go b/internal/worker/sandbox_linux.go
nnew file mode 100644
iindex 0000000..ef01853
---- /dev/null
++++ b/internal/worker/sandbox_linux.go
@@@ -0,0 +1,980 @@
++//go:build linux
++
++package worker
++
++import (
++ "crypto/x509"
++ "encoding/json"
++ "encoding/pem"
++ "errors"
++ "fmt"
++ "io"
++ "io/fs"
++ "os"
++ "path/filepath"
++ "runtime"
++ "slices"
++ "sort"
++ "strconv"
++ "strings"
++ "syscall"
++ "unsafe"
++)
++
++// A job starts as the Node's own executable (/proc/self/exe) with sandboxArg0 as its only argument and its sandboxSpec
++// on specFD (a memfd: in argv, the function's environment would show in /proc/<pid>/cmdline to every host user until
++// the function executes), already in its namespaces and its cgroup, holding CAP_SYS_ADMIN of its user namespace only (an ambient capability).
++// It builds the job's private root, sets the rlimits, gives up every capability, sets no_new_privs and the seccomp
++// filter, and only then executes the function. It reports a failure on statusFD as text. Once confined it writes one
++// NUL byte there and waits for one byte on goFD: the Node checks the confinement and sets RLIMIT_CPU meanwhile, while
++// the process is sure to be alive. The function's execve then closes statusFD. artifactFD is the function's artifact
++// directory, opened by the Node, when the init is to copy it.
++const (
++ sandboxArg0 = "openvibe-node-sandbox"
++ statusFD = 4 // fd 3 is the function's result
++ goFD = 5
++ specFD = 6
++ artifactFD = 7
++)
++
++// sandboxSpec is everything the sandbox init needs, from the Node.
++type sandboxSpec struct {
++ Root string `json:"root"` // the host directory the private root is mounted on
++ Work string `json:"work"` // the working directory, inside the private root
++ Disk int64 `json:"disk"` // the size of the job's /tmp, and the most its artifact's copy may take
++ Binds []bindSpec `json:"binds"`
++ // Artifact is where the copy of the artifact directory open on artifactFD goes (its host path); "" for none.
++ Artifact string `json:"artifact,omitempty"`
++ Links []linkSpec `json:"links"`
++ Limits []rlimit `json:"limits"`
++ Argv []string `json:"argv"`
++ Env []string `json:"env"`
++}
++
++// bindSpec bind-mounts the host's path at the same path in the private root, read-only unless it is a device node.
++// The bind is made by path in the job's own mount namespace: a mount cannot be bound from another namespace.
++type bindSpec struct {
++ Path string `json:"path"`
++ Dev bool `json:"dev,omitempty"`
++}
++
++type linkSpec struct {
++ Path string `json:"path"`
++ Target string `json:"target"`
++}
++
++type rlimit struct {
++ Res int `json:"res"`
++ Cur uint64 `json:"cur"`
++ Max uint64 `json:"max"`
++}
++
++// The rlimit resources (the same numbers on amd64 and arm64; seccomp covers no other architecture).
++const (
++ rlimitCPU = 0
++ rlimitFSIZE = 1
++ rlimitCORE = 4
++ rlimitNPROC = 6
++ rlimitNOFILE = 7
++ rlimitAS = 9
++)
++
++func init() {
++ if len(os.Args) == 1 && os.Args[0] == sandboxArg0 {
++ sandboxMain()
++ }
++}
++
++// sandboxMain never returns: it executes the function or exits 126 after writing why it could not to statusFD.
++func sandboxMain() {
++ // no_new_privs, the seccomp filter and the capabilities belong to this thread, the one that calls execve.
++ runtime.LockOSThread()
++ err := enterSandbox()
++ status := os.NewFile(statusFD, "status")
++ _, _ = status.WriteString(err.Error())
++ os.Exit(126)
++}
++
++func enterSandbox() error {
++ f := os.NewFile(specFD, "spec")
++ arg, err := io.ReadAll(f)
++ f.Close()
++ if err != nil {
++ return fmt.Errorf("reading the sandbox spec: %w", err)
++ }
++ var s sandboxSpec
++ if err := json.Unmarshal(arg, &s); err != nil {
++ return fmt.Errorf("reading the sandbox spec: %w", err)
++ }
++ if len(s.Argv) == 0 {
++ return errors.New("no command")
++ }
++ filter, err := seccompFilter()
++ if err != nil {
++ return err
++ }
++ step := func(what string, err error) error {
++ if err != nil {
++ return fmt.Errorf("%s: %w", what, err)
++ }
++ return nil
++ }
++ r := s.Root
++ if err := step("making mounts private", syscall.Mount("", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")); err != nil {
++ return err
++ }
++ // The private root holds the copies of the CA certificates: a few hundred KiB on a common system.
++ if err := step("mounting the private root", syscall.Mount("tmpfs", r, "tmpfs", syscall.MS_NOSUID|syscall.MS_NODEV, "mode=0755,size=16m")); err != nil {
++ return err
++ }
++ if err := os.Mkdir(r+"/tmp", 0o755); err != nil {
++ return err
++ }
++ if err := step("mounting /tmp", syscall.Mount("tmpfs", r+"/tmp", "tmpfs", syscall.MS_NOSUID|syscall.MS_NODEV,
++ "mode=0700,size="+strconv.FormatInt(s.Disk, 10))); err != nil {
++ return err
++ }
++ if err := os.MkdirAll(r+s.Work, 0o700); err != nil {
++ return err
++ }
++ for _, b := range s.Binds {
++ if err := step("bind-mounting "+b.Path, bind(r, b)); err != nil {
++ return err
++ }
++ }
++ for _, l := range s.Links {
++ if err := os.MkdirAll(filepath.Dir(r+l.Path), 0o755); err != nil {
++ return err
++ }
++ if err := os.Symlink(l.Target, r+l.Path); err != nil {
++ return err
++ }
++ }
++ if err := step("copying the CA certificates", stageCA(r, caPaths)); err != nil {
++ return err
++ }
++ var artifactDev uint64
++ if s.Artifact != "" {
++ if artifactDev, err = stageArtifact(r, s.Artifact, s.Disk); err != nil {
++ return step("copying the artifact directory "+s.Artifact, err)
++ }
++ }
++ if err := os.Mkdir(r+"/proc", 0o755); err != nil {
++ return err
++ }
++ if err := step("mounting /proc", syscall.Mount("proc", r+"/proc", "proc", syscall.MS_NOSUID|syscall.MS_NODEV|syscall.MS_NOEXEC, "")); err != nil {
++ return err
++ }
++ // The private root becomes /; the host's root, stacked on it by pivot_root, is detached with everything under it.
++ if err := step("entering the private root", syscall.Chdir(r)); err != nil {
++ return err
++ }
++ if err := step("pivot_root", syscall.PivotRoot(".", ".")); err != nil {
++ return err
++ }
++ if err := step("detaching the host's root", syscall.Unmount(".", syscall.MNT_DETACH)); err != nil {
++ return err
++ }
++ if err := step("making / read-only", syscall.Mount("", "/", "", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|
++ syscall.MS_NOSUID|syscall.MS_NODEV, "")); err != nil {
++ return err
++ }
++ // The job's own path to its artifact directory must lead to the copy, never to the host's directory under it. The
++ // Node checked that no host user but root or the Node's own can change that path while the job runs (checkRoute).
++ if s.Artifact != "" {
++ var st syscall.Stat_t
++ if err := syscall.Stat(s.Artifact, &st); err != nil || st.Dev != artifactDev {
++ return fmt.Errorf("the artifact directory %s does not lead to its copy in the private root (%v)", s.Artifact, err)
++ }
++ }
++ if err := syscall.Chdir(s.Work); err != nil {
++ return err
++ }
++ // Only fds 0-3 reach the function; statusFD closes when execve succeeds.
++ ents, err := os.ReadDir("/proc/self/fd")
++ if err != nil {
++ return err
++ }
++ for _, e := range ents {
++ if fd, err := strconv.Atoi(e.Name()); err == nil && fd > 3 {
++ syscall.CloseOnExec(fd)
++ }
++ }
++ // execve's arguments are made before the rlimits: past RLIMIT_AS this process may not be able to allocate.
++ path, err := syscall.BytePtrFromString(s.Argv[0])
++ if err != nil {
++ return err
++ }
++ argv, err := syscall.SlicePtrFromStrings(s.Argv)
++ if err != nil {
++ return err
++ }
++ envv, err := syscall.SlicePtrFromStrings(s.Env)
++ if err != nil {
++ return err
++ }
++ for _, l := range s.Limits {
++ lim := syscall.Rlimit{Cur: l.Cur, Max: l.Max}
++ if _, _, e := syscall.RawSyscall6(syscall.SYS_PRLIMIT64, 0, uintptr(l.Res), uintptr(unsafe.Pointer(&lim)), 0, 0, 0); e != 0 {
++ return fmt.Errorf("rlimit %d: %w", l.Res, e)
++ }
++ }
++ const (
++ prCapAmbient, prCapAmbientClearAll = 47, 4
++ prSetNoNewPrivs = 38
++ prSetSeccomp, seccompModeFilter = 22, 2
++ )
++ // The function runs as a non-zero uid of its user namespace: with no ambient capability and no file
++ // capabilities (no_new_privs), execve leaves it none.
++ if _, _, e := syscall.RawSyscall6(syscall.SYS_PRCTL, prCapAmbient, prCapAmbientClearAll, 0, 0, 0, 0); e != 0 {
++ return fmt.Errorf("dropping the ambient capabilities: %w", e)
++ }
++ if _, _, e := syscall.RawSyscall6(syscall.SYS_PRCTL, prSetNoNewPrivs, 1, 0, 0, 0, 0); e != 0 {
++ return fmt.Errorf("no_new_privs: %w", e)
++ }
++ prog := syscall.SockFprog{Len: uint16(len(filter)), Filter: &filter[0]}
++ if _, _, e := syscall.RawSyscall6(syscall.SYS_PRCTL, prSetSeccomp, seccompModeFilter, uintptr(unsafe.Pointer(&prog)), 0, 0, 0); e != 0 {
++ return fmt.Errorf("seccomp: %w", e)
++ }
++ runtime.KeepAlive(filter)
++ var b [1]byte
++ if _, err := syscall.Write(statusFD, b[:]); err != nil {
++ return fmt.Errorf("reporting ready: %w", err)
++ }
++ if n, err := syscall.Read(goFD, b[:]); n != 1 {
++ return fmt.Errorf("the Node did not let the function start (%v)", err)
++ }
++ _, _, e := syscall.RawSyscall(syscall.SYS_EXECVE, uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(&argv[0])),
++ uintptr(unsafe.Pointer(&envv[0])))
++ return fmt.Errorf("executing %s: %w", s.Argv[0], e)
++}
++
++// bind mounts b under the private root r, with every mount under it (a mount nested in /usr or the artifact
++// directory included) read-only, nosuid and nodev: mount_setattr with AT_RECURSIVE adds those flags to the whole
++// tree and leaves the others (noexec, atime, which the kernel may have locked) as they are.
++func bind(r string, b bindSpec) error {
++ dst := r + b.Path
++ fi, err := os.Stat(b.Path)
++ if err != nil {
++ return err
++ }
++ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
++ return err
++ }
++ if fi.IsDir() {
++ if err := os.Mkdir(dst, 0o755); err != nil && !errors.Is(err, os.ErrExist) {
++ return err
++ }
++ } else if f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0o644); err != nil {
++ return err
++ } else {
++ f.Close()
++ }
++ if err := syscall.Mount(b.Path, dst, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
++ return err
++ }
++ if b.Dev {
++ return nil
++ }
++ return readOnlyTree(dst)
++}
++
++// stageArtifact copies the artifact directory open on artifactFD, the host's src, into a tmpfs of at most size bytes
++// mounted where src leads in the private root r, and makes it read-only: the job sees the tree as it was when it
++// started, never what the host adds to it later (a socket or a FIFO a host process could be reached through, say).
++// That holds in a system path too: the copy covers what the bind of /usr (say) shows there, on a path to it the Node
++// checked that only root or the Node's own user can change (checkRoute). It returns the copy's
++// device number, for the job to check that its path to the artifact leads there. The copy is read as the job's own
++// user, who must be able to reach src by its path (search every directory above it) and list it, and holds
++// directories, regular files and symlinks only, with their permission bits and modification times (no owner, set-id
++// or sticky bit); what the job's user may not read is left out; anything else in the tree, or a mount nested in it (a
++// bind mount of the same filesystem included), fails the job. The copy counts towards the job's memory and its reads
++// towards its io.max.
++func stageArtifact(r, src string, size int64) (uint64, error) {
++ // Opened again by its path, as the job's user: the Node opened it with its own rights (root's, with run_as), and
++ // reopening that descriptor through /proc would skip the search permission of the directories above it.
++ f, err := os.OpenFile(src, os.O_RDONLY|syscall.O_DIRECTORY, 0)
++ if err != nil {
++ return 0, err
++ }
++ defer f.Close()
++ var st, node syscall.Stat_t
++ if err := syscall.Fstat(int(f.Fd()), &st); err != nil {
++ return 0, err
++ }
++ if err := syscall.Fstat(artifactFD, &node); err != nil {
++ return 0, err
++ }
++ syscall.Close(artifactFD)
++ if st.Dev != node.Dev || st.Ino != node.Ino {
++ return 0, errors.New("it changed since the Node checked it")
++ }
++ dst, err := inRoot(r, src)
++ if err != nil {
++ return 0, err
++ }
++ if err := syscall.Mount("tmpfs", dst, "tmpfs", syscall.MS_NOSUID|syscall.MS_NODEV,
++ "mode=0700,size="+strconv.FormatInt(size, 10)); err != nil {
++ return 0, err
++ }
++ var copied syscall.Stat_t
++ if err := syscall.Stat(dst, &copied); err != nil {
++ return 0, err
++ }
++ if err := copyDir(f, dst); err != nil {
++ return 0, err
++ }
++ if err := chmodTimes(dst, st); err != nil {
++ return 0, err
++ }
++ return copied.Dev, readOnlyTree(dst)
++}
++
++// inRoot returns the host path, free of symlinks, of the directory path leads to in the private root r (made if it
++// is missing): a symlink on the way, as /lib to /usr/lib or one in a bound system path, is followed as the job will
++// follow it, absolute or not, and never out of r.
++func inRoot(r, path string) (string, error) {
++ const resolveNoMagiclinks, resolveInRoot = 0x02, 0x10
++ root, err := syscall.Open(r, oPath|syscall.O_DIRECTORY|syscall.O_CLOEXEC, 0)
++ if err != nil {
++ return "", err
++ }
++ defer syscall.Close(root)
++ open := func() (int, error) {
++ return openat2(root, path, oPath|syscall.O_DIRECTORY|syscall.O_CLOEXEC, resolveNoMagiclinks|resolveInRoot)
++ }
++ fd, err := open()
++ if errors.Is(err, syscall.ENOENT) {
++ if err := os.MkdirAll(r+path, 0o755); err != nil {
++ return "", err
++ }
++ fd, err = open()
++ }
++ if err != nil {
++ return "", err
++ }
++ defer syscall.Close(fd)
++ p, err := os.Readlink("/proc/self/fd/" + strconv.Itoa(fd))
++ if err != nil {
++ return "", err
++ }
++ if !strings.HasPrefix(p, r+"/") {
++ return "", fmt.Errorf("%s leads to %s, not into the private root", path, p)
++ }
++ return p, nil
++}
++
++// maxCAFile is the largest CA certificate file copied (a bundle of every public CA is about 250 KiB).
++const maxCAFile = 4 << 20
++
++// stageCA copies the CA certificates at paths (those that exist) to the same paths under the private root r, as the
++// job's user, and nothing else: of a regular file, only the PEM CERTIFICATE blocks that parse as X.509 certificates,
++// written again without their headers and whatever else lay between them (a private key filed with them, say); a file
++// with none, one the job's user may not read and anything but a directory, a regular file or a symlink are left out.
++// Directories are walked whatever is mounted in them. A symlink is copied as it is: it leads only to what the private
++// root holds (the copied certificates or the system paths). The copies are read-only with the rest of the root.
++func stageCA(r string, paths []string) error {
++ for _, p := range paths {
++ if err := os.MkdirAll(filepath.Dir(r+p), 0o755); err != nil {
++ return err
++ }
++ if err := copyCA(p, r+p); err != nil {
++ return err
++ }
++ }
++ return nil
++}
++
++func copyCA(src, dst string) error {
++ fi, err := os.Lstat(src)
++ if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrPermission) {
++ return nil
++ } else if err != nil {
++ return err
++ }
++ switch {
++ case fi.Mode()&fs.ModeSymlink != 0:
++ t, err := os.Readlink(src)
++ if err != nil {
++ return err
++ }
++ return os.Symlink(t, dst)
++ case fi.IsDir():
++ f, err := os.Open(src)
++ if errors.Is(err, fs.ErrPermission) {
++ return nil
++ } else if err != nil {
++ return err
++ }
++ names, err := f.Readdirnames(-1)
++ f.Close()
++ if err != nil {
++ return fmt.Errorf("%s: %w", src, err)
++ }
++ if err := os.Mkdir(dst, 0o755); err != nil {
++ return err
++ }
++ for _, n := range names {
++ if err := copyCA(src+"/"+n, dst+"/"+n); err != nil {
++ return err
++ }
++ }
++ case fi.Mode().IsRegular():
++ certs, err := publicCerts(src)
++ if err != nil || len(certs) == 0 {
++ return err
++ }
++ w, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
++ if err != nil {
++ return err
++ }
++ _, err = w.Write(certs)
++ if cerr := w.Close(); err == nil {
++ err = cerr
++ }
++ return err
++ }
++ return nil
++}
++
++// publicCerts returns the certificates of the PEM file path (see stageCA); nil if it holds none, if it is no longer a
++// regular file, or if the job's user may not read it.
++func publicCerts(path string) ([]byte, error) {
++ f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
++ if errors.Is(err, fs.ErrPermission) || errors.Is(err, syscall.ELOOP) || errors.Is(err, fs.ErrNotExist) {
++ return nil, nil
++ } else if err != nil {
++ return nil, err
++ }
++ defer f.Close()
++ if fi, err := f.Stat(); err != nil || !fi.Mode().IsRegular() {
++ return nil, err
++ }
++ b, err := io.ReadAll(io.LimitReader(f, maxCAFile+1))
++ if err != nil {
++ return nil, fmt.Errorf("%s: %w", path, err)
++ }
++ if len(b) > maxCAFile {
++ return nil, nil
++ }
++ var out []byte
++ for {
++ var blk *pem.Block
++ if blk, b = pem.Decode(b); blk == nil {
++ return out, nil
++ }
++ if blk.Type != "CERTIFICATE" {
++ continue
++ }
++ if _, err := x509.ParseCertificate(blk.Bytes); err == nil {
++ out = append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: blk.Bytes})...)
++ }
++ }
++}
++
++// oPath is O_PATH (the same on amd64 and arm64): it opens a name without opening what it names, so no FIFO or device
++// is opened while the copy checks what each entry is.
++const oPath = 0x200000
++
++// copyDir copies the entries of the directory dir into the directory dst.
++func copyDir(dir *os.File, dst string) error {
++ names, err := dir.Readdirnames(-1)
++ if err != nil {
++ return err
++ }
++ for _, n := range names {
++ if err := copyEntry(int(dir.Fd()), n, dst+"/"+n); err != nil {
++ return err
++ }
++ }
++ return nil
++}
++
++// openEntry opens the entry name of the directory dir O_PATH (a symlink itself, not what it points to) with openat2,
++// which fails EXDEV if the entry is a mount point: unlike a device number, that also catches a bind mount of the
++// artifact's own filesystem (of a Node directory, say).
++func openEntry(dir int, name string) (int, error) {
++ const resolveNoXdev, resolveNoSymlinks, resolveBeneath = 0x01, 0x04, 0x08
++ return openat2(dir, name, oPath|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, resolveNoXdev|resolveNoSymlinks|resolveBeneath)
++}
++
++// openat2 opens name, relative to the directory dir, with the open flags and the RESOLVE_* flags resolve.
++func openat2(dir int, name string, flags, resolve uint64) (int, error) {
++ const sysOpenat2 = 437 // the same on amd64 and arm64
++ how := struct{ flags, mode, resolve uint64 }{flags: flags, resolve: resolve}
++ p, err := syscall.BytePtrFromString(name)
++ if err != nil {
++ return -1, err
++ }
++ fd, _, e := syscall.Syscall6(sysOpenat2, uintptr(dir), uintptr(unsafe.Pointer(p)), uintptr(unsafe.Pointer(&how)),
++ unsafe.Sizeof(how), 0, 0)
++ if e != 0 {
++ return -1, e
++ }
++ return int(fd), nil
++}
++
++// copyEntry copies the entry name of the directory open on dir to dst. It opens the entry O_PATH, checks what it is
++// on that descriptor, and opens a directory or a regular file for reading through it (/proc/self/fd), so what it
++// reads is what it checked even if the host swaps the entry meanwhile.
++func copyEntry(dir int, name, dst string) error {
++ fd, err := openEntry(dir, name)
++ if errors.Is(err, syscall.EXDEV) {
++ return fmt.Errorf("%s is a mount point: a mount nested in the artifact directory is not copied", dst)
++ } else if err != nil {
++ return fmt.Errorf("%s: %w", dst, err)
++ }
++ defer syscall.Close(fd)
++ var st syscall.Stat_t
++ if err := syscall.Fstat(fd, &st); err != nil {
++ return err
++ }
++ reopen := func(flags int) (int, error) {
++ return syscall.Open("/proc/self/fd/"+strconv.Itoa(fd), flags|syscall.O_CLOEXEC, 0)
++ }
++ switch st.Mode & syscall.S_IFMT {
++ case syscall.S_IFLNK:
++ var buf [4096]byte
++ empty := [1]byte{}
++ n, _, e := syscall.Syscall6(syscall.SYS_READLINKAT, uintptr(fd), uintptr(unsafe.Pointer(&empty[0])),
++ uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf)), 0, 0)
++ if e != 0 {
++ return fmt.Errorf("%s: %w", dst, e)
++ }
++ return os.Symlink(string(buf[:n]), dst)
++ case syscall.S_IFDIR:
++ d, err := reopen(syscall.O_RDONLY | syscall.O_DIRECTORY)
++ if errors.Is(err, syscall.EACCES) {
++ return nil // the job's user may not list it: it is left out
++ } else if err != nil {
++ return fmt.Errorf("%s: %w", dst, err)
++ }
++ sub := os.NewFile(uintptr(d), dst)
++ defer sub.Close()
++ if err := os.Mkdir(dst, 0o700); err != nil {
++ return err
++ }
++ if err := copyDir(sub, dst); err != nil {
++ return err
++ }
++ case syscall.S_IFREG:
++ r, err := reopen(syscall.O_RDONLY)
++ if errors.Is(err, syscall.EACCES) {
++ return nil // the job's user may not read it: it is left out
++ } else if err != nil {
++ return fmt.Errorf("%s: %w", dst, err)
++ }
++ src := os.NewFile(uintptr(r), dst)
++ defer src.Close()
++ w, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
++ if err != nil {
++ return err
++ }
++ _, err = io.Copy(w, src)
++ if cerr := w.Close(); err == nil {
++ err = cerr
++ }
++ if err != nil {
++ return fmt.Errorf("%s: %w", dst, err)
++ }
++ default:
++ return fmt.Errorf("%s is neither a directory, a regular file nor a symlink (a socket, FIFO or device)", dst)
++ }
++ return chmodTimes(dst, st)
++}
++
++// chmodTimes gives the copy dst the permission bits and the times of the original st.
++func chmodTimes(dst string, st syscall.Stat_t) error {
++ if err := os.Chmod(dst, os.FileMode(st.Mode&0o777)); err != nil {
++ return err
++ }
++ return syscall.UtimesNano(dst, []syscall.Timespec{st.Atim, st.Mtim})
++}
++
++// readOnlyTree makes the mount at path and every mount under it read-only, nosuid and nodev.
++func readOnlyTree(path string) error {
++ const (
++ sysMountSetattr = 442 // the same on amd64 and arm64
++ atRecursive = 0x8000
++ mountAttrRdonly, mountAttrNosuid, mountAttrNodev = 0x1, 0x2, 0x4
++ )
++ attr := struct{ set, clr, propagation, userns uint64 }{set: mountAttrRdonly | mountAttrNosuid | mountAttrNodev}
++ p, err := syscall.BytePtrFromString(path)
++ if err != nil {
++ return err
++ }
++ const atFdcwd = -100 // AT_FDCWD (the syscall package does not export it)
++ cwd := atFdcwd
++ if _, _, e := syscall.Syscall6(sysMountSetattr, uintptr(cwd), uintptr(unsafe.Pointer(p)), atRecursive,
++ uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr), 0); e != 0 {
++ return fmt.Errorf("mount_setattr: %w", e)
++ }
++ return nil
++}
++
++// cloneNamespaces are the clone flags that make a namespace; a job may not use them.
++const cloneNamespaces = syscall.CLONE_NEWNS | syscall.CLONE_NEWCGROUP | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC |
++ syscall.CLONE_NEWUSER | syscall.CLONE_NEWPID | syscall.CLONE_NEWNET
++
++// socketFamilies are the only socket families a job may open: Unix, IPv4, IPv6 and netlink (all scoped by its
++// network namespace; vsock, for one, is not).
++var socketFamilies = []uint32{syscall.AF_UNIX, syscall.AF_INET, syscall.AF_INET6, syscall.AF_NETLINK}
++
++// seccompFilter is the job's seccomp program: another architecture's system calls kill the process; clone3 fails
++// ENOSYS (libc falls back to clone); clone with a namespace flag and socket of another family fail; every call on the
++// allowlist passes; anything else fails EPERM (mount, unshare, setns, ptrace, bpf, keyctl, io_uring, perf_event_open,
++// kernel modules, ...).
++func seccompFilter() ([]syscall.SockFilter, error) {
++ if len(allowedSyscalls) == 0 {
++ return nil, fmt.Errorf("no seccomp allowlist for %s", runtime.GOARCH)
++ }
++ const (
++ ldAbs, jeq, jge, jset, ret = 0x20, 0x15, 0x35, 0x45, 0x06
++ retKill, retAllow, retErrno = 0x80000000, 0x7fff0000, 0x00050000
++ offNr, offArch, offArg0 = 0, 4, 16 // in struct seccomp_data; arg0's low 32 bits on little-endian
++ )
++ stmt := func(code uint16, k uint32) syscall.SockFilter { return syscall.SockFilter{Code: code, K: k} }
++ jump := func(k uint32, jt, jf uint8) syscall.SockFilter {
++ return syscall.SockFilter{Code: jeq, Jt: jt, Jf: jf, K: k}
++ }
++ errno := func(e syscall.Errno) syscall.SockFilter { return stmt(ret, retErrno|uint32(e)) }
++ p := []syscall.SockFilter{stmt(ldAbs, offArch), jump(auditArch, 1, 0), stmt(ret, retKill), stmt(ldAbs, offNr)}
++ if x32ABI {
++ p = append(p, syscall.SockFilter{Code: jge, Jt: 0, Jf: 1, K: 0x40000000}, errno(syscall.EPERM))
++ }
++ p = append(p, jump(sysClone3, 0, 1), errno(syscall.ENOSYS),
++ jump(sysClone, 0, 4), stmt(ldAbs, offArg0), syscall.SockFilter{Code: jset, Jt: 0, Jf: 1, K: cloneNamespaces},
++ errno(syscall.EPERM), stmt(ret, retAllow))
++ n := len(socketFamilies)
++ p = append(p, jump(sysSocket, 0, uint8(n+3)), stmt(ldAbs, offArg0))
++ for i, f := range socketFamilies {
++ p = append(p, jump(f, uint8(n-i), 0))
++ }
++ p = append(p, errno(syscall.EAFNOSUPPORT), stmt(ret, retAllow))
++ for _, nr := range allowedSyscalls {
++ p = append(p, jump(nr, 0, 1), stmt(ret, retAllow))
++ }
++ return append(p, errno(syscall.EPERM)), nil
++}
++
++// The read-only system paths of every private root (symlinks are recreated, as on merged-/usr systems), the few
++// files of /etc programs need to start, the device nodes, and copies of the public CA certificates (stageCA). Nothing
++// else of the host is there: no /home, /root, /var, /run, /srv, /mnt, /media, /sys, no rest of /etc (no
++// /etc/ssl/private, /etc/shadow nor any other key), and none of the Node's directories.
++var (
++ systemPaths = []string{"/usr", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/libx32"}
++ etcPaths = []string{"/etc/alternatives", "/etc/group", "/etc/hosts", "/etc/ld.so.cache", "/etc/ld.so.conf",
++ "/etc/ld.so.conf.d", "/etc/localtime", "/etc/mime.types", "/etc/nsswitch.conf", "/etc/passwd"}
++ // caPaths are where Debian, Fedora, Alpine, Arch and OpenSSL keep the public CA certificates: the directories that
++ // hold certificates (never their parent: /etc/ssl and /etc/pki/tls also hold private keys) and the bundles. They
++ // are copied, certificates only, never bound: a key or a mount filed among them would be seen whole.
++ caPaths = []string{"/etc/ssl/certs", "/etc/ssl/cert.pem", "/etc/ssl/ca-bundle.pem", "/etc/pki/tls/certs",
++ "/etc/pki/tls/cacert.pem", "/etc/pki/ca-trust/extracted", "/etc/ca-certificates/extracted"}
++ devPaths = []string{"/dev/full", "/dev/null", "/dev/random", "/dev/urandom", "/dev/zero"}
++ devLinks = []linkSpec{{"/dev/fd", "/proc/self/fd"}, {"/dev/stdin", "/proc/self/fd/0"},
++ {"/dev/stdout", "/proc/self/fd/1"}, {"/dev/stderr", "/proc/self/fd/2"}, {"/dev/shm", "/tmp"}}
++ // hiddenPaths are host paths the probe checks no job can see.
++ hiddenPaths = []string{"/var", "/etc/shadow", "/etc/ssl/private", "/etc/pki/tls/private"}
++)
++
++// hostBind is a host path to bind-mount at the same path in the private root.
++type hostBind struct {
++ path string
++ dev bool
++}
++
++// rootPlan lists what of the host every job sees bound: the system paths, the /etc files and the devices. It refuses
++// any of them that is, holds or lies in one of the Node's directories or hiddenPaths, as given or where its symlinks
++// lead. The CA certificates and a function's artifact directory, wherever it is, are not bound but copied (stageCA,
++// stageArtifact).
++func rootPlan(nodeDirs []string) (binds []hostBind, links []linkSpec, err error) {
++ for _, p := range systemPaths {
++ fi, err := os.Lstat(p)
++ switch {
++ case err != nil:
++ continue
++ case fi.Mode()&os.ModeSymlink != 0:
++ t, err := os.Readlink(p)
++ if err != nil {
++ return nil, nil, err
++ }
++ links = append(links, linkSpec{p, t})
++ case fi.IsDir():
++ binds = append(binds, hostBind{path: p})
++ }
++ }
++ for _, p := range etcPaths {
++ if _, err := os.Stat(p); err == nil {
++ binds = append(binds, hostBind{path: p})
++ }
++ }
++ for _, b := range binds {
++ if d, ok := overlaps(b.path, nodeDirs); ok {
++ return nil, nil, fmt.Errorf("%s would expose the Node's directory %s to jobs", b.path, d)
++ }
++ if d, ok := overlaps(b.path, hiddenPaths); ok {
++ return nil, nil, fmt.Errorf("%s would expose the host's %s to jobs", b.path, d)
++ }
++ }
++ for _, p := range devPaths {
++ binds = append(binds, hostBind{path: p, dev: true})
++ }
++ // Parents before what they hold.
++ sort.SliceStable(binds, func(i, j int) bool { return strings.Count(binds[i].path, "/") < strings.Count(binds[j].path, "/") })
++ return binds, append(links, devLinks...), nil
++}
++
++// An artifact directory may not be, hold or lie in hostOnlyPaths (kernel interfaces, the host's runtime sockets, its
++// configuration and secrets), nor be or hold sharedPaths (it may lie inside one, as a build directory in /tmp does).
++var (
++ hostOnlyPaths = []string{"/proc", "/sys", "/dev", "/run", "/var/run", "/var/lock", "/etc", "/boot", "/root"}
++ sharedPaths = []string{"/tmp", "/var/tmp", "/home"}
++)
++
++// checkArtifact refuses an artifact directory that would show the job host paths beyond the function's own files:
++// one in or over hostOnlyPaths, one over sharedPaths, one that is, holds or lies in one of the Node's directories.
++func checkArtifact(nodeDirs []string, artifact string) error {
++ if d, ok := overlaps(artifact, hostOnlyPaths); ok {
++ return fmt.Errorf("artifact directory %s would expose the host's %s to jobs", artifact, d)
++ }
++ for _, d := range sharedPaths {
++ for _, a := range resolved(artifact) {
++ for _, b := range resolved(d) {
++ if a == "/" || under(b, []string{a}) {
++ return fmt.Errorf("artifact directory %s would expose the host's %s to jobs", artifact, d)
++ }
++ }
++ }
++ }
++ if d, ok := overlaps(artifact, nodeDirs); ok {
++ return fmt.Errorf("artifact directory %s would expose the Node's directory %s to jobs", artifact, d)
++ }
++ return nil
++}
++
++// scanArtifact refuses, at boot, an artifact directory whose tree holds what its copy would not: a Unix socket, a
++// FIFO, a device. Every job's copy refuses them again (stageArtifact): this only says so before any job runs.
++func scanArtifact(artifact string) error {
++ root, err := filepath.EvalSymlinks(artifact)
++ if errors.Is(err, fs.ErrNotExist) {
++ return nil // nothing to copy; the job fails
++ } else if err != nil {
++ return err
++ }
++ return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
++ if err != nil {
++ return fmt.Errorf("artifact directory %s: %w", artifact, err)
++ }
++ if d.Type()&(fs.ModeSocket|fs.ModeNamedPipe|fs.ModeDevice|fs.ModeCharDevice) != 0 {
++ return fmt.Errorf("artifact directory %s holds %s, a socket, FIFO or device jobs could reach the host through",
++ artifact, p)
++ }
++ return nil
++ })
++}
++
++// checkRoute refuses an artifact directory whose path, followed as the job follows it in its private root (binds and
++// links as rootPlan makes them), crosses a directory of a bound host path that a host user other than one of trusted
++// could change: one owned by another user, writable by a group with a member not among trusted (trustedGroup), or
++// writable by others. In such a
++// directory a host user could, once the job runs, swap a symlink on the path or rename a directory on it (the copy's
++// mount goes with it) and leave the job's path to its artifact leading to a live host directory, a socket or FIFO
++// added there included. The rest of the path, outside the bound paths, is the private root's own.
++func checkRoute(binds []hostBind, links []linkSpec, artifact string, trusted []uint32) error {
++ var bound []string
++ for _, b := range binds {
++ bound = append(bound, b.path)
++ }
++ linked := map[string]string{}
++ for _, l := range links {
++ linked[l.Path] = l.Target
++ }
++ cur, rest, hops := "/", strings.Split(artifact, "/"), 0
++ for len(rest) > 0 {
++ c := rest[0]
++ rest = rest[1:]
++ switch c {
++ case "", ".":
++ continue
++ case "..":
++ cur = filepath.Dir(cur)
++ continue
++ }
++ next := filepath.Join(cur, c)
++ target, isLink := linked[next]
++ if under(cur, bound) {
++ var st syscall.Stat_t
++ if err := syscall.Stat(cur, &st); err != nil {
++ return fmt.Errorf("artifact directory %s: %s: %w", artifact, cur, err)
++ }
++ if !slices.Contains(trusted, st.Uid) || st.Mode&0o002 != 0 || st.Mode&0o020 != 0 && !trustedGroup(st.Gid, trusted) {
++ return fmt.Errorf("artifact directory %s: its path crosses %s, which a host user could change while "+
++ "a job runs (owner %d, group %d, mode %#o): its directories in a system path must be root's and "+
++ "writable by root only", artifact, cur, st.Uid, st.Gid, st.Mode&0o7777)
++ }
++ fi, err := os.Lstat(next)
++ if errors.Is(err, fs.ErrNotExist) {
++ // Nothing past cur exists: only its owner, trusted, could make it. The job fails without its
++ // artifact directory, as scanArtifact says; a job's own check runs on the path as it is then.
++ return nil
++ } else if err != nil {
++ return fmt.Errorf("artifact directory %s: %w", artifact, err)
++ }
++ if isLink = fi.Mode()&fs.ModeSymlink != 0; isLink {
++ if target, err = os.Readlink(next); err != nil {
++ return fmt.Errorf("artifact directory %s: %w", artifact, err)
++ }
++ }
++ }
++ if !isLink {
++ cur = next
++ continue
++ }
++ if hops++; hops > 40 {
++ return fmt.Errorf("artifact directory %s: %w", artifact, syscall.ELOOP)
++ }
++ if filepath.IsAbs(target) {
++ cur = "/"
++ }
++ rest = append(strings.Split(target, "/"), rest...)
++ }
++ return nil
++}
++
++// The account files trustedGroup reads.
++var passwdFile, groupFile = "/etc/passwd", "/etc/group"
++
++// trustedGroup reports whether every member of group gid is a trusted user: each user listed in it in groupFile and
++// each whose primary group it is in passwdFile. A group groupFile does not list (one from a directory service, whose
++// members the Node cannot see) or a member it cannot resolve is not trusted.
++func trustedGroup(gid uint32, trusted []uint32) bool {
++ g, err := os.ReadFile(groupFile)
++ if err != nil {
++ return false
++ }
++ p, err := os.ReadFile(passwdFile)
++ if err != nil {
++ return false
++ }
++ uids := map[string]uint32{}
++ for _, l := range strings.Split(string(p), "\n") {
++ f := strings.Split(l, ":") // name:password:uid:gid:...
++ if len(f) < 4 {
++ continue
++ }
++ uid, err1 := strconv.ParseUint(f[2], 10, 32)
++ pg, err2 := strconv.ParseUint(f[3], 10, 32)
++ if err1 != nil || err2 != nil {
++ continue
++ }
++ uids[f[0]] = uint32(uid)
++ if uint32(pg) == gid && !slices.Contains(trusted, uint32(uid)) {
++ return false
++ }
++ }
++ listed := false
++ for _, l := range strings.Split(string(g), "\n") {
++ f := strings.Split(l, ":") // name:password:gid:members
++ if len(f) < 4 || f[2] != strconv.FormatUint(uint64(gid), 10) {
++ continue
++ }
++ listed = true
++ for _, m := range strings.Split(f[3], ",") {
++ if m == "" {
++ continue
++ }
++ if uid, ok := uids[m]; !ok || !slices.Contains(trusted, uid) {
++ return false
++ }
++ }
++ }
++ return listed
++}
++
++// trustedUIDs are the host users who may change the path to an artifact directory (checkRoute): root and the Node's.
++func trustedUIDs() []uint32 { return []uint32{0, uint32(os.Geteuid())} }
++
++// openArtifact opens the artifact directory a job's sandbox init copies, and checks the directory it opened (where
++// its path leads now, whatever it led to at boot) as checkArtifact does, and its path as checkRoute does: the init
++// copies from this descriptor, not by path. nil: nothing to copy.
++func openArtifact(nodeDirs []string, binds []hostBind, links []linkSpec, artifact string) (*os.File, error) {
++ if artifact == "" {
++ return nil, nil
++ }
++ f, err := os.OpenFile(artifact, os.O_RDONLY|syscall.O_DIRECTORY, 0)
++ if err != nil {
++ return nil, fmt.Errorf("artifact directory: %w", err)
++ }
++ real, err := os.Readlink("/proc/self/fd/" + strconv.Itoa(int(f.Fd())))
++ if err == nil {
++ err = checkArtifact(nodeDirs, artifact)
++ }
++ if err == nil {
++ err = checkArtifact(nodeDirs, real)
++ }
++ if err == nil {
++ err = checkRoute(binds, links, artifact, trustedUIDs())
++ }
++ if err != nil {
++ f.Close()
++ return nil, err
++ }
++ return f, nil
++}
++
++// resolved is p cleaned and, if it differs, with its symlinks resolved.
++func resolved(p string) []string {
++ ps := []string{filepath.Clean(p)}
++ if r, err := filepath.EvalSymlinks(p); err == nil && r != ps[0] {
++ ps = append(ps, r)
++ }
++ return ps
++}
++
++// writableMounts lists the mounts of the process pid that are not read-only, except its /tmp, its /proc and the
++// device nodes: what a job could write to besides its own scratch space.
++func writableMounts(pid int) ([]string, error) {
++ b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/mountinfo")
++ if err != nil {
++ return nil, err
++ }
++ var rw []string
++ for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") {
++ f := strings.Fields(l) // f[4] is the mount point, relative to the process's root; f[5] its mount options
++ if len(f) < 6 {
++ return nil, fmt.Errorf("malformed mountinfo line %q", l)
++ }
++ if f[4] == "/tmp" || f[4] == "/proc" || slices.Contains(devPaths, f[4]) {
++ continue
++ }
++ if !slices.Contains(strings.Split(f[5], ","), "ro") {
++ rw = append(rw, f[4])
++ }
++ }
++ return rw, nil
++}
++
++// under reports whether p is one of dirs or lies inside one.
++func under(p string, dirs []string) bool {
++ for _, d := range dirs {
++ if p == d || strings.HasPrefix(p, d+"/") {
++ return true
++ }
++ }
++ return false
++}
++
++// overlaps returns the first of dirs that p is, holds or lies in, comparing both as given and with symlinks resolved.
++func overlaps(p string, dirs []string) (string, bool) {
++ ps := resolved(p)
++ for _, d := range dirs {
++ for _, a := range ps {
++ for _, b := range resolved(d) {
++ if a == "/" || b == "/" || under(a, []string{b}) || under(b, []string{a}) {
++ return d, true
++ }
++ }
++ }
++ }
++ return "", false
++}
ddiff --git a/internal/worker/seccomp_linux_amd64.go b/internal/worker/seccomp_linux_amd64.go
nnew file mode 100644
iindex 0000000..f6e1f88
---- /dev/null
++++ b/internal/worker/seccomp_linux_amd64.go
@@@ -0,0 +1,29 @@
++//go:build linux
++
++package worker
++
++// The seccomp allowlist on amd64, numbered as in the kernel's asm/unistd_64.h: the system calls a job may
++// make. clone, clone3 and socket are judged apart in seccompFilter.
++const (
++ auditArch = 0xc000003e // AUDIT_ARCH_X86_64
++ x32ABI = true
++ sysClone = 56
++ sysClone3 = 435
++ sysSocket = 41
++ sysMemfdCreate = 319 // the Node's own call, for the sandbox spec
++)
++
++var allowedSyscalls = []uint32{
++ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
++ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 60,
++ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
++ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,
++ 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 135, 137, 138, 140, 141,
++ 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 157, 158, 160, 162, 186, 187, 188, 189, 190, 191, 192, 193,
++ 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 206, 207, 208, 209, 210, 213, 216, 217, 218, 219, 220, 221,
++ 222, 223, 224, 225, 226, 228, 229, 230, 231, 232, 233, 234, 235, 240, 241, 242, 243, 244, 245, 247, 251, 252, 253,
++ 254, 255, 257, 258, 259, 260, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 273, 274, 275, 276, 277, 280, 281,
++ 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 299, 302, 306, 307, 309, 314, 315,
++ 316, 317, 318, 319, 322, 324, 325, 326, 327, 328, 332, 333, 334, 424, 434, 436, 437, 439, 441, 444, 445, 446, 449,
++ 452,
++}
ddiff --git a/internal/worker/seccomp_linux_arm64.go b/internal/worker/seccomp_linux_arm64.go
nnew file mode 100644
iindex 0000000..8f41864
---- /dev/null
++++ b/internal/worker/seccomp_linux_arm64.go
@@@ -0,0 +1,27 @@
++//go:build linux
++
++package worker
++
++// The seccomp allowlist on arm64, numbered as in the kernel's asm-generic/unistd.h: the system calls a job may
++// make. clone, clone3 and socket are judged apart in seccompFilter.
++const (
++ auditArch = 0xc00000b7 // AUDIT_ARCH_AARCH64
++ x32ABI = false
++ sysClone = 220
++ sysClone3 = 435
++ sysSocket = 198
++ sysMemfdCreate = 279 // the Node's own call, for the sandbox spec
++)
++
++var allowedSyscalls = []uint32{
++ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
++ 32, 33, 34, 35, 36, 37, 38, 43, 44, 45, 46, 47, 48, 49, 50, 52, 53, 54, 55, 56, 57, 59, 61, 62, 63, 64, 65, 66,
++ 67, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 90, 91, 92, 93, 94, 95, 96,
++ 98, 99, 100, 101, 102, 103, 107, 108, 109, 110, 111, 113, 114, 115, 118, 119, 120, 121, 122, 123, 124, 125, 126,
++ 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 143, 144, 145, 146, 147, 148, 149, 150,
++ 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 163, 164, 165, 166, 167, 168, 169, 172, 173, 174, 175, 176, 177,
++ 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 199, 200, 201,
++ 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 221, 222, 223, 226, 227, 228, 229, 230,
++ 231, 232, 233, 234, 240, 242, 243, 260, 261, 267, 269, 274, 275, 276, 277, 278, 279, 281, 283, 284, 285, 286, 287,
++ 291, 292, 293, 424, 434, 436, 437, 439, 441, 444, 445, 446, 449, 452,
++}
ddiff --git a/internal/worker/seccomp_linux_other.go b/internal/worker/seccomp_linux_other.go
nnew file mode 100644
iindex 0000000..414cc4c
---- /dev/null
++++ b/internal/worker/seccomp_linux_other.go
@@@ -0,0 +1,15 @@
++//go:build linux && !amd64 && !arm64
++
++package worker
++
++// No seccomp allowlist for this architecture: the probe fails, so the class is not advertised and no job runs.
++const (
++ auditArch = 0
++ x32ABI = false
++ sysClone = 0
++ sysClone3 = 0
++ sysSocket = 0
++ sysMemfdCreate = 0 // never called: policy refuses this architecture first
++)
++
++var allowedSyscalls []uint32
ddiff --git a/internal/worker/worker.go b/internal/worker/worker.go
iindex 8d05c4d..ee00149 100644
---- a/internal/worker/worker.go
++++ b/internal/worker/worker.go
@@@ -1,9 +1,11 @@
// Package worker runs `function` jobs (OpenVibe.Contracts platform.job@1) from the control link. It runs only the
--// functions the owner declared in the local config, each job as one child process in its own process group and in new
--// user, network and PID namespaces (no network at all), with a scrubbed environment and a fresh working directory, and
--// kills it at its ttl, at its limits, on job_cancel and while the stop latch is set. It never runs a job with less
--// isolation: a job whose namespaces cannot be created fails without running, and a Node whose boot-time Probe fails
--// does not advertise the class. docs/worker.md has the model and its limits.
++// functions the owner declared in the local config, each job as one child process in a sandbox: its own process group,
++// user, mount, network (no network at all), PID, IPC, UTS and cgroup namespaces, a private read-only root holding only
++// the system paths and the function's artifact, its own cgroup v2 (cpu, memory, pids, io), rlimits, no_new_privs and a
++// seccomp allowlist, with a scrubbed environment and a fresh /tmp. It kills a job at its ttl, at its limits, on
++// job_cancel and while the stop latch is set. It never runs a job with less isolation: a job whose sandbox cannot be
++// set up fails without running, and a Node whose boot-time Probe fails does not advertise the class. docs/worker.md
++// has the model and its limits.
package worker
import (
@@@ -14,6 +16,7 @@ import (
"log/slog"
"os"
"os/exec"
++ "path/filepath"
"sort"
"strings"
"sync"
@@@ -38,8 +41,8 @@ type Worker struct {
send func(protocol.Message)
log *slog.Logger
-- // isolate puts a job's command in its namespaces; tests replace it to stand for a kernel that cannot.
-- isolate func(*exec.Cmd, config.WorkerConfig) error
++ // isolate puts a job's command in its sandbox; tests replace it to stand for a kernel that cannot.
++ isolate isolateFunc
mu sync.Mutex
stopped string // the fault code while the stop latch is set; "" when clear
@@@ -49,6 +52,16 @@ type Worker struct {
wg sync.WaitGroup
}
++// isolateFunc turns a job's command into the start of its sandbox.
++type isolateFunc func(*exec.Cmd, config.WorkerConfig, plan) (*sandbox, error)
++
++// plan is what isolate needs to know of a job besides its command.
++type plan struct {
++ root string // the host directory (empty, the job's) its private root is mounted on
++ fn config.FunctionConfig // zero for the probe
++ limits protocol.JobLimits
++}
++
type job struct {
req protocol.Job
fn config.FunctionConfig
@@@ -72,8 +85,8 @@ func New(cfg config.WorkerConfig, send func(protocol.Message), log *slog.Logger)
return &Worker{cfg: cfg, caps: cfg.Caps.WithDefaults(), send: send, log: log, isolate: isolate, jobs: map[string]*job{}}
}
--// Probe starts a process the way a job is started and checks that it got user, network and PID namespaces of its own
--// and that its CPU limit can be set and its usage read. The class must not be advertised when Probe fails.
++// Probe starts a process the way a job is started and checks every control of the sandbox (docs/worker.md). The class
++// must not be advertised when Probe fails.
func (w *Worker) Probe() error { return probe(w.cfg, w.isolate) }
// Admit reserves a job, or returns the fault code and reason to nack it with. It runs nothing (Launch does, once the
@@@ -344,12 +357,6 @@ func (w *Worker) execute(jb *job) protocol.JobExit {
return never(protocol.ExitFailed, err)
}
}
-- cmd := exec.Command(jb.fn.Command[0], jb.fn.Command[1:]...)
-- cmd.Dir, cmd.Env = dir, jobEnv(jb.fn, dir, id)
-- cmd.Stdin = bytes.NewReader(append(append([]byte(nil), jb.req.Args...), '\n'))
-- if err := w.isolate(cmd, w.cfg); err != nil {
-- return never(protocol.ExitFailed, err)
-- }
outR, outW, err := os.Pipe()
if err != nil {
return never(protocol.ExitFailed, err)
@@@ -361,7 +368,20 @@ func (w *Worker) execute(jb *job) protocol.JobExit {
return never(protocol.ExitFailed, err)
}
defer resR.Close()
++ // dir is only the mount point of the job's private root; it works in a fresh directory of the same name in its
++ // own /tmp.
++ work := "/tmp/" + filepath.Base(dir)
++ cmd := exec.Command(jb.fn.Command[0], jb.fn.Command[1:]...)
++ cmd.Dir, cmd.Env = work, jobEnv(jb.fn, work, id)
++ cmd.Stdin = bytes.NewReader(append(append([]byte(nil), jb.req.Args...), '\n'))
cmd.Stdout, cmd.ExtraFiles = outW, []*os.File{resW}
++ sb, err := w.isolate(cmd, w.cfg, plan{root: dir, fn: jb.fn, limits: jb.limits})
++ if err != nil {
++ outW.Close()
++ resW.Close()
++ return never(protocol.ExitFailed, err)
++ }
++ defer sb.close()
w.mu.Lock()
if jb.reason == "" && time.Since(jb.received) >= jb.ttl {
@@@ -379,6 +399,7 @@ func (w *Worker) execute(jb *job) protocol.JobExit {
w.mu.Unlock()
outW.Close()
resW.Close()
++ sb.started()
if reason != "" {
return never(reason, nil)
}
@@@ -386,14 +407,27 @@ func (w *Worker) execute(jb *job) protocol.JobExit {
return never(protocol.ExitFailed, err)
}
pid := cmd.Process.Pid
-- // job_started only once RLIMIT_CPU is set; a job whose limit cannot be set is killed and ends failed without it.
-- if err := limitCPU(pid, jb.limits.CPUMS); err != nil {
++ // job_started only once the function runs confined and RLIMIT_CPU is set (both while the sandbox init waits to
++ // execute it, so it cannot have exited yet); otherwise the job is killed and ends failed (or as it was killed
++ // meanwhile) without it.
++ err = sb.ready(pid)
++ if err == nil {
++ err = limitCPU(pid, jb.limits.CPUMS)
++ }
++ if err == nil {
++ err = sb.run()
++ }
++ if err != nil {
w.mu.Lock()
killGroup(cmd.Process)
jb.proc = nil
++ reason = jb.reason
w.mu.Unlock()
_ = cmd.Wait()
-- return never(protocol.ExitFailed, fmt.Errorf("its CPU limit could not be set: %w", err))
++ if reason != "" {
++ return never(reason, nil)
++ }
++ return never(protocol.ExitFailed, fmt.Errorf("it could not run confined: %w", err))
}
w.mu.Lock()
jb.started = true
@@@ -420,7 +454,7 @@ func (w *Worker) execute(jb *job) protocol.JobExit {
code, cpuLimit := exitStatus(cmd.ProcessState)
if reason == "" {
reason = protocol.ExitExited
-- if cpuLimit {
++ if cpuLimit || sb.oomKilled() {
reason = protocol.ExitLimit
}
}
ddiff --git a/internal/worker/worker_linux.go b/internal/worker/worker_linux.go
iindex 7cdbee9..dbbf4eb 100644
---- a/internal/worker/worker_linux.go
++++ b/internal/worker/worker_linux.go
@@@ -3,16 +3,22 @@
package worker
import (
++ "encoding/json"
"errors"
"fmt"
++ "io"
++ "io/fs"
"os"
"os/exec"
++ "path/filepath"
"strconv"
"strings"
"syscall"
++ "time"
"unsafe"
"github.com/OpenVibers/OpenVibe.Node/internal/config"
++ "github.com/OpenVibers/OpenVibe.Node/internal/protocol"
)
// clkTck is USER_HZ, the unit of /proc/<pid>/stat times: 100 on every architecture Go runs Linux on.
@@@ -20,17 +26,131 @@ const clkTck = 100
var pageSize = int64(os.Getpagesize())
--// isolate starts cmd in its own process group and in new user, network and PID namespaces: no network but a loopback
--// that is down, and when the process exits every process it left behind is killed with it. It runs as worker.run_as
--// (with no supplementary groups; the Node must be root) or, with allow_same_user, as the Node's own non-root user.
--func isolate(cmd *exec.Cmd, cfg config.WorkerConfig) error {
++const capSysAdmin = 21
++
++// isolate turns cmd into the start of a sandboxed job (see sandbox_linux.go): the Node's own executable re-run as the
++// sandbox init, in its own process group, in new user, mount, network (a loopback that is down), PID, IPC, UTS and
++// cgroup namespaces, inside the job's cgroup v2 (made here), with PR_SET_PDEATHSIG SIGKILL. It runs as worker.run_as
++// (no supplementary groups; the Node must be root) or, with allow_same_user, as the Node's own non-root user. cmd's
++// path, args, env and dir become the function's, run once the sandbox stands; cmd.ExtraFiles holds at most fd 3. The
++// function's artifact directory is opened here and copied by the init (stageArtifact).
++func isolate(cmd *exec.Cmd, cfg config.WorkerConfig, p plan) (*sandbox, error) {
++ if err := policy(cfg); err != nil {
++ return nil, err
++ }
++ attr, err := jobAttr(cfg)
++ if err != nil {
++ return nil, err
++ }
++ if len(cmd.ExtraFiles) > 1 {
++ return nil, errors.New("isolate: a job has fd 3 only")
++ }
++ binds, links, err := rootPlan(cfg.NodeDirs)
++ if err != nil {
++ return nil, err
++ }
++ caps := cfg.Caps.WithDefaults()
++ sb := &sandbox{}
++ fail := func(err error) (*sandbox, error) { sb.close(); return nil, err }
++ art, err := openArtifact(cfg.NodeDirs, binds, links, p.fn.Artifact())
++ if err != nil {
++ return fail(err)
++ }
++ if art != nil {
++ sb.child = append(sb.child, art)
++ }
++ files := make([]*os.File, 1, 4)
++ copy(files, cmd.ExtraFiles)
++ sr, sw, err := os.Pipe()
++ if err != nil {
++ return fail(err)
++ }
++ sb.status, sb.child = sr, append(sb.child, sw)
++ gr, gw, err := os.Pipe()
++ if err != nil {
++ return fail(err)
++ }
++ sb.goW, sb.child = gw, append(sb.child, gr)
++ files = append(files, sw, gr) // statusFD, goFD
++ spec := sandboxSpec{Root: p.root, Work: cmd.Dir, Disk: caps.MaxDiskBytes, Links: links,
++ Argv: append([]string{cmd.Path}, cmd.Args[1:]...), Env: cmd.Env}
++ if art != nil {
++ spec.Artifact = p.fn.Artifact()
++ }
++ for _, b := range binds {
++ spec.Binds = append(spec.Binds, bindSpec{Path: b.path, Dev: b.dev})
++ }
++ sec := uint64((p.limits.CPUMS + 999) / 1000)
++ pids, disk := uint64(caps.MaxPids), uint64(caps.MaxDiskBytes)
++ spec.Limits = []rlimit{{rlimitCPU, sec, sec + 1}, {rlimitCORE, 0, 0}, {rlimitNOFILE, 1024, 1024},
++ {rlimitNPROC, pids, pids}, {rlimitFSIZE, disk, disk}}
++ if caps.MaxVMBytes <= 0 {
++ return fail(errors.New("isolate: worker.caps.max_vm_bytes must be set"))
++ }
++ spec.Limits = append(spec.Limits, rlimit{rlimitAS, uint64(caps.MaxVMBytes), uint64(caps.MaxVMBytes)})
++ sf, err := specFile(spec)
++ if err != nil {
++ return fail(err)
++ }
++ sb.child = append(sb.child, sf)
++ files = append(files, sf) // specFD
++ if art != nil {
++ files = append(files, art) // artifactFD
++ }
++ if sb.cg, err = newJobCgroup(caps, p.limits); err != nil {
++ return fail(err)
++ }
++ attr.UseCgroupFD, attr.CgroupFD = true, sb.cg.fd
++ cmd.Path, cmd.Args, cmd.Env, cmd.Dir = "/proc/self/exe", []string{sandboxArg0}, []string{}, "/"
++ cmd.ExtraFiles, cmd.SysProcAttr = files, attr
++ return sb, nil
++}
++
++// specFile is spec in a memfd, for the sandbox init to read on specFD. Its environment may hold the function's
++// secrets: unlike argv (/proc/<pid>/cmdline), a descriptor is open only to who may ptrace the process.
++func specFile(spec sandboxSpec) (*os.File, error) {
++ b, err := json.Marshal(spec)
++ if err != nil {
++ return nil, err
++ }
++ const mfdCloexec = 1
++ name := []byte("openvibe-sandbox-spec\x00")
++ fd, _, e := syscall.Syscall(sysMemfdCreate, uintptr(unsafe.Pointer(&name[0])), mfdCloexec, 0)
++ if e != 0 {
++ return nil, fmt.Errorf("memfd_create: %w", e)
++ }
++ f := os.NewFile(fd, "sandbox spec")
++ if _, err := f.Write(b); err != nil {
++ f.Close()
++ return nil, err
++ }
++ if _, err := f.Seek(0, io.SeekStart); err != nil {
++ f.Close()
++ return nil, err
++ }
++ return f, nil
++}
++
++// policy refuses what the sandbox cannot enforce: an egress other than none, an architecture without a seccomp
++// allowlist.
++func policy(cfg config.WorkerConfig) error {
++ if e := cfg.Egress; e != "" && e != config.EgressNone {
++ return fmt.Errorf("worker.egress %s is not enforced yet (only none is)", e)
++ }
++ _, err := seccompFilter()
++ return err
++}
++
++// jobAttr is the namespaces and credentials of a job.
++func jobAttr(cfg config.WorkerConfig) (*syscall.SysProcAttr, error) {
uid, gid := os.Geteuid(), os.Getegid()
-- attr := &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL,
-- Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET | syscall.CLONE_NEWPID}
++ attr := &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL, AmbientCaps: []uintptr{capSysAdmin},
++ Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNS | syscall.CLONE_NEWNET | syscall.CLONE_NEWPID |
++ syscall.CLONE_NEWIPC | syscall.CLONE_NEWUTS | syscall.CLONE_NEWCGROUP}
switch {
case cfg.RunAs != nil:
if uid != 0 {
-- return errors.New("worker.run_as needs the Node to run as root")
++ return nil, errors.New("worker.run_as needs the Node to run as root")
}
r := cfg.RunAs
attr.UidMappings = []syscall.SysProcIDMap{{ContainerID: int(r.UID), HostID: int(r.UID), Size: 1}}
@@@ -39,42 +159,191 @@ func isolate(cmd *exec.Cmd, cfg config.WorkerConfig) error {
attr.Credential = &syscall.Credential{Uid: r.UID, Gid: r.GID} // no Groups: setgroups(0) drops root's groups
case cfg.AllowSameUser:
if uid == 0 {
-- return errors.New("the Node runs as root: set worker.run_as instead of allow_same_user")
++ return nil, errors.New("the Node runs as root: set worker.run_as instead of allow_same_user")
}
attr.UidMappings = []syscall.SysProcIDMap{{ContainerID: uid, HostID: uid, Size: 1}}
attr.GidMappings = []syscall.SysProcIDMap{{ContainerID: gid, HostID: gid, Size: 1}}
default:
-- return errors.New("set worker.run_as (or allow_same_user, for development) to run jobs")
++ return nil, errors.New("set worker.run_as (or allow_same_user, for development) to run jobs")
++ }
++ return attr, nil
++}
++
++// sandbox is a job's sandbox as the Node holds it: the status pipe of its init, the pipe that lets it execute the
++// function, the descriptors the child inherits (closed once it started) and its cgroup.
++type sandbox struct {
++ cg *jobCgroup
++ status *os.File
++ goW *os.File
++ child []*os.File
++}
++
++// started closes the Node's copies of what the child inherited.
++func (s *sandbox) started() {
++ for _, f := range s.child {
++ f.Close()
++ }
++ s.child = nil
++}
++
++// ready waits until the sandbox init stands confined, about to execute the function, and checks from the host that
++// it runs with no_new_privs and the seccomp filter, that nothing but its /tmp, /proc and device nodes is mounted
++// writable and that every block device has the job's io.max. The init waits for run meanwhile, so it cannot have
++// exited. An error means the function must not run.
++func (s *sandbox) ready(pid int) error {
++ _ = s.status.SetReadDeadline(time.Now().Add(10 * time.Second))
++ var b [1]byte
++ if _, err := io.ReadFull(s.status, b[:]); err != nil {
++ return fmt.Errorf("the sandbox did not report: %w", err)
++ }
++ if b[0] != 0 {
++ msg, _ := io.ReadAll(s.status)
++ return fmt.Errorf("the sandbox could not be set up: %s%s", b[:], msg)
++ }
++ // The spec, the function's environment included, must not show to every host user in the init's argv.
++ if c, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/cmdline"); err != nil || string(c) != sandboxArg0+"\x00" {
++ return fmt.Errorf("the sandbox init's command line is not its name alone (%q, %v)", c, err)
++ }
++ st, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/status")
++ if err != nil {
++ return err
++ }
++ if st := string(st); !strings.Contains(st, "\nNoNewPrivs:\t1\n") || !strings.Contains(st, "\nSeccomp:\t2\n") {
++ return errors.New("the job is not confined: no no_new_privs or seccomp filter")
++ }
++ rw, err := writableMounts(pid)
++ if err != nil {
++ return err
++ }
++ if len(rw) > 0 {
++ return fmt.Errorf("the job's root has writable mounts: %s", strings.Join(rw, ", "))
++ }
++ // Again now that the job's root stands: a disk attached since its cgroup was made could back what it mounted.
++ return s.cg.checkIO()
++}
++
++// run lets the init that ready checked execute the function and waits until it did (its status pipe closed).
++func (s *sandbox) run() error {
++ _, err := s.goW.Write([]byte{1})
++ s.goW.Close()
++ if err != nil {
++ return fmt.Errorf("starting the function: %w", err)
++ }
++ _ = s.status.SetReadDeadline(time.Now().Add(10 * time.Second))
++ msg, err := io.ReadAll(s.status)
++ switch {
++ case len(msg) > 0:
++ return fmt.Errorf("the function could not start: %s", msg)
++ case err != nil:
++ return fmt.Errorf("the sandbox did not report: %w", err)
}
-- cmd.SysProcAttr = attr
return nil
}
--// probe starts /bin/sh the way a job is started and checks that it got user, network and PID namespaces of its own,
--// that its CPU limit can be set and that its usage can be read.
--func probe(cfg config.WorkerConfig, isolate func(*exec.Cmd, config.WorkerConfig) error) error {
++// oomKilled reports whether memory.max ended the job.
++func (s *sandbox) oomKilled() bool { return s.cg != nil && s.cg.oomKilled() }
++
++// close releases the sandbox once its process ended: it kills what is left in its cgroup and removes it.
++func (s *sandbox) close() {
++ s.started()
++ for _, f := range []*os.File{s.status, s.goW} {
++ if f != nil {
++ f.Close()
++ }
++ }
++ if s.cg != nil {
++ s.cg.remove()
++ }
++}
++
++// probe starts /bin/sh the way a job is started and checks every control from the host: the sandbox was set up;
++// the process has user, mount, network, PID, IPC, UTS and cgroup namespaces of its own, no_new_privs and the seccomp
++// filter; nothing but its /tmp, /proc and device nodes is mounted writable; it is in its job cgroup; its rlimits
++// (RLIMIT_AS included: max_vm_bytes must be set) are set; every block device has its io.max; its root holds none of
++// the Node's directories nor hiddenPaths (/var, the host's private keys); its CPU limit can be set and its usage read.
++// It also checks that no declared function's artifact directory is refused. Any failure keeps the class off: there is
++// no weaker mode.
++func probe(cfg config.WorkerConfig, isolate isolateFunc) error {
++ if err := policy(cfg); err != nil {
++ return err
++ }
++ binds, links, err := rootPlan(cfg.NodeDirs)
++ if err != nil {
++ return err
++ }
++ for _, f := range cfg.Functions {
++ err := checkArtifact(cfg.NodeDirs, f.Artifact())
++ if err == nil && f.Artifact() != "" {
++ err = checkRoute(binds, links, f.Artifact(), trustedUIDs())
++ }
++ if err == nil && f.Artifact() != "" {
++ err = scanArtifact(f.Artifact())
++ }
++ if err != nil {
++ return fmt.Errorf("function %s@%s: %w", f.Name, f.Version, err)
++ }
++ }
++ root, err := os.MkdirTemp("", "openvibe-probe-")
++ if err != nil {
++ return err
++ }
++ defer os.RemoveAll(root)
++ if r := cfg.RunAs; r != nil {
++ if err := os.Chown(root, int(r.UID), int(r.GID)); err != nil {
++ return err
++ }
++ }
++ caps := cfg.Caps.WithDefaults()
cmd := exec.Command("/bin/sh", "-c", "read x")
-- cmd.Env, cmd.Dir = []string{"PATH=/usr/bin:/bin"}, "/"
-- if err := isolate(cmd, cfg); err != nil {
++ cmd.Env, cmd.Dir = []string{"PATH=/usr/bin:/bin"}, "/tmp"
++ sb, err := isolate(cmd, cfg, plan{root: root, limits: protocol.JobLimits{CPUMS: 1000, MemBytes: caps.MaxMemBytes}})
++ if err != nil {
return err
}
++ defer sb.close()
in, err := cmd.StdinPipe()
if err != nil {
return err
}
-- if err := cmd.Start(); err != nil {
++ err = cmd.Start()
++ sb.started()
++ if err != nil {
in.Close()
-- return fmt.Errorf("starting a process in new user, network and PID namespaces: %w", err)
++ return fmt.Errorf("starting a sandboxed process: %w", err)
}
defer func() { killGroup(cmd.Process); in.Close(); _ = cmd.Wait() }()
pid := cmd.Process.Pid
-- for _, ns := range []string{"user", "net", "pid"} {
++ if err := sb.ready(pid); err != nil {
++ return err
++ }
++ if err := sb.run(); err != nil {
++ return err
++ }
++ proc := "/proc/" + strconv.Itoa(pid)
++ for _, ns := range []string{"user", "mnt", "net", "pid", "ipc", "uts", "cgroup"} {
mine, err1 := os.Readlink("/proc/self/ns/" + ns)
-- its, err2 := os.Readlink("/proc/" + strconv.Itoa(pid) + "/ns/" + ns)
++ its, err2 := os.Readlink(proc + "/ns/" + ns)
if err1 != nil || err2 != nil || mine == its {
return fmt.Errorf("the probe process did not get a %s namespace of its own (%v, %v)", ns, err1, err2)
}
}
++ b, err := os.ReadFile(proc + "/cgroup")
++ if want := "0::" + strings.TrimPrefix(sb.cg.dir, cgroupFS); err != nil || strings.TrimSpace(string(b)) != want {
++ return fmt.Errorf("the probe process is not in its cgroup %s (%q, %v)", want, b, err)
++ }
++ pids, disk := uint64(caps.MaxPids), uint64(caps.MaxDiskBytes)
++ want := map[int]uint64{rlimitNOFILE: 1024, rlimitNPROC: pids, rlimitFSIZE: disk, rlimitAS: uint64(caps.MaxVMBytes)}
++ for res, v := range want {
++ var got syscall.Rlimit
++ if _, _, e := syscall.RawSyscall6(syscall.SYS_PRLIMIT64, uintptr(pid), uintptr(res), 0, uintptr(unsafe.Pointer(&got)), 0, 0); e != 0 || got.Cur != v || got.Max != v {
++ return fmt.Errorf("the probe process's rlimit %d is %d/%d, not %d (%v)", res, got.Cur, got.Max, v, e)
++ }
++ }
++ for _, d := range append(append([]string(nil), hiddenPaths...), cfg.NodeDirs...) {
++ if _, err := os.Lstat(proc + "/root" + filepath.Clean(d)); !errors.Is(err, fs.ErrNotExist) {
++ return fmt.Errorf("the probe process can see %s (%v)", d, err)
++ }
++ }
if err := limitCPU(pid, 1000); err != nil {
return err
}
ddiff --git a/internal/worker/worker_other.go b/internal/worker/worker_other.go
iindex 30ec593..16dec07 100644
---- a/internal/worker/worker_other.go
++++ b/internal/worker/worker_other.go
@@@ -13,9 +13,18 @@ import (
// errNotLinux: the namespaces the worker isolates jobs with exist on Linux only, so elsewhere no job ever runs.
var errNotLinux = errors.New("the job worker runs on Linux only")
--func isolate(*exec.Cmd, config.WorkerConfig) error { return errNotLinux }
++func isolate(*exec.Cmd, config.WorkerConfig, plan) (*sandbox, error) { return nil, errNotLinux }
--func probe(config.WorkerConfig, func(*exec.Cmd, config.WorkerConfig) error) error { return errNotLinux }
++func probe(config.WorkerConfig, isolateFunc) error { return errNotLinux }
++
++// sandbox is never made off Linux.
++type sandbox struct{}
++
++func (*sandbox) started() {}
++func (*sandbox) ready(int) error { return errNotLinux }
++func (*sandbox) run() error { return errNotLinux }
++func (*sandbox) oomKilled() bool { return false }
++func (*sandbox) close() {}
func killGroup(p *os.Process) { _ = p.Kill() }
ddiff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go
iindex 4be1818..915b788 100644
---- a/internal/worker/worker_test.go
++++ b/internal/worker/worker_test.go
@@@ -8,7 +8,6 @@ import (
"log/slog"
"os"
"os/exec"
-- "runtime"
"slices"
"sync"
"testing"
@@@ -99,10 +98,21 @@ func helperConfig(t *testing.T, caps config.WorkerCaps, modes ...string) config.
if err != nil {
t.Fatal(err)
}
++ // A race-instrumented binary maps terabytes of shadow memory up front: RLIMIT_AS 64 TiB leaves it room and is
++ // still a limit. GOMAXPROCS keeps its threads well under max_pids on a machine with many cores.
++ if caps.MaxVMBytes == 0 {
++ caps.MaxVMBytes = 1 << 46
++ }
++ if caps.MaxDiskBytes == 0 { // each job gets a copy of its artifact directory, this test binary, in that much
++ caps.MaxDiskBytes = 256 << 20
++ }
cfg := config.WorkerConfig{Enabled: true, AllowSameUser: true, Caps: caps}
++ if os.Geteuid() == 0 { // as root, jobs run as nobody: the test binary must then lie in a directory nobody can read
++ cfg.AllowSameUser, cfg.RunAs = false, &config.RunAs{UID: 65534, GID: 65534}
++ }
for _, m := range modes {
cfg.Functions = append(cfg.Functions, config.FunctionConfig{Name: m, Version: "1.0.0",
-- Command: []string{exe, "-test.run=^TestHelperProcess$", "--", m}})
++ Command: []string{exe, "-test.run=^TestHelperProcess$", "--", m}, Env: map[string]string{"GOMAXPROCS": "2"}})
}
return cfg
}
@@@ -114,7 +124,9 @@ func quiet() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)
func TestAdmitRefusals(t *testing.T) {
s := &sink{}
w := New(helperConfig(t, config.WorkerCaps{MaxJobs: 1}, "sleep"), s.send, quiet())
-- w.isolate = func(*exec.Cmd, config.WorkerConfig) error { return errors.New("not in this test") }
++ w.isolate = func(*exec.Cmd, config.WorkerConfig, plan) (*sandbox, error) {
++ return nil, errors.New("not in this test")
++ }
wrongVersion := testJob(2, "sleep")
wrongVersion.Artifact.Version = "1.0.1"
@@@ -169,7 +181,9 @@ func TestAdmitRefusals(t *testing.T) {
func TestIsolationUnavailableRefused(t *testing.T) {
s := &sink{}
w := New(helperConfig(t, config.WorkerCaps{}, "sleep"), s.send, quiet())
-- w.isolate = func(*exec.Cmd, config.WorkerConfig) error { return errors.New("no user namespaces") }
++ w.isolate = func(*exec.Cmd, config.WorkerConfig, plan) (*sandbox, error) {
++ return nil, errors.New("no user namespaces")
++ }
t.Cleanup(w.Close)
if err := w.Probe(); err == nil {
t.Fatal("the probe passed without namespaces")
@@@ -193,7 +207,9 @@ func TestIsolationUnavailableRefused(t *testing.T) {
func TestCloseEndsUnlaunched(t *testing.T) {
s := &sink{}
w := New(helperConfig(t, config.WorkerCaps{}, "sleep"), s.send, quiet())
-- w.isolate = func(*exec.Cmd, config.WorkerConfig) error { return errors.New("not in this test") }
++ w.isolate = func(*exec.Cmd, config.WorkerConfig, plan) (*sandbox, error) {
++ return nil, errors.New("not in this test")
++ }
j := testJob(1, "sleep")
if f, r := w.Admit(j); f != "" {
t.Fatalf("%s %s", f, r)
@@@ -236,64 +252,3 @@ func TestUsageHelperProcess(t *testing.T) {
time.Sleep(time.Duration(args.HoldMS) * time.Millisecond)
os.Exit(0)
}
--
--// TestUsageSecondsContiguous: a function that holds for several whole wall-clock seconds after job_started sends
--// exactly one job_usage for each fully elapsed second, `second` 0 then 1 then ... with no gap and no duplicate, and
--// its job_exit's usage is never smaller than the seconds already sent (wall_ms >= (n+1)*1000 for every second n).
--func TestUsageSecondsContiguous(t *testing.T) {
-- if runtime.GOOS != "linux" {
-- t.Skip("function jobs run on Linux only")
-- }
-- exe, err := os.Executable()
-- if err != nil {
-- t.Fatal(err)
-- }
-- // Isolation is not what this test measures: run the helper directly, without the job namespaces.
-- s := &sink{}
-- w := New(config.WorkerConfig{Enabled: true, AllowSameUser: true,
-- Functions: []config.FunctionConfig{{Name: "meter", Version: "1.0.0",
-- Command: []string{exe, "-test.run=^TestUsageHelperProcess$", "--", "meter"}}}}, s.send, quiet())
-- w.isolate = func(*exec.Cmd, config.WorkerConfig) error { return nil }
-- t.Cleanup(w.Close)
--
-- const holdMS = 2500 // two whole seconds, so seconds 0 and 1 are sent before the exit
-- j := testJob(1, "meter")
-- j.Args = json.RawMessage(fmt.Sprintf(`{"hold_ms":%d}`, holdMS))
-- if f, r := w.Admit(j); f != "" {
-- t.Fatalf("%s %s", f, r)
-- }
-- w.Launch(j.ID)
-- ex := s.exit(t, j.ID)
-- if ex.Reason != protocol.ExitExited || ex.Code == nil || *ex.Code != 0 || ex.Usage.StartedMS == nil {
-- t.Fatalf("%+v", ex)
-- }
-- if n := s.count(protocol.TypeJobStarted, j.ID); n != 1 {
-- t.Fatalf("job_started sent %d times, want 1", n)
-- }
-- started := s.wait(t, protocol.TypeJobStarted, j.ID, 1, 5*time.Second).(protocol.JobStarted)
-- if started.StartedMS != *ex.Usage.StartedMS {
-- t.Fatalf("job_started at %d, job_exit usage at %d", started.StartedMS, *ex.Usage.StartedMS)
-- }
-- var seconds []int64
-- for _, m := range s.all() {
-- u, ok := m.(protocol.JobUsage)
-- if !ok || u.ID != j.ID {
-- continue
-- }
-- if u.StartedMS != *ex.Usage.StartedMS || u.CPUMS == nil {
-- t.Fatalf("usage %+v", u)
-- }
-- seconds = append(seconds, u.Second)
-- }
-- if len(seconds) < 2 {
-- t.Fatalf("usage seconds %v, want 0..n-1 for a job that held %d ms", seconds, holdMS)
-- }
-- for n, second := range seconds {
-- if second != int64(n) {
-- t.Fatalf("usage seconds %v, want 0..%d with no gap or duplicate", seconds, len(seconds)-1)
-- }
-- if ex.Usage.WallMS < (second+1)*1000 {
-- t.Fatalf("usage second %d sent but wall_ms is %d, want at least %d", second, ex.Usage.WallMS, (second+1)*1000)
-- }
-- }
--}