openvibers/openvibe.link Commits

Node: worker.dns, so a job with a network can resolve names (install --worker sets public resolvers) (#39)

OP OpenVibers committed 30 days ago · c2d6fc8cb1 · 9 files, +139 −4

Found by the first end-to-end run of a coding task on a person's own machine (openvibe.codes/run → Run → Bot → this Node, on production). Everything reached the job, but its `git clone` failed with `Could not resolve host: github.com`. A job never gets the host's resolver (127.0.0.53 is out of its network namespace), and nothing told it any other.

**worker.dns**
- Up to three IPv4 resolvers, empty by default.
- They become the job's own `/etc/resolv.conf`, a file of its private root that is read-only like the rest. This happens only for a job that runs with a network (public or openvibe-only); a job without one gets none.
- Config validation accepts only addresses such a job could reach anyway:
  - public ones (not private, shared, loopback, link-local or IPv6);
  - under openvibe-only, ones inside `egress_allow`;
  - and only with a policy that gives jobs a network.
- `install.sh --worker` sets `["1.1.1.1", "9.9.9.9"]` unless the config already names others.

**Also**
- `pair` on a worker binding printed "Bound as dev_… (robot )". It now says it is a machine for your jobs.

**Tests**
- `TestWorkerDNS` (config).
- `TestEgressDNS`: a root sandbox test that reads the job's resolv.conf with and without a network.
- The installer's worker-config test checks the default.
- shellcheck.

Changes

ddiff --git a/cmd/openvibe-node/main.go b/cmd/openvibe-node/main.go
iindex ccdd9fd..a9089f0 100644
---- a/cmd/openvibe-node/main.go
++++ b/cmd/openvibe-node/main.go
@@@ -380,7 +380,11 @@ func cmdPairNetwork(g *globals, a pairArgs, stdout io.Writer) error {
if err := saveCredentials(g, bound); err != nil {
return err
}
-- fmt.Fprintf(stdout, "Bound as %s (robot %s).\n", bound.DeviceID, bound.RobotID)
++ if bound.RobotID != "" {
++ fmt.Fprintf(stdout, "Bound as %s (robot %s).\n", bound.DeviceID, bound.RobotID)
++ } else {
++ fmt.Fprintf(stdout, "Bound as %s (a machine for your jobs, no robot).\n", bound.DeviceID)
++ }
}
fmt.Fprintln(stdout, "Start it: `openvibe-node run` or `sudo openvibe-node install`.")
return finishPair(g, a, cfg, noConfig, stdout)
ddiff --git a/docs/worker.md b/docs/worker.md
iindex bd5f92e..dc30591 100644
---- a/docs/worker.md
++++ b/docs/worker.md
@@@ -45,6 +45,7 @@ summary, the changed files and the patch; it never pushes to the branch it start
"run_as": {"uid": 2001, "gid": 2001},
"allow_same_user": false,
"egress": "none",
++ "dns": [],
"media": {
"enabled": false,
"endpoint": "",
@@@ -306,9 +307,11 @@ How `public` and `openvibe-only` are enforced (`internal/worker/egress_linux.go`
`/usr/sbin`, `/sbin`, `/usr/bin` or `/bin`, IPv4 forwarding on (`sysctl net.ipv4.ip_forward=1`: the Node does not
turn it on, which would make the host a router for every interface; the owner does) and an IPv4 default route.
Without one of them the probe fails and so does every job.
--- **No DNS.** The Node runs no resolver for jobs, and the host's (`127.0.0.53` on most systems) is out of a job's
-- reach. A job resolves names only through a resolver its policy lets it reach by address (any public one under
-- `public`; one listed in `egress_allow` under `openvibe-only`), or connects by address.
++- **DNS.** The Node runs no resolver for jobs, and the host's (`127.0.0.53` on most systems) is out of a job's
++ reach. `worker.dns` (up to three IPv4 addresses, empty by default) becomes the `/etc/resolv.conf` of a job that runs
++ with a network: public addresses only, and under `openvibe-only` only inside `egress_allow`, so it names nothing a
++ job could not reach anyway. Without it a job resolves nothing and connects by address. `install.sh --worker` sets
++ `["1.1.1.1", "9.9.9.9"]` (Cloudflare and Quad9) unless the config already names others.
- **`openvibe-only` is configured, not guessed.** No authoritative list of the OpenVibe network's ranges exists in
this repository or OpenVibe.Contracts, so `egress_allow` must name them; without it the config does not load.
ddiff --git a/install/install.sh b/install/install.sh
iindex cbe01fe..d5f8df7 100755
---- a/install/install.sh
++++ b/install/install.sh
@@@ -153,6 +153,9 @@ for k, v in {
}.items():
caps.setdefault(k, v)
worker['caps'] = caps
++# A job resolves names only through resolvers it can reach by address (docs/worker.md, "DNS"): two public ones, kept
++# when the owner already chose others.
++worker.setdefault('dns', ['1.1.1.1', '9.9.9.9'])
functions = [f for f in worker.get('functions') or [] if f.get('name') != 'openvibe-codes']
if node and codes:
previous = next((f for f in worker.get('functions') or [] if f.get('name') == 'openvibe-codes'), {})
ddiff --git a/internal/config/config.go b/internal/config/config.go
iindex 02aa455..466868a 100644
---- a/internal/config/config.go
++++ b/internal/config/config.go
@@@ -21,6 +21,7 @@ import (
"path/filepath"
"regexp"
"runtime"
++ "slices"
"strings"
)
@@@ -139,6 +140,10 @@ type WorkerConfig struct {
// EgressDeny is more IPv4 CIDRs a public or openvibe-only job is refused (the Node's cell, WireGuard peers with
// public addresses), on top of the ranges always refused.
EgressDeny []string `json:"egress_deny,omitempty"`
++ // DNS is up to three IPv4 resolvers written to the /etc/resolv.conf of a job that runs with a network (public or
++ // openvibe-only); none by default, when a job resolves no names (the host's resolver is out of its reach). Only
++ // addresses such a job could reach anyway: public ones, and under openvibe-only ones inside EgressAllow.
++ DNS []string `json:"dns,omitempty"`
// Media is where a job's inputs come from and a result over 256 KiB is uploaded to. Off (the default), a job
// naming inputs is refused and a result over 256 KiB ends the job limit.
Media MediaConfig `json:"media,omitzero"`
@@@ -470,6 +475,9 @@ func (w WorkerConfig) validate() error {
}
}
}
++ if err := w.validateDNS(); err != nil {
++ return err
++ }
switch {
case w.Egress == EgressOpenVibeOnly && len(w.EgressAllow) == 0:
return errors.New("config: worker.egress openvibe-only needs worker.egress_allow, the CIDRs a job may reach")
@@@ -512,3 +520,30 @@ func (c *Config) PythonPath(paths Paths) string {
}
return "python3"
}
++
++// validateDNS checks worker.dns: at most three IPv4 resolvers (the most a resolv.conf is read for), only with a policy
++// that gives jobs a network, each public, and under openvibe-only inside egress_allow.
++func (w WorkerConfig) validateDNS() error {
++ if len(w.DNS) == 0 {
++ return nil
++ }
++ if w.Egress != EgressPublic && w.Egress != EgressOpenVibeOnly {
++ return errors.New("config: worker.dns needs worker.egress public or openvibe-only (a job without a network resolves nothing)")
++ }
++ if len(w.DNS) > 3 {
++ return errors.New("config: worker.dns lists at most three resolvers")
++ }
++ for _, d := range w.DNS {
++ a, err := netip.ParseAddr(d)
++ if err != nil || !a.Is4() || !a.IsGlobalUnicast() || a.IsPrivate() || netip.MustParsePrefix("100.64.0.0/10").Contains(a) {
++ return fmt.Errorf("config: worker.dns %q is not a public IPv4 address such as 1.1.1.1", d)
++ }
++ if w.Egress == EgressOpenVibeOnly && !slices.ContainsFunc(w.EgressAllow, func(c string) bool {
++ p, err := netip.ParsePrefix(c)
++ return err == nil && p.Contains(a)
++ }) {
++ return fmt.Errorf("config: worker.dns %s is outside worker.egress_allow, so an openvibe-only job could not reach it", d)
++ }
++ }
++ return nil
++}
ddiff --git a/internal/config/config_test.go b/internal/config/config_test.go
iindex faf7278..aa02c1d 100644
---- a/internal/config/config_test.go
++++ b/internal/config/config_test.go
@@@ -192,3 +192,30 @@ func TestWorkerMedia(t *testing.T) {
t.Fatal("a token written in the config file loaded")
}
}
++
++// TestWorkerDNS: worker.dns takes up to three public IPv4 resolvers, only with a policy that gives jobs a network, and
++// under openvibe-only only inside egress_allow.
++func TestWorkerDNS(t *testing.T) {
++ ok := []WorkerConfig{{Egress: EgressPublic, DNS: []string{"1.1.1.1", "9.9.9.9"}},
++ {Egress: EgressOpenVibeOnly, EgressAllow: []string{"9.9.9.0/24"}, DNS: []string{"9.9.9.9"}}}
++ for _, w := range ok {
++ c := Default()
++ c.Worker = w
++ if err := c.Validate(); err != nil {
++ t.Fatalf("%+v: %v", w, err)
++ }
++ }
++ bad := []WorkerConfig{{DNS: []string{"1.1.1.1"}}, {Egress: EgressNone, DNS: []string{"1.1.1.1"}},
++ {Egress: EgressPublic, DNS: []string{"1.1.1.1", "1.0.0.1", "9.9.9.9", "8.8.8.8"}},
++ {Egress: EgressPublic, DNS: []string{"127.0.0.53"}}, {Egress: EgressPublic, DNS: []string{"192.168.1.1"}},
++ {Egress: EgressPublic, DNS: []string{"100.64.0.1"}}, {Egress: EgressPublic, DNS: []string{"169.254.169.253"}},
++ {Egress: EgressPublic, DNS: []string{"2606:4700:4700::1111"}}, {Egress: EgressPublic, DNS: []string{"dns.google"}},
++ {Egress: EgressOpenVibeOnly, EgressAllow: []string{"9.9.9.0/24"}, DNS: []string{"1.1.1.1"}}}
++ for _, w := range bad {
++ c := Default()
++ c.Worker = w
++ if err := c.Validate(); err == nil {
++ t.Fatalf("%+v was accepted", w)
++ }
++ }
++}
ddiff --git a/internal/worker/run_linux_test.go b/internal/worker/run_linux_test.go
iindex 698ef2a..e9bbbf9 100644
---- a/internal/worker/run_linux_test.go
++++ b/internal/worker/run_linux_test.go
@@@ -100,6 +100,10 @@ func TestHelperProcess(t *testing.T) {
}()
}
wg.Wait()
++ got["resolv.conf"] = "absent"
++ if b, err := os.ReadFile("/etc/resolv.conf"); err == nil {
++ got["resolv.conf"] = string(b)
++ }
_ = json.NewEncoder(result).Encode(got)
case "escape": // every attempt must fail but the write to /tmp; the result is each attempt's error
node := os.Getenv("OPENVIBE_TEST_NODE_DIR")
@@@ -1339,6 +1343,34 @@ func TestEgress(t *testing.T) {
}
}
++// TestEgressDNS: worker.dns becomes the /etc/resolv.conf of a job that runs with a network, and of no other.
++func TestEgressDNS(t *testing.T) {
++ if _, err := egressReady(); errors.Is(err, errEgressTool) {
++ t.Skip(err)
++ }
++ for _, c := range []struct{ name, net, want string }{
++ {"public", protocol.NetPublic, "nameserver 1.1.1.1\nnameserver 9.9.9.9\noptions timeout:2 attempts:2\n"},
++ {"no net", "", "absent"},
++ } {
++ t.Run(c.name, func(t *testing.T) {
++ cfg := helperConfig(t, config.WorkerCaps{}, "egress")
++ cfg.Egress, cfg.DNS = config.EgressPublic, []string{"1.1.1.1", "9.9.9.9"}
++ w, s := newRunWorkerConfig(t, cfg)
++ j := testJob(1, "egress")
++ j.Net = c.net
++ run(t, w, j)
++ ex := s.exit(t, j.ID)
++ var got map[string]string
++ if err := json.Unmarshal(ex.Result, &got); err != nil || ex.Reason != protocol.ExitExited {
++ t.Fatalf("%+v %s: %v", ex, ex.Result, err)
++ }
++ if got["resolv.conf"] != c.want {
++ t.Errorf("resolv.conf %q, want %q", got["resolv.conf"], c.want)
++ }
++ })
++ }
++}
++
// TestEgressRuleset: the table refuses IPv6, other routes than the default one, the private, shared, link-local and
// reserved ranges and egress_deny before anything is accepted; openvibe-only then accepts egress_allow alone.
func TestEgressRuleset(t *testing.T) {
ddiff --git a/internal/worker/sandbox_linux.go b/internal/worker/sandbox_linux.go
iindex 518a23e..87ac4ed 100644
---- a/internal/worker/sandbox_linux.go
++++ b/internal/worker/sandbox_linux.go
@@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"io/fs"
++ "net/netip"
"os"
"path/filepath"
"runtime"
@@@ -51,6 +52,8 @@ type sandboxSpec struct {
Limits []rlimit `json:"limits"`
Argv []string `json:"argv"`
Env []string `json:"env"`
++ // Resolvers are the nameservers of the job's /etc/resolv.conf (worker.dns, for a job with a network); none, none.
++ Resolvers []string `json:"resolvers,omitempty"`
}
// bindSpec bind-mounts the host's path at the same path in the private root, read-only unless it is a device node.
@@@ -166,6 +169,9 @@ func enterSandbox() error {
if err := step("copying the CA certificates", stageCA(r, caPaths)); err != nil {
return err
}
++ if err := step("writing /etc/resolv.conf", writeResolvConf(r, s.Resolvers)); err != nil {
++ return err
++ }
var artifactDev uint64
if s.Artifact != "" {
if artifactDev, err = stageArtifact(r, s.Artifact, s.Disk); err != nil {
@@@ -414,6 +420,27 @@ func inRoot(r, path string) (string, error) {
// maxCAFile is the largest CA certificate file copied (a bundle of every public CA is about 250 KiB).
const maxCAFile = 4 << 20
++// writeResolvConf writes the private root r's /etc/resolv.conf naming resolvers, the job's only way to resolve a name
++// (the host's resolver, 127.0.0.53 on most systems, is out of its reach); nothing for no resolvers. It is a file of
++// the private root, never the host's, and read-only once the root is.
++func writeResolvConf(r string, resolvers []string) error {
++ if len(resolvers) == 0 {
++ return nil
++ }
++ var b strings.Builder
++ for _, a := range resolvers {
++ if ip, err := netip.ParseAddr(a); err != nil || !ip.Is4() {
++ return fmt.Errorf("%q is not an IPv4 address", a)
++ }
++ b.WriteString("nameserver " + a + "\n")
++ }
++ b.WriteString("options timeout:2 attempts:2\n")
++ if err := os.MkdirAll(r+"/etc", 0o755); err != nil {
++ return err
++ }
++ return os.WriteFile(r+"/etc/resolv.conf", []byte(b.String()), 0o644)
++}
++
// stageCA copies the CA certificates at paths (those that exist) to the same paths under the private root r, as the
// job's user, and nothing else: of a regular file, only the PEM CERTIFICATE blocks that parse as X.509 certificates,
// written again without their headers and whatever else lay between them (a private key filed with them, say); a file
ddiff --git a/internal/worker/worker_linux.go b/internal/worker/worker_linux.go
iindex 62a2932..a8515a6 100644
---- a/internal/worker/worker_linux.go
++++ b/internal/worker/worker_linux.go
@@@ -77,6 +77,9 @@ func isolate(cmd *exec.Cmd, cfg config.WorkerConfig, p plan) (*sandbox, error) {
if art != nil {
spec.Artifact = p.fn.Artifact()
}
++ if p.egress != "" {
++ spec.Resolvers = cfg.DNS
++ }
for _, b := range binds {
spec.Binds = append(spec.Binds, bindSpec{Path: b.path, Dev: b.dev})
}
ddiff --git a/scripts/test-install-worker-config.sh b/scripts/test-install-worker-config.sh
iindex 2557fcb..6879434 100755
---- a/scripts/test-install-worker-config.sh
++++ b/scripts/test-install-worker-config.sh
@@@ -25,6 +25,7 @@ assert w['caps']['max_wall_ms'] == 3600000 and w['caps']['max_mem_bytes'] == 4 <
assert (f['name'], f['version'], f['class'], f['artifact_dir']) == ('openvibe-codes', '1.0.0', 'code', '/opt/openvibe-codes')
assert f['command'] == ['/usr/bin/node', '/opt/openvibe-codes/node_modules/openvibe-codes/bin/openvibe-codes.js', 'job']
assert f['env']['DEEPSEEK_API_KEY'] == '' and f['env']['HOME'] == '/tmp'
++assert w['dns'] == ['1.1.1.1', '9.9.9.9'], 'a coding job needs names resolved'
PY
# The owner adds a key and changes a cap; running the installer again keeps both and declares the runner once.
python3 - "$conf" <<'PY'