tenant.openagents/omega

1# NixOS VM tests for the Linux Bubblewrap (`bwrap`) sandbox.
2#
3# What actually changes the sandbox's behavior is the *host configuration* —
4# whether a usable `bwrap` exists and whether the kernel lets us create user
5# namespaces. We can only set those at boot, which is exactly what a VM gives
6# us. So we run one machine per host configuration:
7#
8#   working          - `bwrap` present, unprivileged user namespaces available;
9#                      the sandbox must be enforced.
10#   no-bwrap         - no `bwrap` on PATH; `Sandbox::can_create` must report
11#                      `BwrapNotFound` and the consumer must fail closed.
12#   setuid-bwrap     - the only `bwrap` is setuid-root; `can_create` must report
13#                      `BwrapSetuidRejected` (we refuse to run setuid bwrap).
14#   userns-disabled  - `bwrap` present but unprivileged user namespaces are
15#                      disabled via sysctl; `can_create` must report
16#                      `SandboxProbeFailed`.
17#
18# The behavior to assert is declared *here* as a list of `checks`, serialized to
19# JSON and handed to the `bwrap_test_helper` binary, which builds the described
20# sandbox policy via the sandbox crate's public API, performs the operation, and
21# asserts the outcome. Each check is one of:
22#
23#   { read = "/path"; succeeds = true; }          # read a host file
24#   { write = "/path"; succeeds = false; }        # write a host file
25#   { network = "echo1"; succeeds = true; }       # connect to an echo server
26#   { socketPath = "/run/x.sock"; succeeds = false; }   # connect a unix socket
27#   { canCreate = false; error = "bwrap_not_found"; }   # Sandbox::can_create
28#
29# plus optional policy fields applied to that check (defaults shown):
30#
31#   fs = "restricted";          # or "unrestricted"
32#   writablePaths = [ ];        # writable subtrees when fs = "restricted"
33#   networkAccess = "blocked";  # or "unrestricted" / "restricted"
34#   allowedDomains = [ ];       # allowed hosts when networkAccess = "restricted"
35#   protectedPaths = [ ];       # paths that remain readable but not writable,
36#                               # even if they fall under a writable subtree.
37#
38# Two echo servers (`echo1`, `echo2`) on separate nodes give the network checks
39# real peers, so a restricted-network policy that allowlists `echo1` can be
40# shown to reach `echo1` while `echo2` is blocked.
41#
42# Build one scenario:
43#   nix build .#checks.x86_64-linux.sandbox-working
44# Interactive:
45#   nix run .#checks.x86_64-linux.sandbox-working.driverInteractive
46{
47  pkgs,
48  inputs,
49}:
50let
51  lib = pkgs.lib;
52
53  bwrap-test-helper = import ./helper.nix { inherit pkgs inputs; };
54
55  # Both echo nodes listen on the same port; the helper appends it to a bare
56  # hostname (`echo1` -> `echo1:7000`).
57  echoPort = 7000;
58
59  # A node running a raw TCP echo server (socat EXEC:cat). The helper round-trips
60  # a byte through it, directly or via the sandbox's HTTP CONNECT proxy.
61  mkEchoNode = {
62    networking.firewall.allowedTCPPorts = [ echoPort ];
63    systemd.services.echo-server = {
64      description = "TCP echo server";
65      wantedBy = [ "multi-user.target" ];
66      after = [ "network.target" ];
67      serviceConfig = {
68        ExecStart = "${pkgs.socat}/bin/socat -d TCP-LISTEN:${toString echoPort},reuseaddr,fork EXEC:cat";
69        DynamicUser = true;
70        Restart = "on-failure";
71      };
72    };
73  };
74
75  # A unix-domain socket, owned by a process *outside* the sandbox, that a
76  # sandboxed command must not be able to `connect()` to. It lives under `/run`
77  # (which the sandbox `--ro-bind`s along with the rest of `/`) and NOT under
78  # `/tmp` (which the restricted-fs sandbox masks with a tmpfs, hiding anything
79  # there), so it stays visible inside the sandbox and the block is what's
80  # actually under test.
81  unixSocketPath = "/run/zed-sandbox-test.sock";
82
83  # Quiet boot + a couple of cores; shared by every machine-under-test.
84  baseMachine = {
85    boot.consoleLogLevel = lib.mkForce 3; # be quiet pls :)
86    services.journald.extraConfig = lib.mkAfter "MaxLevelConsole=notice";
87    environment.systemPackages = [ bwrap-test-helper ];
88    virtualisation = {
89      memorySize = 1024;
90      cores = 2;
91    };
92
93    # A unix-socket echo server outside the sandbox, so a sandboxed `connect()`
94    # has a real peer: without a listener the connect would fail with
95    # ECONNREFUSED even absent the sandbox, giving a false "blocked" pass.
96    systemd.services.unix-echo-server = {
97      description = "Unix-domain-socket echo server";
98      wantedBy = [ "multi-user.target" ];
99      serviceConfig = {
100        ExecStart = "${pkgs.socat}/bin/socat -d UNIX-LISTEN:${unixSocketPath},fork,reuseaddr EXEC:cat";
101        Restart = "on-failure";
102      };
103    };
104  };
105
106  # Per-scenario host configuration. Each entry layers onto `baseMachine`.
107  machineConfigs = {
108    # Plain, working host: bwrap on PATH, user namespaces available.
109    working =
110      { ... }:
111      lib.mkMerge [
112        baseMachine
113        { environment.systemPackages = [ pkgs.bubblewrap ]; }
114      ];
115
116    # No bwrap anywhere on PATH.
117    no-bwrap = { ... }: baseMachine;
118
119    # The only bwrap is a setuid-root wrapper. We deliberately do NOT also put a
120    # plain `pkgs.bubblewrap` on PATH: `locate_bwrap` returns the first `bwrap`
121    # it finds on PATH, and we want that to be the setuid one so the helper sees
122    # `BwrapSetuidRejected`.
123    setuid-bwrap =
124      { ... }:
125      lib.mkMerge [
126        baseMachine
127        {
128          security.wrappers.bwrap = {
129            source = "${pkgs.bubblewrap}/bin/bwrap";
130            owner = "root";
131            group = "root";
132            setuid = true;
133          };
134        }
135      ];
136
137    # bwrap present, but unprivileged user namespaces disabled at boot. Capping
138    # the count at 0 makes `bwrap --unshare-user` fail, exactly like a hardened
139    # host, so the probe reports `SandboxProbeFailed`.
140    userns-disabled =
141      { ... }:
142      lib.mkMerge [
143        baseMachine
144        {
145          environment.systemPackages = [ pkgs.bubblewrap ];
146          boot.kernel.sysctl."user.max_user_namespaces" = 0;
147        }
148      ];
149  };
150
151  # Build a VM test for `machine` that runs `checks` through the helper.
152  mkTest =
153    {
154      name,
155      machine,
156      checks,
157    }:
158    let
159      checksFile = pkgs.writeText "${name}-checks.json" (builtins.toJSON checks);
160    in
161    pkgs.testers.nixosTest {
162      inherit name;
163
164      nodes = {
165        echo1 = { ... }: mkEchoNode;
166        echo2 = { ... }: mkEchoNode;
167        machine = machine;
168      };
169
170      testScript = ''
171        start_all()
172
173        echo1.wait_for_unit("echo-server.service")
174        echo1.wait_for_open_port(${toString echoPort})
175        echo2.wait_for_unit("echo-server.service")
176        echo2.wait_for_open_port(${toString echoPort})
177
178        machine.wait_for_unit("multi-user.target")
179        machine.wait_until_succeeds("getent hosts echo1", timeout=30)
180        machine.wait_until_succeeds("getent hosts echo2", timeout=30)
181
182        # The unix-socket checks need a real peer outside the sandbox; wait for
183        # the listener to be up before running the helper.
184        machine.wait_until_succeeds("test -S ${unixSocketPath}", timeout=30)
185
186        # The helper logs each check tagged `[sandbox_test]:`. `succeed` fails the
187        # whole test on a non-zero exit; we print its output so the per-check
188        # results show up in the build log.
189        print(machine.succeed(
190            "ZED_SANDBOX_CHECKS=${checksFile} "
191            "ZED_TEST_ECHO_PORT=${toString echoPort} "
192            "bwrap_test_helper 2>&1"
193        ))
194      '';
195    };
196
197in
198{
199  sandbox-working = mkTest {
200    name = "sandbox-working";
201    machine = machineConfigs.working;
202    # Each check is written out in full (no shared fragments) so the policy and
203    # expected outcome for every case are obvious at a glance.
204    checks = [
205      # Reads of host files are always allowed, regardless of the write policy.
206      {
207        fs = "restricted";
208        writablePaths = [ "/sandbox-test/writable" ];
209        networkAccess = "blocked";
210        read = "/sandbox-test/readable/host.txt";
211        succeeds = true;
212      }
213
214      # Write inside the granted subtree: allowed.
215      {
216        fs = "restricted";
217        writablePaths = [ "/sandbox-test/writable" ];
218        networkAccess = "blocked";
219        write = "/sandbox-test/writable/ok.txt";
220        succeeds = true;
221      }
222
223      # Write outside the granted subtree: denied.
224      {
225        fs = "restricted";
226        writablePaths = [ "/sandbox-test/writable" ];
227        networkAccess = "blocked";
228        write = "/sandbox-test/forbidden/no.txt";
229        succeeds = false;
230      }
231
232      # A writable path that doesn't exist yet must be bound at its exact
233      # location (created up front), never widened to an existing ancestor. If it
234      # widened to `/sandbox-test`, this write to a read-only file under that
235      # ancestor would succeed — the scope-widening sandbox escape this guards
236      # against. It must be denied.
237      {
238        fs = "restricted";
239        writablePaths = [ "/sandbox-test/not-yet-created/deep" ];
240        networkAccess = "blocked";
241        write = "/sandbox-test/readable/host.txt";
242        succeeds = false;
243      }
244
245      # ...and the not-yet-existing path itself is created and writable.
246      {
247        fs = "restricted";
248        writablePaths = [ "/sandbox-test/not-yet-created/deep" ];
249        networkAccess = "blocked";
250        write = "/sandbox-test/not-yet-created/deep/ok.txt";
251        succeeds = true;
252      }
253
254      # The fs escape hatch lets a command write anywhere.
255      {
256        fs = "unrestricted";
257        networkAccess = "blocked";
258        write = "/sandbox-test/forbidden/anywhere.txt";
259        succeeds = true;
260      }
261
262      # ---- Protected paths ---------------------------------------------------
263      # A path inside a writable subtree can be re-bound read-only over the
264      # writable bind (order matters: later binds win). On Linux, protected paths
265      # remain readable but writes are denied.
266      {
267        fs = "restricted";
268        writablePaths = [ "/sandbox-test/repo" ];
269        protectedPaths = [ "/sandbox-test/repo/protected" ];
270        networkAccess = "blocked";
271        write = "/sandbox-test/repo/protected/test";
272        succeeds = false;
273      }
274
275      # Protected paths affect only the protected subtree: ordinary writes
276      # elsewhere in the writable worktree are unaffected.
277      {
278        fs = "restricted";
279        writablePaths = [ "/sandbox-test/repo" ];
280        protectedPaths = [ "/sandbox-test/repo/protected" ];
281        networkAccess = "blocked";
282        write = "/sandbox-test/repo/src/main.rs";
283        succeeds = true;
284      }
285
286      # Protected paths block writes but NOT reads on Linux because the protected
287      # subtree is read-only.
288      {
289        fs = "restricted";
290        writablePaths = [ "/sandbox-test/repo" ];
291        protectedPaths = [ "/sandbox-test/repo/protected" ];
292        networkAccess = "blocked";
293        read = "/sandbox-test/repo/protected/HEAD";
294        succeeds = true;
295      }
296
297      # Protected paths stay read-only even when ordinary filesystem writes are
298      # otherwise unrestricted.
299      {
300        fs = "unrestricted";
301        protectedPaths = [ "/sandbox-test/repo/protected" ];
302        networkAccess = "blocked";
303        write = "/sandbox-test/repo/protected/test";
304        succeeds = false;
305      }
306
307      # Blocked network: the echo server is unreachable.
308      {
309        fs = "restricted";
310        writablePaths = [ "/sandbox-test/writable" ];
311        networkAccess = "blocked";
312        network = "echo1";
313        succeeds = false;
314      }
315
316      # Unrestricted network: both echo servers are reachable.
317      {
318        fs = "restricted";
319        writablePaths = [ "/sandbox-test/writable" ];
320        networkAccess = "unrestricted";
321        network = "echo1";
322        succeeds = true;
323      }
324      {
325        fs = "restricted";
326        writablePaths = [ "/sandbox-test/writable" ];
327        networkAccess = "unrestricted";
328        network = "echo2";
329        succeeds = true;
330      }
331
332      # Restricted network: only the allowlisted host is reachable.
333      {
334        fs = "restricted";
335        writablePaths = [ "/sandbox-test/writable" ];
336        networkAccess = "restricted";
337        allowedDomains = [ "echo1" ];
338        network = "echo1";
339        succeeds = true;
340      }
341      {
342        fs = "restricted";
343        writablePaths = [ "/sandbox-test/writable" ];
344        networkAccess = "restricted";
345        allowedDomains = [ "echo1" ];
346        network = "echo2";
347        succeeds = false;
348      }
349
350      # ---- Unix-domain socket escape ----------------------------------------
351      # A sandboxed command must NOT be able to connect to a unix-domain socket
352      # owned by a process outside the sandbox (session-IPC escape). Currently
353      # FAILS (no seccomp guard yet); becomes a regression test once the
354      # socket(AF_UNIX) seccomp filter lands.
355      {
356        fs = "restricted";
357        writablePaths = [ "/sandbox-test/writable" ];
358        networkAccess = "blocked";
359        socketPath = unixSocketPath;
360        succeeds = false;
361      }
362
363      # The unix-socket block must hold regardless of network policy: our design
364      # decouples unix-socket blocking from the network grant, so even an
365      # unrestricted-network command must not reach an outside-the-sandbox
366      # session IPC socket. Also currently FAILS until the seccomp filter lands.
367      {
368        fs = "restricted";
369        writablePaths = [ "/sandbox-test/writable" ];
370        networkAccess = "unrestricted";
371        socketPath = unixSocketPath;
372        succeeds = false;
373      }
374
375      # On a working host the sandbox can be created.
376      {
377        fs = "restricted";
378        networkAccess = "blocked";
379        canCreate = true;
380      }
381    ];
382  };
383
384  sandbox-no-bwrap = mkTest {
385    name = "sandbox-no-bwrap";
386    machine = machineConfigs.no-bwrap;
387    checks = [ { canCreate = false; error = "bwrap_not_found"; } ];
388  };
389
390  sandbox-setuid-bwrap = mkTest {
391    name = "sandbox-setuid-bwrap";
392    machine = machineConfigs.setuid-bwrap;
393    checks = [ { canCreate = false; error = "setuid_rejected"; } ];
394  };
395
396  sandbox-userns-disabled = mkTest {
397    name = "sandbox-userns-disabled";
398    machine = machineConfigs.userns-disabled;
399    checks = [ { canCreate = false; error = "probe_failed"; } ];
400  };
401}
402