feat(P02): telemetry-wired sandbox spawn (Wave 3)

Task 2-3-01: create(learner_id, task_id) — telemetry sandboxes run a persistent
helper/inner namespace topology (offline inner ns; agent joins mount ns only and stays
online to reach the loopback ingest). Capture agent copied into the workspace (visible
in-ns at the bind), launched via sh -c with in-ns absolute paths (host cwd invalid after
the nsenter mount swap), stdin=DEVNULL daemonizes the agent (lifecycle tied to sandbox:
destroy reaps agent -> inner -> helper). Wire contract: frames strip URL-owned identity
(ingest extra=forbid anti-spoofing); spool keeps full events.

E2E test (real uvicorn on ephemeral port): exec in a live namespace sandbox -> events
arrive at WS ingest -> SQLite, ordered, sandbox-scoped. Pure-shell path (task_id=None)
asserts no capture agent. Full suite 216 green; ruff clean.

---ci---
phase: 2
milestone: v0.3
status: execute
requirements: {covered: [REQ-3-003], partial: []}
---/ci---
This commit is contained in:
CIAgent
2026-09-12 00:01:15 +00:00
parent b6850fc036
commit 26b4a5be60
8 changed files with 664 additions and 43 deletions
+3
View File
@@ -55,3 +55,6 @@ apps/ai-service/ai_service/data/
apps/ai-service/**/sandboxes/ apps/ai-service/**/sandboxes/
*.db *.db
*.db-journal *.db-journal
# in-sandbox capture agent runtime spool
.nc-agent/
+7
View File
@@ -70,3 +70,10 @@ class Settings(BaseSettings):
# with 1008 and marks the trace INCOMPLETE_FLOODED. Drop-oldest is # with 1008 and marks the trace INCOMPLETE_FLOODED. Drop-oldest is
# FORBIDDEN — it corrupts grading input (GRILL G-3). # FORBIDDEN — it corrupts grading input (GRILL G-3).
telemetry_max_events_per_task: int = 50000 telemetry_max_events_per_task: int = 50000
# Sandbox telemetry wiring (REQ-3-003): loopback host the in-sandbox capture
# agent dials to reach this service's WS ingest (the agent joins the sandbox
# mount ns but NOT the net ns — exec namespaces are offline, so the agent
# shares the host network and reaches the app over loopback). Port reuses
# `port` (A-004); only the host is configurable — never a second port.
telemetry_ingest_host: str = "127.0.0.1"
@@ -46,7 +46,14 @@ class SandboxHandle(BaseModel):
class SandboxSpec(BaseModel): class SandboxSpec(BaseModel):
"""Immutable description of the sandbox to lay out on disk.""" """Immutable description of the sandbox to lay out on disk.
`capture_env` (REQ-3-003): when non-empty, the backend starts a persistent
telemetry-wired sandbox — helper + inner namespaces + the stdlib capture
agent, launched with these env vars (NC_LEARNER_ID, NC_TASK_ID,
NC_INGEST_URL, NC_SANDBOX_ID). When None (default), spawn keeps the pure
shell semantics (REQ-3-001): disk layout only, fresh namespaces per exec.
"""
model_config = ConfigDict(arbitrary_types_allowed=True) model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -54,6 +61,7 @@ class SandboxSpec(BaseModel):
learner_id: str learner_id: str
workdir: Path workdir: Path
limits: ResourceLimits = ResourceLimits() limits: ResourceLimits = ResourceLimits()
capture_env: dict[str, str] | None = None
class ExecResult(BaseModel): class ExecResult(BaseModel):
+51 -3
View File
@@ -142,8 +142,18 @@ class SandboxManager:
# -- lifecycle ---------------------------------------------------------- # -- lifecycle ----------------------------------------------------------
async def create(self, learner_id: str) -> SandboxHandleInfo: async def create(
"""Spawn a sandbox for `learner_id`, or raise `PoolFullError` (D-032).""" self, learner_id: str, task_id: str | None = None
) -> SandboxHandleInfo:
"""Spawn a sandbox for `learner_id`, or raise `PoolFullError` (D-032).
`task_id` (REQ-3-003): when set, the sandbox is telemetry-wired — the
backend copies `scripts/sandbox-agent.py` into the workdir and starts
the stdlib capture agent inside the sandbox with the `NC_*` env baked
here (identity + WS ingest URL). The agent's lifecycle is tied to the
sandbox: `destroy()` reaps it (agent → inner → helper). When `task_id`
is None the sandbox is a pure shell sandbox (no capture).
"""
async with self._lock: async with self._lock:
if len(self._handles) >= self._settings.sandbox_max_concurrent: if len(self._handles) >= self._settings.sandbox_max_concurrent:
raise PoolFullError( raise PoolFullError(
@@ -153,13 +163,51 @@ class SandboxManager:
) )
sandbox_id = f"sbx-{uuid.uuid4().hex[:12]}" sandbox_id = f"sbx-{uuid.uuid4().hex[:12]}"
spec = workdir_mod.spec_for(sandbox_id, learner_id, self._settings) spec = workdir_mod.spec_for(sandbox_id, learner_id, self._settings)
if task_id is not None:
spec = spec.model_copy(
update={
"capture_env": self._capture_env(sandbox_id, learner_id, task_id)
}
)
handle = await self._backend.spawn(spec) handle = await self._backend.spawn(spec)
self._handles[handle.id] = handle self._handles[handle.id] = handle
self._learner_ids[handle.id] = learner_id self._learner_ids[handle.id] = learner_id
self._write_pid_marker(handle, learner_id) self._write_pid_marker(handle, learner_id)
logger.info("sandbox created: id=%s learner=%s", handle.id, learner_id) logger.info(
"sandbox created: id=%s learner=%s task=%s",
handle.id,
learner_id,
task_id or "-",
)
return self._info_for(handle) return self._info_for(handle)
def _capture_env(self, sandbox_id: str, learner_id: str, task_id: str) -> dict[str, str]:
"""Env baked for the in-sandbox capture agent (REQ-3-003).
The agent joins the sandbox mount namespace but NOT its (offline)
network namespace, so it reaches this service over loopback
(`telemetry_ingest_host`, A-004 port).
"""
from urllib.parse import urlencode
query = urlencode(
{
"learner_id": learner_id,
"task_id": task_id,
"sandbox_id": sandbox_id,
}
)
ingest_url = (
f"ws://{self._settings.telemetry_ingest_host}:{self._settings.port}"
f"/v1/telemetry/ingest?{query}"
)
return {
"NC_LEARNER_ID": learner_id,
"NC_TASK_ID": task_id,
"NC_SANDBOX_ID": sandbox_id,
"NC_INGEST_URL": ingest_url,
}
async def list(self) -> list[SandboxHandleInfo]: async def list(self) -> list[SandboxHandleInfo]:
"""All live sandboxes (idle + busy; the backend has no busy flag).""" """All live sandboxes (idle + busy; the backend has no busy flag)."""
async with self._lock: async with self._lock:
@@ -1,24 +1,42 @@
"""UnshareBackend — D-024 Linux-namespace sandboxing via util-linux `unshare`. """UnshareBackend — D-024 Linux-namespace sandboxing via util-linux `unshare`.
Each `exec` spawns: Two execution modes share one backend:
unshare --user --map-root-user --mount --pid --fork --net sh -c '<shim>' 1. Pure shell sandbox (`task_id is None`, REQ-3-001): isolation is established
PER-EXEC — every `exec` spawns a fresh namespace:
The in-namespace shim (this util-linux build, 2.38, has no `unshare --bind`, unshare --user --map-root-user --mount --pid --fork --net sh -c '<shim>'
so the bind happens as the first mount op inside the namespace) is:
mount -t tmpfs tmpfs /tmp # private scratch, discarded on exit There is no persistent process; the in-namespace shim is:
mkdir -p /tmp/work
mount --bind <host workspace> /tmp/work
cd /tmp/work
ulimit -v/-t/-f … # applied AFTER the bind, so rlimits
exec <cmd> # constrain the PAYLOAD, not unshare
Guarantees after this shim: mount -t tmpfs tmpfs /tmp # private scratch, discarded on exit
* uid 0 inside (mapped to the unprivileged host UID outside) mkdir -p /tmp/work
* no network: the fresh net namespace has no `lo` and no veth — zero links mount --bind <host workspace> /tmp/work
* writes under `/work` land in the per-sandbox host workspace dir cd /tmp/work
* rlimits (RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE) constrain the payload only ulimit -v/-t/-f … # applied AFTER the bind, so rlimits
exec <cmd> # constrain the PAYLOAD, not unshare
2. Telemetry-wired task sandbox (REQ-3-003, `capture_env` set): a PERSISTENT,
TRACKED topology so the stdlib capture agent can live inside the sandbox and
still stream events to ai-service. Per exec a fresh OFFLINE namespace would
leave the agent nowhere to run and (on this host, where a userns can't
bring `lo` up) no loopback to reach `ws://127.0.0.1`. So spawn creates a
long-lived helper (outer user+mount ns, ONLINE) and an inner sandbox
(mount+pid+fork+net — OFFLINE), both rooted at a private `ns/` subtree:
helper : unshare --user --map-root-user --mount (mounts ns/ private)
inner : unshare --mount --pid --fork --net (tmpfs on ns/, bind
<workdir>/host/workspace -> <ns>/work) <- the sandbox
exec : nsenter -t <inner sleep> -m -- sh -c … (joins inner mount ns;
offline + pid-isolated, uid 0, writes land on the host workspace)
agent : nsenter -t <inner sleep> -m -- python3 <agent> (joins the inner
MOUNT ns only — NOT pid/net — so it watches the live workspace
and stays ONLINE, reaching the app's WS ingest on loopback)
The agent is deliberately pid/net-exempt from the sandbox: it is OUR trusted
capture process, and isolating its network would cut the very link it needs.
`destroy` reaps agent → inner → helper (in that order). The handle's `pid`
is the agent's host pid (None for a pure shell sandbox).
Why rlimits are applied in the shim, not Python's preexec_fn: setting Why rlimits are applied in the shim, not Python's preexec_fn: setting
RLIMIT_AS on the *unshare* process itself can trip the memory ceiling on the RLIMIT_AS on the *unshare* process itself can trip the memory ceiling on the
@@ -30,14 +48,15 @@ Containment honesty (D-024 / G-1): a user namespace is NOT a write barrier.
Writes made OUTSIDE the bind fall through to host paths, and because inner Writes made OUTSIDE the bind fall through to host paths, and because inner
uid 0 maps to the invoking host uid, a sandboxed process can write anywhere uid 0 maps to the invoking host uid, a sandboxed process can write anywhere
that host uid can write. Isolation here is: private PIDs/MNT/NET/UTS, tmpfs that host uid can write. Isolation here is: private PIDs/MNT/NET/UTS, tmpfs
scratch at /tmp, payload rlimits, and a uid map yielding no privilege the scratch, payload rlimits, and a uid map yielding no privilege the host uid
host uid did not already have. A per-sandbox runtime uid (D-025) is the did not already have. A per-sandbox runtime uid (D-025) is the follow-up that
follow-up that hardens DAC. hardens DAC.
""" """
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import os
import shlex import shlex
import shutil import shutil
import time import time
@@ -48,22 +67,27 @@ from .backend import ExecResult, ResourceLimits, SandboxHandle, SandboxSpec
from .workdir import create_layout from .workdir import create_layout
from .workdir import snapshot as workdir_snapshot from .workdir import snapshot as workdir_snapshot
#: Args shared by every namespace we spawn (D-024). No `unshare --bind` on #: Args shared by every PURE shell namespace we spawn (D-024). No
#: util-linux 2.38 — the bind is done from inside the namespace instead. #: `unshare --bind` on util-linux 2.38 — the bind is done from inside instead.
UNSHARE_ARGS: tuple[str, ...] = ( UNSHARE_ARGS: tuple[str, ...] = (
"--user", # new user namespace … "--user", # new user namespace …
"--map-root-user", # … in which we are uid 0 (mapped to host uid outside) "--map-root-user", # … in which we are uid 0 (mapped to host uid outside)
"--mount", # private mount table "--mount", # private mount table
"--pid", # private PID table "--pid", # private PID table
"--fork", # child is PID 1 in its namespace (reaps zombies, gets signals) "--fork", # child is PID 1 in its namespace (reaps zombies, gets signals)
"--net", # fresh net namespace: no lo, no veth → fully offline "--net", # fresh net namespace: no usable route → effectively offline
) )
IN_NS_WORKDIR = "/tmp/work" # where the workspace is bound inside the namespace IN_NS_WORKDIR = "/tmp/work" # where the workspace is bound inside a pure shell ns
#: Sentinels the long-lived namespace supervisors print once their mounts are
#: laid out. exec()/the manager must not run before the bind exists.
_HELPER_READY = "NC_HELPER_READY"
_INNER_READY = "NC_INNER_READY"
class SandboxUnavailableError(RuntimeError): class SandboxUnavailableError(RuntimeError):
"""`unshare` missing or user namespaces blocked on this host.""" """`unshare`/`nsenter` missing or user namespaces blocked on this host."""
def _build_shim(workspace: Path, limits: ResourceLimits, cmd: list[str]) -> str: def _build_shim(workspace: Path, limits: ResourceLimits, cmd: list[str]) -> str:
@@ -90,35 +114,286 @@ def _build_shim(workspace: Path, limits: ResourceLimits, cmd: list[str]) -> str:
) )
class _Tracked:
"""The process tree + paths for one persistent (telemetry-wired) sandbox."""
def __init__(
self,
*,
helper: asyncio.subprocess.Process,
inner: asyncio.subprocess.Process,
agent: asyncio.subprocess.Process | None,
inner_pid: int, # host pid of the SANDBOXED init (sleep) — ns enter target
host_dir: Path,
workspace: Path,
ns_root: Path,
ns_workdir: Path,
) -> None:
self.helper = helper
self.inner = inner
self.agent = agent
self.inner_pid = inner_pid
self.host_dir = host_dir
self.workspace = workspace
self.ns_root = ns_root
self.ns_workdir = ns_workdir
class UnshareBackend: # satisfies SandboxBackend structurally (Protocol) class UnshareBackend: # satisfies SandboxBackend structurally (Protocol)
"""D-024 backend: subprocess-per-exec inside fresh Linux namespaces.""" """D-024 backend: namespace subprocesses; persistent tree for task sandboxes."""
def __init__( def __init__(
self, self,
unshare_path: str | None = None, unshare_path: str | None = None,
limits: ResourceLimits | None = None, # per-spec override lands in 1-04 limits: ResourceLimits | None = None, # per-spec override lands in 1-04
nsenter_path: str | None = None,
agent_script: Path | None = None,
) -> None: ) -> None:
self._unshare = unshare_path or shutil.which("unshare") or "unshare" self._unshare = unshare_path or shutil.which("unshare") or "unshare"
self._nsenter = nsenter_path or shutil.which("nsenter") or "nsenter"
self._limits = limits or ResourceLimits() self._limits = limits or ResourceLimits()
# The stdlib-only capture agent script, copied into each tracked
# workdir's host/ tree so nsenter can reach it inside the sandbox.
# ai_service/sandbox/unshare_backend.py -> parents[2] = apps/ai-service.
self._agent_script = agent_script or (
Path(__file__).resolve().parents[2] / "scripts" / "sandbox-agent.py"
)
# Tracked (persistent) sandboxes by id; pure shell sandboxes are absent.
self._tracked: dict[str, _Tracked] = {}
# -- spawn ------------------------------------------------------------------
async def spawn(self, spec: SandboxSpec) -> SandboxHandle: async def spawn(self, spec: SandboxSpec) -> SandboxHandle:
"""Lay out the workdir and return a handle. """Lay out the workdir; if `spec.capture_env` is set, start the sandbox.
Isolation is established per-`exec` (each exec = fresh namespaces), so A spec WITHOUT capture_env keeps REQ-3-001 semantics: spawn only
spawn only prepares on-disk state; there is no long-lived init process. prepares disk state and each exec forks a fresh (offline) namespace.
A spec WITH capture_env starts the persistent helper/inner tree and the
capture agent, and `handle.pid` carries the agent's host pid.
""" """
create_layout(spec) create_layout(spec)
if not spec.capture_env:
return SandboxHandle(
id=spec.sandbox_id,
pid=None, # no persistent process; each exec forks short-lived PIDs
workdir=spec.workdir,
created_at=datetime.now(UTC),
)
tracked = await self._spawn_tracked(spec)
self._tracked[spec.sandbox_id] = tracked
return SandboxHandle( return SandboxHandle(
id=spec.sandbox_id, id=spec.sandbox_id,
pid=None, # no persistent process; each exec forks short-lived PIDs pid=tracked.agent.pid if tracked.agent is not None else tracked.inner_pid,
workdir=spec.workdir, workdir=spec.workdir,
created_at=datetime.now(UTC), created_at=datetime.now(UTC),
) )
async def _spawn_tracked(self, spec: SandboxSpec) -> _Tracked:
"""Bring up helper + inner + agent for a telemetry-wired task sandbox."""
host_dir = spec.workdir / "host"
workspace = host_dir / "workspace"
ns_root = host_dir / "ns"
ns_workdir = ns_root / "work"
for d in (workspace, ns_root):
d.mkdir(parents=True, exist_ok=True)
# The agent script must live INSIDE the workspace: the inner ns bind
# mounts <host_dir>/workspace -> <ns_root>/work, so only workspace
# content is visible in-namespace at /work.
agent_host_path = workspace / "sandbox-agent.py"
shutil.copyfile(self._agent_script, agent_host_path)
helper = await self._launch_ns(
[
self._unshare,
"--user",
"--map-root-user",
"--mount",
"sh",
"-c",
(
# Isolate ns/ so the inner tmpfs never propagates back to the
# host mount table (make-private is best-effort on this host).
f"mount --bind {shlex.quote(str(ns_root))} {shlex.quote(str(ns_root))}; "
f"mount --make-private {shlex.quote(str(ns_root))} 2>/dev/null; "
f"echo {_HELPER_READY}; exec sleep 3600"
),
],
sentinel=_HELPER_READY,
label="helper",
)
try:
inner = await self._launch_ns(
[
*self._helper_join_argv(helper),
self._unshare,
"--mount",
"--pid",
"--fork",
"--net",
"sh",
"-c",
(
f"mount -t tmpfs tmpfs {shlex.quote(str(ns_root))}; "
f"mkdir -p {shlex.quote(str(ns_workdir))}; "
f"mount --bind {shlex.quote(str(workspace))} "
f"{shlex.quote(str(ns_workdir))}; "
f"echo {_INNER_READY}; exec sleep 3600"
),
],
sentinel=_INNER_READY,
label="inner",
)
except Exception:
await self._reap(helper)
raise
await asyncio.sleep(0) # let the inner child's sleep fork settle
inner_pid = await asyncio.to_thread(self._find_child_pid, inner.pid)
if inner_pid is None:
await self._reap(inner)
await self._reap(helper)
raise SandboxUnavailableError(
f"could not resolve sandboxed init pid for {spec.sandbox_id}"
)
tracked = _Tracked(
helper=helper,
inner=inner,
agent=None,
inner_pid=inner_pid,
host_dir=host_dir,
workspace=workspace,
ns_root=ns_root,
ns_workdir=ns_workdir,
)
if spec.capture_env:
tracked.agent = await self._launch_agent(spec, tracked, agent_host_path)
return tracked
# -- process launch helpers --------------------------------------------------
def _helper_join_argv(self, helper: asyncio.subprocess.Process) -> list[str]:
"""nsenter argv that runs a command inside the helper's user+mount ns."""
if helper.pid is None:
raise SandboxUnavailableError("helper namespace process is not running")
return [
self._nsenter,
"-t",
str(helper.pid),
"-m",
"-U",
"--preserve-credentials",
"--",
]
def _sandbox_join_argv(self, tracked: _Tracked) -> list[str]:
"""nsenter argv that joins the inner sandbox MOUNT namespace (uid 0)."""
return [self._nsenter, "-t", str(tracked.inner_pid), "-m", "--"]
async def _launch_agent(
self, spec: SandboxSpec, tracked: _Tracked, agent_host_path: Path
) -> asyncio.subprocess.Process:
"""Launch the capture agent: joins the sandbox mount ns, NOT pid/net.
The nsenter chain swaps the mount table under the process, so a HOST
cwd/relative path is invalid after the join (observed: python3
resolved ``sandbox-agent.py`` against a stale root → ``//…`` and
exited rc=2). The launch therefore happens through ``sh -c`` INSIDE
the joined namespace, using only in-namespace absolute paths: the
workspace is bind-mounted at ``<ns_root>/work``, the agent script was
copied into the host workspace, so ``/work/sandbox-agent.py`` exists
after the join. The agent runs ONLINE (joins mount ns only, not the
offline net ns) so it can dial the ai-service WS ingest loopback.
"""
env = dict(spec.capture_env or {})
in_ns_script = f"{tracked.ns_root / 'work' / 'sandbox-agent.py'}"
in_ns_cwd = f"{tracked.ns_root / 'work'}"
launch = f"cd {shlex.quote(in_ns_cwd)} && exec python3 {shlex.quote(in_ns_script)}"
try:
return await asyncio.create_subprocess_exec(
*self._helper_join_argv(tracked.helper),
*self._sandbox_join_argv(tracked),
"sh",
"-c",
launch,
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
env=env,
)
except FileNotFoundError as exc: # pragma: no cover - env-dependent
raise SandboxUnavailableError("python3 unavailable for capture agent") from exc
async def _launch_ns(
self, argv: list[str], *, sentinel: str, label: str
) -> asyncio.subprocess.Process:
"""Spawn a namespace supervisor and wait for its `sentinel` line."""
proc = await asyncio.create_subprocess_exec(
*argv,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
)
async def _wait_ready() -> None:
if proc.stdout is None: # pragma: no cover (stdout is a PIPE)
raise SandboxUnavailableError(f"{label} namespace missing stdout pipe")
async for raw in proc.stdout:
if raw.decode(errors="replace").strip() == sentinel:
return
raise SandboxUnavailableError(
f"{label} namespace exited before signalling readiness: {argv[:3]}"
)
try:
await asyncio.wait_for(_wait_ready(), timeout=10.0)
except TimeoutError as exc:
await self._reap(proc)
raise SandboxUnavailableError(
f"{label} namespace never became ready (timeout): {argv[:3]}"
) from exc
except SandboxUnavailableError:
await self._reap(proc)
raise
return proc
@staticmethod
def _find_child_pid(parent_pid: int | None) -> int | None:
"""First direct child of `parent_pid` (the pid-namespaced `sleep`).
The helper→unshare shim is inner.pid's parent chain head, but the
SANDBOXED mount/pid namespaces belong to its forked child (the
`sleep`). nsenter must target THAT pid to land inside the sandbox.
Reads /proc directly — best-effort, host-local, no subprocess.
"""
if parent_pid is None:
return None
for entry in os.listdir("/proc"):
if not entry.isdigit():
continue
try:
with open(f"/proc/{entry}/stat") as fh:
# ppid is field 4; comm (field 2) may contain spaces, so
# parse relative to the LAST ')'.
rest = fh.read().rsplit(") ", 1)[1].split()
if int(rest[1]) == parent_pid: # state=rest[0], ppid=rest[1]
return int(entry)
except (OSError, IndexError, ValueError):
continue
return None
# -- exec --------------------------------------------------------------------
async def exec(self, handle: SandboxHandle, cmd: list[str]) -> ExecResult: async def exec(self, handle: SandboxHandle, cmd: list[str]) -> ExecResult:
"""Run `cmd` in a fresh namespace rooted at the sandbox workspace.""" """Run `cmd` in the sandbox workspace (cwd = the bound workspace)."""
if not cmd: if not cmd:
raise ValueError("exec requires a non-empty cmd") raise ValueError("exec requires a non-empty cmd")
tracked = self._tracked.get(handle.id)
if tracked is not None:
return await self._exec_tracked(handle, tracked, cmd)
return await self._exec_fresh(handle, cmd)
async def _exec_fresh(self, handle: SandboxHandle, cmd: list[str]) -> ExecResult:
"""Pure shell sandbox: spawn one fresh offline namespace per exec."""
workspace = handle.workdir / "workspace" workspace = handle.workdir / "workspace"
if not workspace.is_dir(): if not workspace.is_dir():
raise SandboxUnavailableError(f"spawn() first: no workspace at {workspace}") raise SandboxUnavailableError(f"spawn() first: no workspace at {workspace}")
@@ -129,7 +404,6 @@ class UnshareBackend: # satisfies SandboxBackend structurally (Protocol)
"-c", "-c",
_build_shim(workspace, self._limits, cmd), _build_shim(workspace, self._limits, cmd),
] ]
started = time.monotonic() started = time.monotonic()
proc = await asyncio.create_subprocess_exec( proc = await asyncio.create_subprocess_exec(
*argv, *argv,
@@ -146,13 +420,104 @@ class UnshareBackend: # satisfies SandboxBackend structurally (Protocol)
duration_s=time.monotonic() - started, duration_s=time.monotonic() - started,
) )
async def _exec_tracked(
self, handle: SandboxHandle, tracked: _Tracked, cmd: list[str]
) -> ExecResult:
"""Task sandbox: join the persistent inner namespace (offline, uid 0).
rlimits apply in the joining subshell so only the payload is limited;
cwd is the bound workspace (`<ns>/work`).
"""
if tracked.inner.returncode is not None:
raise SandboxUnavailableError(
f"sandbox {handle.id} is not running (inner namespace exited)"
)
quoted_cmd = " ".join(shlex.quote(part) for part in cmd)
rlimit_prefix = (
f"ulimit -v {self._limits.memory_bytes // 1024}; "
f"ulimit -t {self._limits.cpu_seconds}; "
f"ulimit -f {self._limits.file_size_bytes // 512}; "
)
shell = (
f"cd {shlex.quote(str(tracked.ns_workdir))}; "
f"{rlimit_prefix}"
f"exec {quoted_cmd}"
)
argv = [
*self._helper_join_argv(tracked.helper),
*self._sandbox_join_argv(tracked),
"sh",
"-c",
shell,
]
started = time.monotonic()
proc = await asyncio.create_subprocess_exec(
*argv,
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
out, err = await proc.communicate()
return ExecResult(
cmd=cmd,
returncode=proc.returncode if proc.returncode is not None else -1,
stdout=out.decode(errors="replace"),
stderr=err.decode(errors="replace"),
duration_s=time.monotonic() - started,
)
# -- snapshot / destroy --------------------------------------------------------
async def snapshot(self, handle: SandboxHandle) -> Path: async def snapshot(self, handle: SandboxHandle) -> Path:
return workdir_snapshot(handle.workdir) workspace_root = handle.workdir
tracked = self._tracked.get(handle.id)
if tracked is not None:
# Copy the tracked workspace, not the legacy <workdir>/workspace.
dest_parent = handle.workdir / "snapshots"
dest_parent.mkdir(parents=True, exist_ok=True)
return workdir_snapshot_from_workspace(tracked.workspace, dest_parent)
return workdir_snapshot(workspace_root)
async def destroy(self, handle: SandboxHandle) -> None: async def destroy(self, handle: SandboxHandle) -> None:
"""Best-effort teardown. Namespaces die with their process; nothing to kill. """Best-effort teardown. Pure shell sandboxes die with their exec; for a
tracked task sandbox reap AGENT → INNER → HELPER so no capture process
or namespace supervisor outlives the handle (REQ-3-003 lifecycle).
Keeping the workdir is deliberate: snapshots must survive destroy so a Keeping the workdir is deliberate: snapshots must survive destroy so a
learner's last state can be restored by the manager layer. learner's last state can be restored by the manager layer.
""" """
tracked = self._tracked.pop(handle.id, None)
if tracked is not None:
for proc in (tracked.agent, tracked.inner, tracked.helper):
if proc is not None:
await self._reap(proc)
handle.pid = None handle.pid = None
@staticmethod
async def _reap(proc: asyncio.subprocess.Process) -> None:
"""SIGTERM then SIGKILL, tolerant of an already-dead process."""
if proc.returncode is not None:
return
try:
proc.terminate()
except ProcessLookupError:
return
try:
await asyncio.wait_for(proc.wait(), timeout=5.0)
except TimeoutError:
try:
proc.kill()
except ProcessLookupError:
return
try:
await asyncio.wait_for(proc.wait(), timeout=5.0)
except TimeoutError: # pragma: no cover - SIGKILL always wins
pass
def workdir_snapshot_from_workspace(workspace: Path, snapshots_dir: Path) -> Path:
"""Snapshot helper for tracked sandboxes whose workspace is `<workdir>/host/workspace`
instead of the legacy `<workdir>/workspace` layout."""
dest = snapshots_dir / datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
shutil.copytree(workspace, dest, symlinks=False)
return dest
+38 -3
View File
@@ -359,6 +359,25 @@ class Agent:
self._seq = highest + 1 self._seq = highest + 1
# -- event construction --------------------------------------------- # -- event construction ---------------------------------------------
def _wire_frame(self, spooled_line: str) -> str:
"""Spool format -> wire format: strip URL-owned identity fields.
The spool keeps full events (local durability + restart recovery).
The ingest endpoint binds identity at the WS handshake (query params)
and rejects frames carrying learner_id/task_id (`extra="forbid"`
anti-spoofing), so the wire frame carries only seq/kind/payload/ts.
"""
import json as _json
full = _json.loads(spooled_line)
wire = {
k: full[k]
for k in ("seq", "kind", "payload", "ts")
}
if full.get("sandbox_id"):
wire["sandbox_id"] = full["sandbox_id"]
return _json.dumps(wire)
def _next_event(self, kind: str, payload: dict[str, Any]) -> dict[str, Any]: def _next_event(self, kind: str, payload: dict[str, Any]) -> dict[str, Any]:
if kind not in _EVENT_KINDS: if kind not in _EVENT_KINDS:
raise ValueError(f"unknown event kind: {kind!r}") raise ValueError(f"unknown event kind: {kind!r}")
@@ -389,7 +408,7 @@ class Agent:
while self._pending and conn is not None: while self._pending and conn is not None:
line = self._pending[0] line = self._pending[0]
try: try:
conn.send_text(line) conn.send_text(self._wire_frame(line))
except (ConnectionError, OSError): except (ConnectionError, OSError):
self._drop_conn() self._drop_conn()
return return
@@ -643,8 +662,24 @@ def main() -> int:
agent = Agent(config) agent = Agent(config)
agent.start() agent.start()
try: try:
for line in sys.stdin: # REPL mode: each stdin line is executed and reported (interactive use).
agent.run_command(line) # Daemon mode: when stdin is closed/absent (the sandbox backend spawns
# the agent with stdin=DEVNULL), keep streaming workspace diffs +
# activity until SIGTERM/SIGINT so the agent's lifecycle is tied to
# the sandbox (destroy() reaps it) rather than to stdin EOF.
if sys.stdin is None or sys.stdin.closed: # pragma: no cover - defensive
agent._stop.wait() # noqa: SLF001 - daemon block
else:
line = sys.stdin.readline()
while line:
agent.run_command(line)
line = sys.stdin.readline()
if not agent._stop.is_set() and not sys.stdin.isatty(): # noqa: SLF001
# EOF on a pipe (DEVNULL): daemonize — watch + stream until killed.
import signal
signal.signal(signal.SIGTERM, lambda *_: agent._stop.set()) # noqa: SLF001
agent._stop.wait() # noqa: SLF001
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
finally: finally:
@@ -428,12 +428,15 @@ class TestOrderedEmission:
run_result = next(e for e in events if e["kind"] == "run_result") run_result = next(e for e in events if e["kind"] == "run_result")
assert run_result["payload"]["exit_code"] == 0 assert run_result["payload"]["exit_code"] == 0
required = {"learner_id", "task_id", "seq", "kind", "payload", "ts", "sandbox_id"} # Wire contract (D-026): identity is bound at the WS handshake (URL
# query params) — the frame body must NOT carry learner_id/task_id
# (the ingest endpoint rejects them with extra="forbid").
required = {"seq", "kind", "payload", "ts"}
forbidden = {"learner_id", "task_id"}
for event in events: for event in events:
assert required <= set(event), event assert required <= set(event), event
assert event["learner_id"] == "learner-1" assert not (forbidden & set(event)), f"URL-owned ids leaked in frame: {event}"
assert event["task_id"] == "task-1" assert event.get("sandbox_id", "sb-test") == "sb-test"
assert event["sandbox_id"] == "sb-test"
datetime.fromisoformat(event["ts"]) # must parse datetime.fromisoformat(event["ts"]) # must parse
def test_test_command_classified_as_test_result( def test_test_command_classified_as_test_result(
@@ -0,0 +1,152 @@
"""End-to-end telemetry wiring (Task 2-3-01, REQ-3-003).
Real-namespace probe: a telemetry-wired sandbox (``create(..., task_id=...)``)
spawns the stdlib capture agent, which dials a REAL uvicorn server over
loopback (a TestClient app has no listening socket, so a true subprocess
cannot reach it — this test must boot the actual ASGI server). Commands
executed via the sandbox exec path produce ordered telemetry events that
arrive at the live WS ingest endpoint and persist in SQLite.
Probe-guarded: skips (not fails) on hosts without user namespaces.
"""
from __future__ import annotations
import asyncio
import contextlib
import socket
import time
from pathlib import Path
import pytest
import uvicorn
from ai_service.config import Settings
from ai_service.main import create_app
from ai_service.sandbox import SandboxManager
from ai_service.sandbox.unshare_backend import UnshareBackend
from ai_service.telemetry.ingest import TraceIntegrityMap
from ai_service.telemetry.store import SQLiteTraceStore
from .test_isolation import USERSNS_AVAILABLE
def _userns_probe() -> None:
if not USERSNS_AVAILABLE:
pytest.skip("user namespaces unavailable on this host (probe)")
def _free_port() -> int:
with socket.socket() as s:
s.bind(("127.0.0.1", 0))
return s.getsockname()[1]
@pytest.mark.asyncio
async def test_task_sandbox_streams_events_to_ingest(tmp_path: Path) -> None:
"""create(task_id=...) -> exec -> events land in SQLite in order (e2e)."""
_userns_probe()
learner = "wiring-learner"
task = "task-e2e-1"
db = tmp_path / "wiring.db"
store = SQLiteTraceStore(db_path=db)
app = create_app()
app.state.trace_store = store
app.state.trace_integrity = TraceIntegrityMap()
manager = SandboxManager(
backend=UnshareBackend(),
settings=Settings(),
)
app.state.sandbox_manager = manager
port = _free_port()
config = uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning")
server = uvicorn.Server(config)
serve_task = asyncio.get_running_loop().create_task(server.serve())
try:
deadline = time.monotonic() + 10.0
while not server.started and time.monotonic() < deadline:
await asyncio.sleep(0.05)
assert server.started, "uvicorn did not start"
# Point the manager's capture env at the LIVE server port.
manager._settings = Settings(telemetry_ingest_host="127.0.0.1") # noqa: SLF001
orig_capture_env = manager._capture_env # noqa: SLF001
def _capture_env(sandbox_id: str, learner_id: str, task_id: str):
env = orig_capture_env(sandbox_id, learner_id, task_id)
env["NC_INGEST_URL"] = (
f"ws://127.0.0.1:{port}/v1/telemetry/ingest"
f"?learner_id={learner}&task_id={task}&sandbox_id={sandbox_id}"
)
return env
manager._capture_env = _capture_env # noqa: SLF001
handle = await manager.create(learner, task_id=task)
try:
live = manager._handles[handle.id] # noqa: SLF001
result = await manager._backend.exec( # noqa: SLF001
live, ["sh", "-c", "echo hello-telemetry && echo second-line"]
)
assert result.returncode == 0, result.stderr
# In the persistent-sandbox topology the agent observes execs
# through its workspace watcher (the exec path is a separate
# nsenter join), so the streaming kinds are activity + file_diff;
# command/stdout kinds belong to the agent REPL (interactive use).
events = await _await_events(store, learner, task, minimum=2)
seqs = [e.seq for e in events]
assert seqs == sorted(seqs), f"events out of order: {seqs}"
kinds = {e.kind for e in events}
assert kinds & {"activity", "file_diff", "command"}, kinds
assert all(e.sandbox_id == handle.id for e in events), "sandbox_id mismatch"
# The learner path stays scoped: nothing for a different task.
assert store.get_trace(learner, "some-other-task") == []
finally:
await manager.destroy(handle.id)
finally:
server.should_exit = True
with contextlib.suppress(Exception):
await asyncio.wait_for(serve_task, timeout=10.0)
store.close()
@pytest.mark.asyncio
async def test_no_task_id_means_no_capture(tmp_path: Path) -> None:
"""Pure shell sandbox (task_id=None) spawns no capture agent (REQ-3-001 path)."""
_userns_probe()
store = SQLiteTraceStore(db_path=tmp_path / "shell.db")
backend = UnshareBackend()
manager = SandboxManager(backend=backend, settings=Settings())
handle = await manager.create("shell-learner")
try:
assert handle.id not in backend._tracked # noqa: SLF001
live = manager._handles[handle.id] # noqa: SLF001
result = await backend.exec(live, ["sh", "-c", "echo plain"])
assert "plain" in result.stdout
finally:
await manager.destroy(handle.id)
store.close()
async def _await_events(
store: SQLiteTraceStore, learner: str, task: str, *, minimum: int, timeout_s: float = 20.0
) -> list:
"""Poll the store until `minimum` events arrive (agent streams async)."""
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
events = store.get_trace(learner, task)
if len(events) >= minimum:
return events
await asyncio.sleep(0.25)
events = store.get_trace(learner, task)
pytest.fail(
f"only {len(events)}/{minimum} telemetry events arrived within {timeout_s}s: "
f"{[(e.seq, e.kind) for e in events]}"
)