CIAgent
1a88ab3199
The v0.8 milestone: installable on any Debian-based server via docker (Gitea registry image, honest sandbox degrade), deb (amd64+arm64, advisory postinst), and arch-aware binary — with release consistency enforced by pre-commit hooks + Gitea Actions (3-runner fleet on CT 113, machine-made releases with the full 8-asset matrix + registry image), the CLI carrying the full suite (guide, channel-aware doctor), and the admin runbook (deploy/README.md, evidence-backed recipes). ---ci--- phase: 6 milestone: v0.8 status: complete requirements: covered: [REQ-8-001, REQ-8-002, REQ-8-003, REQ-8-004, REQ-8-005] partial: [] ---/ci---
476 lines
19 KiB
Python
476 lines
19 KiB
Python
"""/v1/sandboxes API tests — lifecycle, pool guard, G-5 abuse control (REQ-3-001).
|
|
|
|
Every test here runs against `StubBackend` (workdir layout without namespace
|
|
spawning) EXCEPT the final probe-guarded real-backend path; Wave 1/2 already
|
|
covers boundary isolation, so the API layer only needs the manager contract.
|
|
|
|
DI/lifespan wiring note: tests build the manager themselves, pre-set
|
|
`app.state.sandbox_manager` BEFORE the TestClient lifespan runs, and the
|
|
lifespan adopts that instance (state-injection override) instead of
|
|
constructing the real UnshareBackend one. The lifespan still `start()`s it,
|
|
runs the periodic reaper, and `destroy_all()`s on shutdown — the same
|
|
production path, just with a stub at the port.
|
|
|
|
Snapshot/destroy content plumbing for stub tests uses the explicit test seam
|
|
`app.state.sandbox_test_layout` → `X-Snapshot-Copy` / `X-Workspace-Copy`
|
|
response headers (production never sets them).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import time
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from ai_service.config import Settings
|
|
from ai_service.main import create_app
|
|
from ai_service.sandbox import SandboxHandle, SandboxManager, UnshareBackend
|
|
from ai_service.sandbox.backend import ExecResult, SandboxSpec
|
|
from ai_service.sandbox.workdir import SandboxDir, create_layout
|
|
from ai_service.sandbox.workdir import snapshot as workdir_snapshot
|
|
from tests.sandbox.test_isolation import requires_userns
|
|
|
|
PILOT = "pilot-learner"
|
|
OTHER = "pilot-learner-2"
|
|
|
|
BASE_SETTINGS: dict = {
|
|
"provider": "mock",
|
|
"sandbox_max_concurrent": 5,
|
|
"sandbox_max_per_learner": 1,
|
|
"sandbox_creates_per_min": 10,
|
|
# G-9: allowlist widened to the suite roster (identity records seeded
|
|
# per-app below); the gate tests live in test_identity.py.
|
|
"learner_allowlist": __import__("tests.conftest", fromlist=["SUITE_LEARNERS"]).SUITE_LEARNERS,
|
|
}
|
|
|
|
|
|
def _seed_identity(app, tmp_path: Path) -> None:
|
|
"""G-9: every ad-hoc app in this module gets the suite's verified
|
|
identity store (allowlist widened via BASE_SETTINGS)."""
|
|
from ai_service.identity.store import SQLiteIdentityStore
|
|
from tests.conftest import seed_verified_identity
|
|
|
|
store = SQLiteIdentityStore(db_path=tmp_path / f"identity-{id(app):x}.db")
|
|
seed_verified_identity(store)
|
|
app.state.identity_store = store
|
|
|
|
|
|
class StubBackend:
|
|
"""Structural SandboxBackend: lays out the workdir, spawns nothing.
|
|
|
|
Tracks spawn/destroy calls so lifespan shutdown behaviour is assertable.
|
|
`structural = True` exempts it from the v0.8 degrade guard (D-068): the
|
|
stub fabric never touches userns, so it must stay usable inside
|
|
unprivileged CI job containers (D-063 fast lane).
|
|
"""
|
|
|
|
structural = True
|
|
|
|
def __init__(self) -> None:
|
|
self.spawned_specs: list[SandboxSpec] = []
|
|
self.destroyed_ids: list[str] = []
|
|
|
|
async def spawn(self, spec: SandboxSpec) -> SandboxHandle:
|
|
create_layout(spec)
|
|
self.spawned_specs.append(spec)
|
|
return SandboxHandle(
|
|
id=spec.sandbox_id, pid=None, workdir=spec.workdir,
|
|
created_at=datetime.now(UTC),
|
|
)
|
|
|
|
async def exec(self, handle: SandboxHandle, cmd: list[str]) -> ExecResult:
|
|
raise NotImplementedError("API tests never exec")
|
|
|
|
async def snapshot(self, handle: SandboxHandle) -> Path:
|
|
return workdir_snapshot(handle.workdir)
|
|
|
|
async def destroy(self, handle: SandboxHandle) -> None:
|
|
self.destroyed_ids.append(handle.id)
|
|
handle.pid = None
|
|
|
|
|
|
def _seed_workspace(layout: SandboxDir, name: str, content: str) -> None:
|
|
layout.workspace.mkdir(parents=True, exist_ok=True)
|
|
(layout.workspace / name).write_text(content)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clear_create_window() -> Iterator[None]:
|
|
"""Module-global rate window must start empty in every test."""
|
|
from ai_service.api import sandboxes as sandboxes_api
|
|
|
|
sandboxes_api._CREATE_TIMES.clear()
|
|
yield
|
|
sandboxes_api._CREATE_TIMES.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
def stub_backend() -> StubBackend:
|
|
return StubBackend()
|
|
|
|
|
|
@pytest.fixture()
|
|
def client(
|
|
tmp_path: Path, stub_backend: StubBackend, monkeypatch: pytest.MonkeyPatch
|
|
) -> Iterator[TestClient]:
|
|
"""App + adopted stub manager, sandbox dir rooted in a per-test tmp_path."""
|
|
monkeypatch.setenv("AI_SANDBOX_CREATES_PER_MIN", "150") # shared-window headroom
|
|
settings = Settings(**{**BASE_SETTINGS, "sandbox_dir": tmp_path / "sandboxes"})
|
|
app = create_app(settings)
|
|
_seed_identity(app, tmp_path)
|
|
app.state.sandbox_manager = SandboxManager(backend=stub_backend, settings=settings)
|
|
with TestClient(app) as c:
|
|
yield c
|
|
|
|
|
|
# -- lifecycle roundtrip --------------------------------------------------------
|
|
|
|
|
|
def test_create_list_get_snapshot_delete_roundtrip(client: TestClient) -> None:
|
|
created = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert created.status_code == 201
|
|
body = created.json()
|
|
assert body["id"].startswith("sbx-")
|
|
assert body["learner_id"] == PILOT
|
|
assert body["created_at"]
|
|
assert "pid" not in body # host-process detail, excluded from the contract
|
|
sandbox_id = body["id"]
|
|
|
|
listed = client.get("/v1/sandboxes")
|
|
assert listed.status_code == 200
|
|
rows = listed.json()["sandboxes"]
|
|
assert [r["id"] for r in rows] == [sandbox_id]
|
|
assert rows[0]["learner_id"] == PILOT
|
|
|
|
got = client.get(f"/v1/sandboxes/{sandbox_id}")
|
|
assert got.status_code == 200
|
|
assert got.json()["id"] == sandbox_id
|
|
|
|
# Seed workspace content on the host side, then snapshot it away.
|
|
layout = SandboxDir(
|
|
root=Path(body["workdir"]),
|
|
workspace=Path(body["workdir"]) / "workspace",
|
|
snapshots=Path(body["workdir"]) / "snapshots",
|
|
)
|
|
_seed_workspace(layout, "solution.py", "print(42)\n")
|
|
|
|
snap = client.post(f"/v1/sandboxes/{sandbox_id}/snapshot")
|
|
assert snap.status_code == 200
|
|
snap_body = snap.json()
|
|
assert snap_body["sandbox_id"] == sandbox_id
|
|
assert snap_body["files"] == ["solution.py"]
|
|
snapshot_path = Path(snap_body["snapshot_path"])
|
|
assert (snapshot_path / "solution.py").read_text() == "print(42)\n"
|
|
|
|
# Destroy: 204, registry empties, but the workdir (and snapshot) survives.
|
|
deleted = client.delete(f"/v1/sandboxes/{sandbox_id}")
|
|
assert deleted.status_code == 204
|
|
assert deleted.content == b""
|
|
assert client.get("/v1/sandboxes").json()["sandboxes"] == []
|
|
assert client.get(f"/v1/sandboxes/{sandbox_id}").status_code == 404
|
|
assert layout.root.is_dir() # kept for restore; not purged (API semantic)
|
|
assert (snapshot_path / "solution.py").is_file()
|
|
|
|
|
|
def test_create_validates_body_422(client: TestClient) -> None:
|
|
assert client.post("/v1/sandboxes", json={}).status_code == 422
|
|
assert client.post("/v1/sandboxes", json={"learner_id": ""}).status_code == 422
|
|
|
|
|
|
def test_snapshot_unknown_id_404(client: TestClient) -> None:
|
|
response = client.post("/v1/sandboxes/sbx-nope/snapshot")
|
|
assert response.status_code == 404
|
|
assert "sbx-nope" in response.json()["detail"]
|
|
|
|
|
|
def test_delete_unknown_id_404(client: TestClient) -> None:
|
|
response = client.delete("/v1/sandboxes/sbx-nope")
|
|
assert response.status_code == 404
|
|
assert "sbx-nope" in response.json()["detail"]
|
|
|
|
|
|
def test_get_unknown_id_404(client: TestClient) -> None:
|
|
assert client.get("/v1/sandboxes/sbx-nope").status_code == 404
|
|
|
|
|
|
# -- D-032 pool guard → 503 ------------------------------------------------------
|
|
|
|
|
|
def test_pool_full_returns_503(tmp_path: Path, stub_backend: StubBackend) -> None:
|
|
settings = Settings(
|
|
provider="mock",
|
|
sandbox_dir=tmp_path / "sandboxes",
|
|
sandbox_max_concurrent=2,
|
|
sandbox_max_per_learner=5, # cap disabled for this test
|
|
sandbox_creates_per_min=150,
|
|
)
|
|
app = create_app(settings)
|
|
_seed_identity(app, tmp_path)
|
|
app.state.sandbox_manager = SandboxManager(backend=stub_backend, settings=settings)
|
|
with TestClient(app) as client:
|
|
ids = [
|
|
client.post("/v1/sandboxes", json={"learner_id": PILOT}).json()["id"]
|
|
for _ in range(2)
|
|
]
|
|
overflow = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert overflow.status_code == 503
|
|
assert "pool full" in overflow.json()["detail"]
|
|
# Capacity frees on destroy: the next create succeeds.
|
|
client.delete(f"/v1/sandboxes/{ids[0]}")
|
|
refilled = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert refilled.status_code == 201
|
|
|
|
|
|
# -- G-5 abuse control ------------------------------------------------------------
|
|
|
|
|
|
def test_non_allowlisted_learner_403(client: TestClient) -> None:
|
|
response = client.post("/v1/sandboxes", json={"learner_id": "intruder-7"})
|
|
assert response.status_code == 403
|
|
assert "allowlist" in response.json()["detail"]
|
|
assert client.get("/v1/sandboxes").json()["sandboxes"] == [] # nothing spawned
|
|
|
|
|
|
def test_second_active_sandbox_for_same_learner_429(
|
|
tmp_path: Path, stub_backend: StubBackend
|
|
) -> None:
|
|
settings = Settings(
|
|
provider="mock",
|
|
sandbox_dir=tmp_path / "sandboxes",
|
|
learner_allowlist=[PILOT, OTHER],
|
|
sandbox_max_concurrent=10,
|
|
sandbox_max_per_learner=1,
|
|
sandbox_creates_per_min=150,
|
|
)
|
|
app = create_app(settings)
|
|
_seed_identity(app, tmp_path)
|
|
app.state.sandbox_manager = SandboxManager(backend=stub_backend, settings=settings)
|
|
with TestClient(app) as client:
|
|
first = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert first.status_code == 201
|
|
second = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert second.status_code == 429
|
|
assert "per-learner cap" in second.json()["detail"]
|
|
# Cap counts ACTIVE sandboxes: after destroy the learner can create again.
|
|
client.delete(f"/v1/sandboxes/{first.json()['id']}")
|
|
again = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert again.status_code == 201
|
|
# A different allowlisted learner was never blocked by the pilot's sandbox.
|
|
other = client.post("/v1/sandboxes", json={"learner_id": OTHER})
|
|
assert other.status_code == 201
|
|
|
|
|
|
def test_burst_over_global_create_rate_429(tmp_path: Path, stub_backend: StubBackend) -> None:
|
|
settings = Settings(
|
|
provider="mock",
|
|
sandbox_dir=tmp_path / "sandboxes",
|
|
sandbox_max_concurrent=10,
|
|
sandbox_max_per_learner=10, # per-learner cap disabled for this test
|
|
sandbox_creates_per_min=3,
|
|
)
|
|
app = create_app(settings)
|
|
_seed_identity(app, tmp_path)
|
|
manager = SandboxManager(backend=stub_backend, settings=settings)
|
|
app.state.sandbox_manager = manager
|
|
with TestClient(app) as client:
|
|
codes = [
|
|
client.post("/v1/sandboxes", json={"learner_id": PILOT}).status_code
|
|
for _ in range(3)
|
|
]
|
|
assert codes == [201, 201, 201]
|
|
burst = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert burst.status_code == 429
|
|
assert "create rate" in burst.json()["detail"]
|
|
assert manager.active_count == 3 # the 429 spawned nothing
|
|
|
|
# Rejected creates never consume budget; still full one moment later.
|
|
assert client.post("/v1/sandboxes", json={"learner_id": PILOT}).status_code == 429
|
|
|
|
|
|
def test_create_rate_window_slides(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The window is monotonic-time based; fake the clock, skip real sleeping."""
|
|
from fastapi import HTTPException
|
|
|
|
from ai_service.api import sandboxes as sandboxes_api
|
|
|
|
sandboxes_api._CREATE_TIMES.clear()
|
|
fake_now = 1_000.0
|
|
monkeypatch.setattr(time, "monotonic", lambda: fake_now)
|
|
settings = Settings(provider="mock", sandbox_creates_per_min=2)
|
|
sandboxes_api._check_global_create_rate(settings)
|
|
sandboxes_api._check_global_create_rate(settings)
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
sandboxes_api._check_global_create_rate(settings)
|
|
assert excinfo.value.status_code == 429
|
|
|
|
fake_now += 61.0 # window slides: oldest entries age out
|
|
sandboxes_api._check_global_create_rate(settings) # admitted again
|
|
|
|
|
|
# -- lifespan wiring --------------------------------------------------------------
|
|
|
|
|
|
def test_lifespan_start_and_shutdown_destroy(
|
|
tmp_path: Path, stub_backend: StubBackend
|
|
) -> None:
|
|
"""a-1 at boot: an orphan workdir with a dead recorded pid is reaped;
|
|
at shutdown every live sandbox is destroyed (no orphans outlive us)."""
|
|
orphan_root = tmp_path / "sandboxes" / "sbx-orphaned"
|
|
(orphan_root / "workspace").mkdir(parents=True)
|
|
(orphan_root / "sandbox.json").write_text(
|
|
'{"sandbox_id": "sbx-orphaned", "learner_id": "?", "pid": 4194304}'
|
|
)
|
|
settings = Settings(
|
|
provider="mock",
|
|
sandbox_dir=tmp_path / "sandboxes",
|
|
sandbox_creates_per_min=150,
|
|
)
|
|
manager = SandboxManager(backend=stub_backend, settings=settings)
|
|
app = create_app(settings)
|
|
_seed_identity(app, tmp_path)
|
|
app.state.sandbox_manager = manager
|
|
with TestClient(app) as client:
|
|
assert not orphan_root.exists() # startup reaper ran during lifespan boot
|
|
assert any(e.kind == "orphan_reaped" for e in manager.integrity_events)
|
|
created = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
sandbox_id = created.json()["id"]
|
|
assert manager.active_count == 1
|
|
# Exiting the TestClient runs the shutdown half of the lifespan.
|
|
assert stub_backend.destroyed_ids == [sandbox_id]
|
|
assert manager.active_count == 0
|
|
|
|
|
|
def test_lifespan_constructs_real_manager_when_not_overridden(tmp_path: Path) -> None:
|
|
"""No override → the lifespan builds the production UnshareBackend manager."""
|
|
settings = Settings(provider="mock", sandbox_dir=tmp_path / "sandboxes")
|
|
app = create_app(settings)
|
|
_seed_identity(app, tmp_path)
|
|
with TestClient(app):
|
|
manager = app.state.sandbox_manager
|
|
assert isinstance(manager, SandboxManager)
|
|
|
|
|
|
# -- real-backend path (probe-guarded; Wave 1/2 owns deep isolation) --------------
|
|
|
|
#: Repo-anchored, gitignored scratch root (tests/sandboxes/ like isolation
|
|
#: tests) — /tmp is off-limits: the in-namespace tmpfs shadows host /tmp.
|
|
REAL_SANDBOXES_ROOT = Path(__file__).resolve().parents[1] / "sandboxes"
|
|
|
|
|
|
@requires_userns
|
|
def test_real_backend_create_path_runs(tmp_path: Path) -> None:
|
|
"""One cheap real-namespace pass through the API (create only, no exec)."""
|
|
sandbox_root = REAL_SANDBOXES_ROOT / f"api-{tmp_path.name}"
|
|
settings = Settings(
|
|
provider="mock",
|
|
sandbox_dir=sandbox_root,
|
|
sandbox_creates_per_min=150,
|
|
)
|
|
app = create_app(settings) # no override → lifespan wires UnshareBackend
|
|
_seed_identity(app, tmp_path)
|
|
try:
|
|
with TestClient(app) as client:
|
|
created = client.post("/v1/sandboxes", json={"learner_id": PILOT})
|
|
assert created.status_code == 201
|
|
sandbox_id = created.json()["id"]
|
|
assert (sandbox_root / sandbox_id / "workspace").is_dir()
|
|
assert isinstance(app.state.sandbox_manager._backend, UnshareBackend)
|
|
# Shutdown destroyed the live sandbox; the kept workdir still exists.
|
|
assert app.state.sandbox_manager.active_count == 0
|
|
assert (sandbox_root / sandbox_id / "workspace").is_dir()
|
|
finally:
|
|
shutil.rmtree(sandbox_root, ignore_errors=True)
|
|
|
|
|
|
class TestFilesAndExecRoutes:
|
|
"""Workspace CRUD + Run/Test exec (Phase 6, REQ-3-008, CUT-2)."""
|
|
|
|
def test_file_write_read_list_roundtrip(self, client):
|
|
handle = client.post(
|
|
"/v1/sandboxes", json={"learner_id": "pilot-learner"}
|
|
).json()
|
|
sbx = handle["id"]
|
|
put = client.put(
|
|
f"/v1/sandboxes/{sbx}/files/main.py",
|
|
json={"path": "main.py", "content": "print('hi')"},
|
|
)
|
|
assert put.status_code == 200, put.text
|
|
got = client.get(f"/v1/sandboxes/{sbx}/files/main.py")
|
|
assert got.status_code == 200
|
|
assert "print('hi')" in got.json()["content"]
|
|
listed = client.get(f"/v1/sandboxes/{sbx}/files")
|
|
assert "main.py" in listed.json()["files"]
|
|
client.delete(f"/v1/sandboxes/{sbx}")
|
|
|
|
def test_traversal_rejected(self, client):
|
|
handle = client.post(
|
|
"/v1/sandboxes", json={"learner_id": "pilot-learner"}
|
|
).json()
|
|
sbx = handle["id"]
|
|
bad = client.put(
|
|
f"/v1/sandboxes/{sbx}/files/..%2Fescape.txt",
|
|
json={"path": "../escape.txt", "content": "x"},
|
|
)
|
|
assert bad.status_code == 422
|
|
client.delete(f"/v1/sandboxes/{sbx}")
|
|
|
|
def test_unknown_file_404(self, client):
|
|
handle = client.post(
|
|
"/v1/sandboxes", json={"learner_id": "pilot-learner"}
|
|
).json()
|
|
sbx = handle["id"]
|
|
assert client.get(f"/v1/sandboxes/{sbx}/files/ghost.py").status_code == 404
|
|
client.delete(f"/v1/sandboxes/{sbx}")
|
|
|
|
def test_exec_unknown_sandbox_404(self, client):
|
|
resp = client.post(
|
|
"/v1/sandboxes/sbx-nope/exec", json={"cmd": ["echo", "hi"]}
|
|
)
|
|
assert resp.status_code == 404
|
|
|
|
def test_unknown_sandbox_file_routes_404_not_500(self, client):
|
|
"""P7: read/write on an unknown sandbox must 404 (SandboxNotFoundError
|
|
previously escaped _workspace_dir as an unhandled 500)."""
|
|
assert (
|
|
client.get("/v1/sandboxes/sbx-nope/files/whatever.py").status_code == 404
|
|
)
|
|
put = client.put(
|
|
"/v1/sandboxes/sbx-nope/files/whatever.py",
|
|
json={"path": "whatever.py", "content": "x"},
|
|
)
|
|
assert put.status_code == 404
|
|
|
|
def test_symlink_escape_rejected(self, client):
|
|
"""P7: an exec-planted symlink in the workspace must not let the
|
|
file routes read/write OUTSIDE the bind (lexical traversal checks
|
|
cannot see symlinks — resolve + containment re-check is the gate)."""
|
|
handle = client.post(
|
|
"/v1/sandboxes", json={"learner_id": "pilot-learner"}
|
|
).json()
|
|
sbx = handle["id"]
|
|
workspace = Path(handle["workdir"]) / "workspace"
|
|
outside = workspace.parent / "secret.txt"
|
|
outside.write_text("host secret") # a host file OUTSIDE the bind
|
|
try:
|
|
(workspace / "leak.txt").symlink_to(outside)
|
|
read = client.get(f"/v1/sandboxes/{sbx}/files/leak.txt")
|
|
assert read.status_code == 422, (
|
|
f"symlink escape read must 422, got {read.status_code}: {read.text}"
|
|
)
|
|
write = client.put(
|
|
f"/v1/sandboxes/{sbx}/files/leak.txt",
|
|
json={"path": "leak.txt", "content": "pwned"},
|
|
)
|
|
assert write.status_code == 422, (
|
|
f"symlink escape write must 422, got {write.status_code}: {write.text}"
|
|
)
|
|
assert outside.read_text() == "host secret" # untouched
|
|
finally:
|
|
client.delete(f"/v1/sandboxes/{sbx}")
|
|
outside.unlink(missing_ok=True)
|