nextcraft/apps/ai-service/tests/variants/test_environments.py

"""Design/simulation environment tests (REQ-5-005/006, D-044, G-15).

MH-4a: templates generate kind-tagged variants with correct starter files +
commands; command fields roundtrip the shlex validator; wire response
carries both fields (required); TS types match Python field-for-field
(the dual-schema rule — checked in review by the TS typecheck + here by
the response shape).
MH-4b: exec policy — design/sim kinds reject out-of-policy argv[0] (422
naming the allowed set); sh -c passthrough rejected; build kind unchanged.
MH-4d: design-kind E2E in the real-server harness with concrete
assertions (stored seqs contiguous; digest computes over a design-kind
trace — kind-agnostic by construction, now pinned).
"""

from __future__ import annotations

import pytest
from fastapi.testclient import TestClient

from ai_service.config import Settings
from ai_service.grading.features import TraceDigest, compute_digest
from ai_service.identity.store import SQLiteIdentityStore
from ai_service.llm.mock import MockProvider
from ai_service.main import create_app
from ai_service.telemetry.models import TelemetryEvent
from ai_service.telemetry.store import SQLiteTraceStore
from ai_service.variants.generator import VariantGenerator
from ai_service.variants.store import SQLiteVariantStore
from ai_service.variants.templates import TEMPLATES, validate_simple_argv

from ..conftest import SUITE_LEARNERS, seed_verified_identity

# P1 tier marker (Task 1-1-02): test_design_kind_e2e_real_server (below)
# boots a REAL uvicorn server + REAL namespace sandbox — real_server
# tier, 180s timeout via the conftest collection hook. The remaining
# classes here are in-process TestClient-only.
pytestmark = [pytest.mark.real_server]

DESIGN_LEARNER = "pilot-learner"  # verified 18+ via the suite seed


@pytest.fixture()
def env_client(tmp_path):
    store = SQLiteVariantStore(db_path=tmp_path / "v.db")
    identity = SQLiteIdentityStore(db_path=tmp_path / "i.db")
    seed_verified_identity(identity)
    settings = Settings(
        provider="mock",
        learner_allowlist=SUITE_LEARNERS,
        db_path=tmp_path / "env-app.db",
    )
    app = create_app(settings)
    app.state.identity_store = identity
    app.state.variant_store = store
    app.state.variant_generator = VariantGenerator(store, MockProvider(), model="gemma4:31b")
    with TestClient(app) as c:
        yield c


class TestTemplateRegistry:
    """MH-4a: registry + generator + wire."""

    def test_all_kinds_present(self) -> None:
        kinds = {t.environment for t in TEMPLATES.values()}
        assert kinds == {"build", "design", "simulation"}

    def test_g15_command_roundtrip_validator(self) -> None:
        assert validate_simple_argv("python simulate.py") == "python simulate.py"
        with pytest.raises(ValueError, match="whitespace-joinable"):
            validate_simple_argv('sh -c "echo hi"')
        with pytest.raises(ValueError, match="not be empty"):
            validate_simple_argv("   ")

    def test_design_variant_generates_with_kind_and_files(self, env_client) -> None:
        resp = env_client.post(
            "/v1/variants",
            json={
                "learner_id": DESIGN_LEARNER,
                "template_id": "tpl-conversation-flow-design",
            },
        )
        assert resp.status_code == 200, resp.text
        body = resp.json()
        assert body["environment"] == "design"
        assert body["test_command"] == "python3 validate_flow.py"
        assert "flow.md" in body["starter_files"]
        assert "validate_flow.py" in body["starter_files"]

    def test_simulation_variant_generates_with_kind(self, env_client) -> None:
        resp = env_client.post(
            "/v1/variants",
            json={
                "learner_id": DESIGN_LEARNER,
                "template_id": "tpl-sensor-benchmark",
            },
        )
        assert resp.status_code == 200, resp.text
        body = resp.json()
        assert body["environment"] == "simulation"
        assert body["test_command"] == "pytest -q"
        assert "simulate.py" in body["starter_files"]

    def test_build_variants_default_kind(self, env_client) -> None:
        resp = env_client.post(
            "/v1/variants",
            json={"learner_id": DESIGN_LEARNER, "template_id": "tpl-llm-judge"},
        )
        assert resp.status_code == 200, resp.text
        body = resp.json()
        assert body["environment"] == "build"


class TestExecPolicy:
    """MH-4b: exact-token, per-kind; sh -c disallowed for design/sim."""

    @pytest.fixture()
    def exec_client(self, tmp_path):
        """App + a STUB backend sandbox bound to a DESIGN-kind variant
        (policy check happens before execution — no real namespace needed)."""
        from ai_service.sandbox.backend import ExecResult
        from ai_service.sandbox.manager import SandboxManager
        from tests.api.test_sandboxes import StubBackend

        class ExecStubBackend(StubBackend):
            """StubBackend + a working exec (policy fires BEFORE exec)."""

            async def exec(self, handle, cmd):
                return ExecResult(
                    cmd=list(cmd),
                    returncode=0,
                    stdout="ok",
                    stderr="",
                    duration_s=0.0,
                )

        vstore = SQLiteVariantStore(db_path=tmp_path / "v.db")
        identity = SQLiteIdentityStore(db_path=tmp_path / "i.db")
        seed_verified_identity(identity)
        settings = Settings(
            provider="mock",
            learner_allowlist=SUITE_LEARNERS,
            sandbox_dir=tmp_path / "sandboxes",
            db_path=tmp_path / "exec-app.db",
        )
        app = create_app(settings)
        app.state.identity_store = identity
        app.state.variant_store = vstore
        app.state.variant_generator = VariantGenerator(
            vstore, MockProvider(), model="gemma4:31b"
        )
        stub = ExecStubBackend()
        manager = SandboxManager(backend=stub, settings=settings)
        app.state.sandbox_manager = manager
        with TestClient(app) as c:
            # create a design variant + a sandbox for its task
            var = c.post(
                "/v1/variants",
                json={
                    "learner_id": DESIGN_LEARNER,
                    "template_id": "tpl-conversation-flow-design",
                },
            ).json()
            sbx = c.post(
                "/v1/sandboxes",
                json={"learner_id": DESIGN_LEARNER, "task_id": var["task_id"]},
            ).json()
            c._sandbox_id = sbx["id"]  # P5: private-attr seam the fixture sets; reads stay untyped
            yield c

    def test_design_kind_rejects_out_of_policy_command(self, exec_client) -> None:
        # P5: _sandbox_id was set on the client by the fixture (Any-typed seam).
        sbx = exec_client._sandbox_id
        resp = exec_client.post(f"/v1/sandboxes/{sbx}/exec", json={"cmd": ["rm", "-rf", "/"]})
        assert resp.status_code == 422
        assert "'rm'" in resp.json()["detail"]
        assert "allowed" in resp.json()["detail"]

    def test_design_kind_rejects_shell_passthrough(self, exec_client) -> None:
        # P5: _sandbox_id was set on the client by the fixture (Any-typed seam).
        sbx = exec_client._sandbox_id
        resp = exec_client.post(
            f"/v1/sandboxes/{sbx}/exec", json={"cmd": ["sh", "-c", "anything"]}
        )
        assert resp.status_code == 422
        assert "passthrough" in resp.json()["detail"]

    def test_design_kind_allows_declared_harness(self, exec_client) -> None:
        """The policy passes the declared harness (StubBackend.exec raises
        NotImplementedError by design — any status EXCEPT 422 proves the
        policy allowed the command through)."""
        # P5: _sandbox_id was set on the client by the fixture (Any-typed seam).
        sbx = exec_client._sandbox_id
        resp = exec_client.post(
            f"/v1/sandboxes/{sbx}/exec",
            json={"cmd": ["python3", "validate_flow.py"]},
        )
        assert resp.status_code != 422, resp.text

    def test_build_kind_policy_unchanged(self, tmp_path) -> None:
        from ai_service.api.sandboxes import _enforce_exec_policy

        _enforce_exec_policy(["whatever", "anywhere"], "build")  # no raise
        _enforce_exec_policy(["sh", "-c", "x"], None)  # unknown env: no raise


class TestDigestKindAgnostic:
    """MH-4d (part): compute_digest over a synthetic DESIGN-kind trace —
    the digest derives from event kinds, never environment types."""

    def test_design_trace_digests_like_build_traces(self) -> None:
        """compute_digest(trace) over a synthetic design-kind event stream —
        same feature classes as a build trace: command counts, run results,
        edit cadence. The environment kind never enters the computation."""
        from datetime import UTC, datetime

        ts = datetime(2026, 9, 13, 12, 0, 0, tzinfo=UTC)
        base = {
            "learner_id": "digest-learner",
            "task_id": "task-design-1",
            "sandbox_id": "sbx-design",
        }
        events = [
            TelemetryEvent(
                seq=0,
                kind="command",
                payload={"cmd": "python validate_flow.py"},
                ts=ts,
                **base,
            ),
            TelemetryEvent(
                seq=1,
                kind="file_diff",
                payload={"path": "flow.md", "diff": "+## Turn 2"},
                ts=ts,
                **base,
            ),
            TelemetryEvent(
                seq=2,
                kind="run_result",
                payload={
                    "cmd": "python validate_flow.py",
                    "exit_code": 0,
                    "stdout": "VALID",
                },
                ts=ts,
                **base,
            ),
        ]
        digest = compute_digest(events)
        assert digest.command_count == 1
        assert digest.run_count == 1
        # The digest model has NO environment/kind field — kind-agnostic by
        # construction; assert it stays that way.
        assert "environment" not in TraceDigest.model_fields


@pytest.mark.asyncio
async def test_design_kind_e2e_real_server(tmp_path, sandbox_dir) -> None:
    """MH-4d (G-17 — concrete assertions, no hope-shaped must-haves):
    a design-kind variant → REAL namespace sandbox → starter files →
    harness exec in-ns → telemetry flows → contiguous seq chain stored.
    The ack/trim coverage is the P1 suite's (real agent); here the REAL
    agent runs too — the spool assertion rides the stored contiguity."""
    import asyncio

    from ai_service.sandbox import SandboxManager
    from ai_service.sandbox.unshare_backend import UnshareBackend
    from ai_service.telemetry.ingest import TraceIntegrityMap
    from tests.helpers import free_port, serve, skip_if_no_userns

    skip_if_no_userns()

    store = SQLiteVariantStore(db_path=tmp_path / "v.db")
    trace_store = SQLiteTraceStore(db_path=tmp_path / "t.db")
    identity = SQLiteIdentityStore(db_path=tmp_path / "i.db")
    seed_verified_identity(identity)

    port = free_port()
    settings = Settings(
        provider="mock",
        learner_allowlist=SUITE_LEARNERS,
        db_path=tmp_path / "e2e-app.db",
        sandbox_dir=sandbox_dir,
        port=port,
        telemetry_ingest_host="127.0.0.1",
    )
    app = create_app(settings)
    app.state.identity_store = identity
    app.state.variant_store = store
    app.state.variant_generator = VariantGenerator(
        store, MockProvider(), model="gemma4:31b"
    )
    app.state.trace_store = trace_store
    app.state.trace_integrity = TraceIntegrityMap()
    # v0.4.7 (P1): pre-set BEFORE the lifespan — the manager mints, the WS
    # endpoint verifies; one shared instance.
    from ai_service.telemetry.tokens import IngestToken

    app.state.ingest_token = IngestToken("e2e-secret")
    manager = SandboxManager(
        backend=UnshareBackend(), settings=settings, ingest_token=app.state.ingest_token
    )
    app.state.sandbox_manager = manager

    try:
        async with serve(app, "127.0.0.1", port):
            import httpx

            async with httpx.AsyncClient(
                base_url=f"http://127.0.0.1:{port}", timeout=60.0
            ) as client:
                # 1. design variant (kind-tagged, python3 harness — in-ns PATH)
                var = (
                    await client.post(
                        "/v1/variants",
                        json={
                            "learner_id": "pilot-learner",
                            "template_id": "tpl-conversation-flow-design",
                        },
                    )
                ).json()
                assert var["environment"] == "design"

                # 2. sandbox for the design task
                sbx = (
                    await client.post(
                        "/v1/sandboxes",
                        json={"learner_id": "pilot-learner", "task_id": var["task_id"]},
                    )
                ).json()

                # 3. materialize starter files (the client's job — mirror it)
                for path, content in var["starter_files"].items():
                    resp = await client.put(
                        f"/v1/sandboxes/{sbx['id']}/files/{path}",
                        json={"path": path, "content": content},
                    )
                    assert resp.status_code == 200, resp.text

                # 4. run the design harness IN THE NAMESPACE (python3 resolves)
                run = (
                    await client.post(
                        f"/v1/sandboxes/{sbx['id']}/exec",
                        json={"cmd": ["python3", "validate_flow.py"]},
                    )
                ).json()
                # starter flow.md fails validation on purpose (needs learner edits)
                assert "ISSUES" in run.get("stdout", "") or run.get("returncode") in (0, 1)

                # 5. out-of-policy command is 422 at the exec route (G-15)
                rejected = await client.post(
                    f"/v1/sandboxes/{sbx['id']}/exec",
                    json={"cmd": ["nmap", "-p", "1-1000", "localhost"]},
                )
                assert rejected.status_code == 422

                # 6. telemetry flowed: contiguous seq chain, kind-agnostic.
                #    The in-ns exec + file writes stream through the capture
                #    agent (watcher ~250ms + command events + heartbeats).
                import time as _time

                deadline = _time.monotonic() + 15.0
                events = trace_store.get_trace("pilot-learner", var["task_id"])
                while _time.monotonic() < deadline and len(events) < 2:
                    await asyncio.sleep(0.5)
                    events = trace_store.get_trace("pilot-learner", var["task_id"])
                seqs = [e.seq for e in events]
                assert len(seqs) >= 2, f"no telemetry flowed: {seqs}"
                assert seqs == sorted(seqs), f"out of order: {seqs}"
                assert len(set(seqs)) == len(seqs), f"duplicates: {seqs}"
                assert seqs == list(range(seqs[0], seqs[-1] + 1)), f"gaps: {seqs}"

                await client.delete(f"/v1/sandboxes/{sbx['id']}")
    finally:
        trace_store.close()