nextcraft/apps/ai-service/tests/agents/test_examiner.py

"""Examiner agent tests (Task 5-2-01, REQ-3-006)."""

from __future__ import annotations

import json

import pytest

from ai_service.agents.examiner import DefenseVerdict, ExaminerAgent
from ai_service.agents.registry import AgentRegistry, register_builtin_agents
from ai_service.config import Settings
from ai_service.grading.features import compute_digest
from ai_service.llm.mock import MockProvider
from ai_service.llm.types import Message

VERDICT_JSON = json.dumps(
    {
        "verdict": "developing",
        "understanding": "Explains the retry loop clearly.",
        "process_justification": "Justifies the edit-then-test cadence from the digest.",
        "communication": "Answers are specific and on-topic.",
        "strengths": ["Grounded the fix in a failed test."],
        "gaps": ["Did not justify the chunk-size choice."],
    }
)


class RecordingProvider(MockProvider):
    """Mock provider that records every message list (prompt assertions)."""

    def __init__(self, replies: list[str] | None = None) -> None:
        super().__init__()
        self.replies = list(replies or [])
        self.requests: list[list[Message]] = []

    async def chat(self, messages, *, model, temperature=0.7, response_format=None):
        self.requests.append([Message(role=m.role, content=m.content) for m in messages])
        if self.replies:
            return self.replies.pop(0)
        return "Tell me about your build."


def _digest():
    from datetime import UTC, datetime, timedelta

    from ai_service.telemetry.models import TelemetryEvent

    t0 = datetime(2026, 9, 12, tzinfo=UTC)
    events = [
        TelemetryEvent(
            learner_id="examiner-learner",
            task_id="examiner-task",
            seq=n,
            kind=kind,
            payload=payload,
            ts=t0 + timedelta(seconds=n * 10),
            sandbox_id="sbx-examiner",
        )
        for n, (kind, payload) in enumerate(
            [
                ("file_diff", {"path": "a.py"}),
                ("command", {"cmd": "pytest -q"}),
                ("test_result", {"passed": False, "exit_code": 1}),
                ("file_diff", {"path": "a.py"}),
                ("test_result", {"passed": True, "exit_code": 0}),
            ]
        )
    ]
    return compute_digest(events)


@pytest.fixture()
def provider() -> RecordingProvider:
    return RecordingProvider()


@pytest.fixture()
def settings() -> Settings:
    return Settings(provider="mock")


@pytest.fixture()
def examiner(provider, settings) -> ExaminerAgent:
    return ExaminerAgent(provider, settings)


class TestNextQuestion:
    async def test_prompt_contains_digest_but_no_learner_id(
        self, examiner, provider
    ) -> None:
        await examiner.next_question(
            history=[Message(role="assistant", content="First question?")],
            trace_digest=_digest(),
            variant_statement="Build a chunker.",
        )
        all_content = "\n".join(
            m.content for request in provider.requests for m in request
        )
        assert "error_fix_cycles" in all_content  # digest JSON grounded
        assert "examiner-learner" not in all_content  # D-028 anonymity
        assert "Build a chunker." in all_content  # variant statement grounded
        assert all_content.count('"examiner-learner"') == 0

    async def test_question_returned_from_provider(self, examiner) -> None:
        question = await examiner.next_question(
            history=[], trace_digest=_digest()
        )
        assert isinstance(question, str)


class TestFinalVerdict:
    async def test_verdict_validates_via_d020(self, examiner, provider) -> None:
        provider.replies = [VERDICT_JSON]
        verdict = await examiner.final_verdict(
            history=[Message(role="assistant", content="Q?")],
            trace_digest=_digest(),
        )
        assert isinstance(verdict, DefenseVerdict)
        assert verdict.verdict == "developing"
        assert verdict.strengths and verdict.gaps

    async def test_malformed_then_good_exercises_retry(self, examiner, provider) -> None:
        provider.replies = ["not json", VERDICT_JSON]
        verdict = await examiner.final_verdict(history=[], trace_digest=_digest())
        assert verdict.verdict == "developing"
        assert len(provider.requests) == 2  # D-020 bounded retry


class TestRegistry:
    def test_all_seven_agents_resolve(self, provider, settings) -> None:
        registry = AgentRegistry()
        register_builtin_agents(registry)
        assert registry.names() == [
            "assessor",
            "coach",
            "examiner",
            "lab",
            "mentor",
            "proctor",
            "tutor",
        ]
        agent = registry.get(provider, settings, "examiner")
        assert isinstance(agent, ExaminerAgent)


class TestBoundary:
    def test_examiner_never_imports_voice_or_api(self) -> None:
        import ast
        from pathlib import Path

        py = Path(__file__).parents[2] / "ai_service" / "agents" / "examiner.py"
        tree = ast.parse(py.read_text())
        for node in ast.walk(tree):
            if isinstance(node, ast.ImportFrom) and node.module:
                assert "voice" not in node.module, "examiner must not import voice/"
                assert not node.module.startswith("ai_service.api")
            if isinstance(node, ast.Import):
                for alias in node.names:
                    assert alias.name != "fastapi"