docs(P05): complete proctor-mentor-agents phase
---ci--- phase: 5 milestone: v0.2 status: complete ---/ci--- REQ-2-009/010 complete. All six agents live: Coach, Tutor, Lab, Assessor, Proctor, Mentor. Proctor returns structured integrity signals + coaching interventions; Mentor streams session-backed career narratives. 131/131 tests, ruff clean.
This commit is contained in:
CIAgent
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"phase": 4,
|
||||
"phase": 5,
|
||||
"stage": "verify",
|
||||
"milestone": "v0.2",
|
||||
"phase_role": "execution",
|
||||
"attempts": 0,
|
||||
"updated_at": "2026-09-11T19:50:00Z"
|
||||
"updated_at": "2026-09-11T21:10:00Z"
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
"""MentorAgent — long-horizon career narrative (REQ-2-010).
|
||||
|
||||
Streaming, session-backed conversational agent: the learner can ask
|
||||
follow-up questions about their trajectory and the Mentor keeps context.
|
||||
"""
|
||||
|
||||
from ..corpus.learner_context import LearnerContext, get_learner_context
|
||||
from ..llm.types import Message
|
||||
from ..prompts.mentor import SYSTEM_PROMPT, render_context
|
||||
from .base import BaseAgent
|
||||
|
||||
|
||||
class MentorAgent(BaseAgent):
|
||||
name = "mentor"
|
||||
|
||||
def system_prompt(self, learner_context: LearnerContext | None = None) -> str:
|
||||
ctx = learner_context or get_learner_context()
|
||||
return SYSTEM_PROMPT.format_map(render_context(ctx))
|
||||
|
||||
def build_messages(
|
||||
self,
|
||||
history: list[Message] | None = None,
|
||||
user_input: str = "",
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> list[Message]:
|
||||
return super().build_messages(history, user_input, learner_context)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""ProctorAgent — integrity signals + coaching interventions (REQ-2-009).
|
||||
|
||||
Consumes a ProctorScenario from the corpus, returns pydantic-validated
|
||||
signal classifications via structured_reply (4-layer defense).
|
||||
Mock engine inputs; real identity/attention signals are v0.3+.
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..corpus.learner_context import LearnerContext, get_learner_context
|
||||
from ..corpus.telemetry import ProctorScenario, summarize_proctor_scenario
|
||||
from ..llm.types import Message
|
||||
from ..prompts.proctor import SYSTEM_PROMPT, render_context
|
||||
from .base import BaseAgent
|
||||
|
||||
|
||||
class IntegritySignal(BaseModel):
|
||||
signal_type: str # e.g. "context_switch" | "idle_gap" | "large_paste"
|
||||
severity: str # "low" | "medium" | "high"
|
||||
note: str
|
||||
|
||||
|
||||
class ProctorAssessment(BaseModel):
|
||||
scenario_id: str
|
||||
signals: list[IntegritySignal] = Field(min_length=0)
|
||||
intervention: str # ONE supportive coaching recommendation
|
||||
summary: str
|
||||
|
||||
|
||||
PROCTOR_ASSESSMENT_SCHEMA_HINT = (
|
||||
'{"scenario_id": "<id>", "signals": [{"signal_type": "<type>", '
|
||||
'"severity": "low"|"medium"|"high", "note": "<one sentence>"}], '
|
||||
'"intervention": "<one supportive recommendation>", '
|
||||
'"summary": "<one sentence>"}'
|
||||
)
|
||||
|
||||
|
||||
class ProctorAgent(BaseAgent):
|
||||
name = "proctor"
|
||||
|
||||
def system_prompt(self, learner_context: LearnerContext | None = None) -> str:
|
||||
ctx = learner_context or get_learner_context()
|
||||
return SYSTEM_PROMPT.format_map(render_context(ctx))
|
||||
|
||||
async def assess(
|
||||
self,
|
||||
scenario: ProctorScenario,
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> ProctorAssessment:
|
||||
timeline = summarize_proctor_scenario(scenario)
|
||||
assessment: ProctorAssessment = await self.structured_reply(
|
||||
history=None,
|
||||
user_input=timeline,
|
||||
learner_context=learner_context,
|
||||
schema=ProctorAssessment,
|
||||
schema_hint=PROCTOR_ASSESSMENT_SCHEMA_HINT,
|
||||
)
|
||||
return assessment
|
||||
|
||||
def build_messages(
|
||||
self,
|
||||
history: list[Message] | None = None,
|
||||
user_input: str = "",
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> list[Message]:
|
||||
return super().build_messages(history, user_input, learner_context)
|
||||
@@ -21,6 +21,8 @@ def register_builtin_agents(registry: "AgentRegistry") -> None:
|
||||
from .assessor import AssessorAgent
|
||||
from .coach import CoachAgent
|
||||
from .lab import LabAgent
|
||||
from .mentor import MentorAgent
|
||||
from .proctor import ProctorAgent
|
||||
from .tutor import TutorAgent
|
||||
|
||||
registry.register("coach", lambda provider, settings: CoachAgent(provider, settings))
|
||||
@@ -29,6 +31,12 @@ def register_builtin_agents(registry: "AgentRegistry") -> None:
|
||||
registry.register(
|
||||
"assessor", lambda provider, settings: AssessorAgent(provider, settings)
|
||||
)
|
||||
registry.register(
|
||||
"proctor", lambda provider, settings: ProctorAgent(provider, settings)
|
||||
)
|
||||
registry.register(
|
||||
"mentor", lambda provider, settings: MentorAgent(provider, settings)
|
||||
)
|
||||
|
||||
|
||||
class UnknownAgentError(KeyError):
|
||||
|
||||
@@ -6,5 +6,13 @@ Boundary rule: api/ composes agents/ and llm/; they never import api/.
|
||||
from .assessment import router as assessment_router
|
||||
from .chat import router as chat_router
|
||||
from .lab import router as lab_router
|
||||
from .mentor import router as mentor_router
|
||||
from .proctor import router as proctor_router
|
||||
|
||||
__all__ = ["assessment_router", "chat_router", "lab_router"]
|
||||
__all__ = [
|
||||
"assessment_router",
|
||||
"chat_router",
|
||||
"lab_router",
|
||||
"mentor_router",
|
||||
"proctor_router",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""POST /v1/mentor/narrative — SSE career narrative stream (REQ-2-010).
|
||||
|
||||
D-016 envelope with agent=mentor. Session-backed: the client supplies a
|
||||
session_id; the Mentor keeps conversation context across follow-ups.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from ..agents.registry import AgentRegistry, UnknownAgentError
|
||||
from ..agents.session import SessionStore
|
||||
from ..config import Settings
|
||||
from ..corpus.learner_context import get_learner_context
|
||||
from ..llm.types import Message
|
||||
from .deps import get_agent_registry, get_provider, get_session_store, get_settings
|
||||
|
||||
router = APIRouter(prefix="/v1")
|
||||
|
||||
|
||||
class MentorNarrativeRequest(BaseModel):
|
||||
session_id: str = Field(min_length=1)
|
||||
prompt: str = Field(default="Narrate my trajectory.")
|
||||
learner_id: str | None = None
|
||||
|
||||
|
||||
@router.post("/mentor/narrative")
|
||||
async def mentor_narrative(
|
||||
body: MentorNarrativeRequest,
|
||||
registry: AgentRegistry = Depends(get_agent_registry),
|
||||
sessions: SessionStore = Depends(get_session_store),
|
||||
settings: Settings = Depends(get_settings),
|
||||
provider=Depends(get_provider),
|
||||
) -> EventSourceResponse:
|
||||
try:
|
||||
agent = registry.get(provider, settings, "mentor")
|
||||
except UnknownAgentError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from None
|
||||
|
||||
learner_context = get_learner_context(body.learner_id)
|
||||
|
||||
session = await sessions.get(body.session_id)
|
||||
if session is None:
|
||||
session = await sessions.create(
|
||||
body.session_id, agent="mentor", learner_id=body.learner_id or "learner-001"
|
||||
)
|
||||
history = await sessions.history_window(body.session_id)
|
||||
user_message = Message(role="user", content=body.prompt)
|
||||
await sessions.append(body.session_id, user_message)
|
||||
|
||||
async def event_stream() -> AsyncIterator[dict]:
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "meta",
|
||||
"agent": "mentor",
|
||||
"session_id": body.session_id,
|
||||
"model": settings.model,
|
||||
})}
|
||||
first_byte = True
|
||||
reply_parts: list[str] = []
|
||||
try:
|
||||
async for token in agent.stream_reply(
|
||||
history=history,
|
||||
user_input=body.prompt,
|
||||
learner_context=learner_context,
|
||||
):
|
||||
first_byte = False
|
||||
reply_parts.append(token)
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "delta", "content": token
|
||||
})}
|
||||
full_reply = "".join(reply_parts)
|
||||
if full_reply:
|
||||
await sessions.append(
|
||||
body.session_id, Message(role="assistant", content=full_reply)
|
||||
)
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "done", "finish_reason": "stop"
|
||||
})}
|
||||
except Exception as exc:
|
||||
code = "provider_unavailable" if first_byte else "provider_error"
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "error", "code": code, "message": str(exc)
|
||||
})}
|
||||
finally:
|
||||
yield {"event": "message", "data": "[DONE]"}
|
||||
|
||||
return EventSourceResponse(
|
||||
event_stream(),
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""POST /v1/proctor/signals — structured integrity signals (REQ-2-009).
|
||||
|
||||
JSON response (not SSE): a pydantic-validated ProctorAssessment.
|
||||
Unknown scenario → 404. Coaching-shaped interventions only.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..agents.proctor import ProctorAssessment
|
||||
from ..agents.registry import AgentRegistry
|
||||
from ..config import Settings
|
||||
from ..corpus.learner_context import get_learner_context
|
||||
from ..corpus.telemetry import get_proctor_scenario
|
||||
from .deps import get_agent_registry, get_provider, get_settings
|
||||
|
||||
router = APIRouter(prefix="/v1")
|
||||
|
||||
|
||||
class ProctorRequest(BaseModel):
|
||||
scenario_id: str = Field(min_length=1)
|
||||
learner_id: str | None = None
|
||||
|
||||
|
||||
@router.post("/proctor/signals", response_model=ProctorAssessment)
|
||||
async def proctor_signals(
|
||||
body: ProctorRequest,
|
||||
registry: AgentRegistry = Depends(get_agent_registry),
|
||||
settings: Settings = Depends(get_settings),
|
||||
provider=Depends(get_provider),
|
||||
) -> ProctorAssessment:
|
||||
scenario = get_proctor_scenario(body.scenario_id)
|
||||
if scenario is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"unknown scenario {body.scenario_id!r}"
|
||||
)
|
||||
agent = registry.get(provider, settings, "proctor")
|
||||
learner_context = get_learner_context(body.learner_id)
|
||||
try:
|
||||
return await agent.assess(scenario, learner_context)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"proctor assessment failed: {exc}"
|
||||
) from exc
|
||||
@@ -24,6 +24,80 @@ class LabTelemetryScenario(BaseModel):
|
||||
events: list[TelemetryEvent]
|
||||
|
||||
|
||||
class ProctorEvent(BaseModel):
|
||||
timestamp: int # seconds since session start
|
||||
kind: str # "tab_switch" | "idle" | "paste_large" | "focus_lost" | "keystroke_burst"
|
||||
detail: str = ""
|
||||
|
||||
|
||||
class ProctorScenario(BaseModel):
|
||||
scenario_id: str
|
||||
title: str
|
||||
competency_id: str
|
||||
events: list[ProctorEvent]
|
||||
|
||||
|
||||
_PROCTOR_SCENARIO_HEALTHY = ProctorScenario(
|
||||
scenario_id="proctor-scenario-healthy",
|
||||
title="Healthy defense session — focused throughout",
|
||||
competency_id="stack-orchestration-c002",
|
||||
events=[
|
||||
ProctorEvent(timestamp=0, kind="keystroke_burst", detail="session begins"),
|
||||
ProctorEvent(timestamp=310, kind="keystroke_burst", detail="long answer in progress"),
|
||||
ProctorEvent(timestamp=640, kind="keystroke_burst", detail="revision pass"),
|
||||
ProctorEvent(timestamp=900, kind="keystroke_burst", detail="final answer"),
|
||||
],
|
||||
)
|
||||
|
||||
_PROCTOR_SCENARIO_DISTRACTED = ProctorScenario(
|
||||
scenario_id="proctor-scenario-distracted",
|
||||
title="Distracted defense session — tab switches and idle gaps",
|
||||
competency_id="stack-orchestration-c002",
|
||||
events=[
|
||||
ProctorEvent(timestamp=0, kind="keystroke_burst", detail="session begins"),
|
||||
ProctorEvent(timestamp=120, kind="tab_switch", detail="to docs.nextjs.org"),
|
||||
ProctorEvent(timestamp=125, kind="focus_lost", detail="window blur 40s"),
|
||||
ProctorEvent(timestamp=300, kind="idle", detail="no activity for 5 minutes"),
|
||||
ProctorEvent(timestamp=600, kind="tab_switch", detail="to github.com"),
|
||||
ProctorEvent(timestamp=605, kind="focus_lost", detail="window blur 2m"),
|
||||
ProctorEvent(timestamp=720, kind="keystroke_burst", detail="resumes typing"),
|
||||
],
|
||||
)
|
||||
|
||||
_PROCTOR_SCENARIO_FLAGGED = ProctorScenario(
|
||||
scenario_id="proctor-scenario-flagged",
|
||||
title="Flagged defense session — large paste during exam",
|
||||
competency_id="stack-orchestration-c003",
|
||||
events=[
|
||||
ProctorEvent(timestamp=0, kind="keystroke_burst", detail="short intro typed"),
|
||||
ProctorEvent(timestamp=85, kind="paste_large", detail="3,100 chars pasted in 2s"),
|
||||
ProctorEvent(timestamp=90, kind="idle", detail="no activity for 4 minutes"),
|
||||
ProctorEvent(timestamp=330, kind="paste_large", detail="2,800 chars pasted in 2s"),
|
||||
],
|
||||
)
|
||||
|
||||
PROCTOR_SCENARIOS: dict[str, ProctorScenario] = {
|
||||
s.scenario_id: s
|
||||
for s in (
|
||||
_PROCTOR_SCENARIO_HEALTHY,
|
||||
_PROCTOR_SCENARIO_DISTRACTED,
|
||||
_PROCTOR_SCENARIO_FLAGGED,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def get_proctor_scenario(scenario_id: str) -> ProctorScenario | None:
|
||||
return PROCTOR_SCENARIOS.get(scenario_id)
|
||||
|
||||
|
||||
def summarize_proctor_scenario(scenario: ProctorScenario) -> str:
|
||||
"""Render the proctor event timeline as compact text for prompt injection."""
|
||||
lines = [f"Defense session: {scenario.title} (competency {scenario.competency_id})"]
|
||||
for event in scenario.events:
|
||||
lines.append(f"t+{event.timestamp}s {event.kind}: {event.detail}".rstrip(": "))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
_LAB_SCENARIO_STRONG = LabTelemetryScenario(
|
||||
scenario_id="lab-scenario-strong",
|
||||
title="Strong build session — multi-agent research assistant",
|
||||
|
||||
@@ -8,7 +8,13 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .agents.registry import AgentRegistry, register_builtin_agents
|
||||
from .agents.session import InMemorySessionStore
|
||||
from .api import assessment_router, chat_router, lab_router
|
||||
from .api import (
|
||||
assessment_router,
|
||||
chat_router,
|
||||
lab_router,
|
||||
mentor_router,
|
||||
proctor_router,
|
||||
)
|
||||
from .config import Settings
|
||||
from .llm import create_provider
|
||||
|
||||
@@ -51,6 +57,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.include_router(chat_router)
|
||||
app.include_router(lab_router)
|
||||
app.include_router(assessment_router)
|
||||
app.include_router(mentor_router)
|
||||
app.include_router(proctor_router)
|
||||
return app
|
||||
|
||||
|
||||
|
||||
@@ -1,25 +1,44 @@
|
||||
"""Mentor agent prompt — long-horizon career narrative (REQ-2-010).
|
||||
|
||||
Persona: wise career guide. Connects today's competencies to a long-horizon
|
||||
trajectory in AI-era roles. Versioned: v1 draft (Phase 2); final in Phase 5.
|
||||
Final persona (Phase 5). Mentor is a wise career guide: connects today's
|
||||
competencies and artifacts to a long-horizon AI-era trajectory.
|
||||
Version: mentor-v2 (final for v0.2).
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Mentor, the long-horizon career agent of an AI-native competency school.
|
||||
Learner: {learner_name}. Active stack: {stacks}. Progress: {progress}.
|
||||
Microcredentials earned: {microcredentials}. Recent artifacts: {artifacts}.
|
||||
Your job: narrate the learner's trajectory — where they are now, what their
|
||||
competency progress unlocks next, and how their artifacts position them in
|
||||
the AI-era labor market. Two to three paragraphs, forward-looking, concrete."""
|
||||
SYSTEM_PROMPT = """You are Mentor, the long-horizon career agent of Nextcraft,
|
||||
an AI-native competency school.
|
||||
|
||||
PROMPT_VERSION = "mentor-v1-draft"
|
||||
Learner: {learner_name}
|
||||
Active stacks: {stacks}
|
||||
Current focus: {progress}
|
||||
Microcredentials earned: {microcredentials}
|
||||
Recent artifacts: {artifacts}
|
||||
|
||||
Your job: narrate the learner's trajectory in two to three paragraphs:
|
||||
1. Where they are now — what their competency progress and artifacts say
|
||||
about them as a builder (specific, evidence-based).
|
||||
2. What their current stack unlocks next — name the next competency or
|
||||
microcredential worth chasing and the role it points toward.
|
||||
3. How they position in the AI-era labor market — which employer problems
|
||||
their profile already answers.
|
||||
|
||||
Rules:
|
||||
- Forward-looking and concrete. No fortune-telling, no flattery.
|
||||
- Reference their artifacts by name at least once.
|
||||
- Write like a mentor writing to one person, not a career-services brochure."""
|
||||
|
||||
PROMPT_VERSION = "mentor-v2"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
|
||||
in_progress = [
|
||||
c for c in learner_context.active_competencies if c.status == "in_progress"
|
||||
]
|
||||
progress = (
|
||||
f"{learner_context.active_competencies[0].title} in progress"
|
||||
if learner_context.active_competencies
|
||||
else "no active competencies"
|
||||
f"{in_progress[0].title} ({in_progress[0].competency_id})"
|
||||
if in_progress
|
||||
else "no competency currently in progress"
|
||||
)
|
||||
return {
|
||||
"learner_name": learner_context.name,
|
||||
|
||||
@@ -1,19 +1,34 @@
|
||||
"""Proctor agent prompt — integrity signals with coaching interventions (REQ-2-009).
|
||||
|
||||
Persona: supportive observer, not punitive. Classifies integrity signals from
|
||||
telemetry and recommends coaching interventions. Versioned: v1 draft (Phase 2);
|
||||
final persona + signal models in Phase 5.
|
||||
Final persona (Phase 5). Proctor is a supportive observer, never punitive:
|
||||
classifies signals, recommends ONE coaching intervention. Assume good
|
||||
faith — most signals have innocent explanations.
|
||||
Version: proctor-v2 (final for v0.2).
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Proctor, the integrity-support agent of an AI-native competency school.
|
||||
SYSTEM_PROMPT = """You are Proctor, the integrity-support agent of Nextcraft,
|
||||
an AI-native competency school.
|
||||
Learner: {learner_name}.
|
||||
You receive a telemetry timeline of session events (focus, tab switches,
|
||||
paste events, idle time). Your job: classify each signal by type and severity,
|
||||
then recommend ONE supportive coaching intervention — never punitive, never
|
||||
accusatory. Assume good faith; most signals have innocent explanations.
|
||||
Respond with ONLY valid JSON matching the provided signals schema."""
|
||||
|
||||
PROMPT_VERSION = "proctor-v1-draft"
|
||||
You receive a telemetry timeline of defense-session events (tab switches,
|
||||
idle gaps, large pastes, focus loss, keystroke bursts). Your job:
|
||||
- Classify EACH notable signal: type (e.g. "context_switch", "idle_gap",
|
||||
"large_paste"), severity ("low" | "medium" | "high"), and a one-sentence
|
||||
note citing the event (timestamps and details).
|
||||
- Recommend exactly ONE supportive coaching intervention for the session
|
||||
overall — never punitive, never accusatory. Frame around helping the
|
||||
learner succeed, e.g. "offer a short break", "invite them to explain
|
||||
the pasted section in their own words".
|
||||
|
||||
Rules:
|
||||
- Assume good faith. Tab switches to documentation are normal engineering.
|
||||
- Idle gaps are often thinking. Only unusual patterns deserve higher severity.
|
||||
- A large paste during an assessment deserves "high" severity but the
|
||||
intervention stays coaching-shaped: verification, not punishment.
|
||||
- Respond with ONLY a valid JSON object matching the provided schema —
|
||||
no markdown fences, no prose outside the JSON."""
|
||||
|
||||
PROMPT_VERSION = "proctor-v2"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Mentor agent tests — career narrative, session-backed (REQ-2-010)."""
|
||||
|
||||
from ai_service.agents.mentor import MentorAgent
|
||||
from ai_service.config import Settings
|
||||
from ai_service.corpus.learner_context import get_learner_context
|
||||
from ai_service.llm.mock import MockProvider
|
||||
from ai_service.llm.types import Message
|
||||
|
||||
|
||||
def make_mentor() -> MentorAgent:
|
||||
return MentorAgent(MockProvider(), Settings(provider="mock"))
|
||||
|
||||
|
||||
def test_system_prompt_carries_full_learner_context():
|
||||
prompt = make_mentor().system_prompt(get_learner_context())
|
||||
assert "Alex Rivera" in prompt
|
||||
assert "AI Orchestration Engineer (62%)" in prompt
|
||||
assert "Multi-agent research assistant" in prompt # artifacts by name
|
||||
assert "4" in prompt # microcredential count
|
||||
|
||||
|
||||
def test_build_messages_system_history_user():
|
||||
mentor = make_mentor()
|
||||
history = [Message(role="user", content="what next?"),
|
||||
Message(role="assistant", content="trajectory...")]
|
||||
messages = mentor.build_messages(history, "tell me more", get_learner_context())
|
||||
assert messages[0].role == "system"
|
||||
assert "Mentor" in messages[0].content
|
||||
assert [m.content for m in messages[1:]] == ["what next?", "trajectory...", "tell me more"]
|
||||
|
||||
|
||||
async def test_stream_reply_mentions_learner_context_in_output():
|
||||
"""Mock-scripted: narrative derives from context-injected messages —
|
||||
different learner contexts produce distinct (deterministic) replies."""
|
||||
mentor = make_mentor()
|
||||
alex = get_learner_context("learner-001")
|
||||
priya = get_learner_context("learner-002")
|
||||
alex_reply = "".join(
|
||||
[t async for t in mentor.stream_reply(user_input="narrate", learner_context=alex)]
|
||||
)
|
||||
priya_reply = "".join(
|
||||
[t async for t in mentor.stream_reply(user_input="narrate", learner_context=priya)]
|
||||
)
|
||||
assert alex_reply
|
||||
assert alex_reply != priya_reply
|
||||
|
||||
|
||||
def test_agent_name():
|
||||
assert make_mentor().name == "mentor"
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Proctor agent tests — structured integrity signals (REQ-2-009)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from ai_service.agents.proctor import ProctorAgent, ProctorAssessment
|
||||
from ai_service.agents.structured import StructuredOutputError
|
||||
from ai_service.config import Settings
|
||||
from ai_service.corpus.learner_context import get_learner_context
|
||||
from ai_service.corpus.telemetry import (
|
||||
PROCTOR_SCENARIOS,
|
||||
get_proctor_scenario,
|
||||
summarize_proctor_scenario,
|
||||
)
|
||||
from ai_service.llm.mock import MockProvider, ScriptedJSONProvider
|
||||
|
||||
VALID_ASSESSMENT = {
|
||||
"scenario_id": "proctor-scenario-distracted",
|
||||
"signals": [
|
||||
{"signal_type": "context_switch", "severity": "low",
|
||||
"note": "Tab switch to docs at t+120s — normal engineering behavior"},
|
||||
{"signal_type": "idle_gap", "severity": "medium",
|
||||
"note": "5-minute idle at t+300s followed by more tab switches"},
|
||||
],
|
||||
"intervention": "Offer a short break and ask the learner to restate their answer plan",
|
||||
"summary": "Distracted but explainable session; coach the focus pattern, don't flag it",
|
||||
}
|
||||
|
||||
|
||||
def make_proctor(provider=None) -> ProctorAgent:
|
||||
return ProctorAgent(provider or MockProvider(), Settings(provider="mock"))
|
||||
|
||||
|
||||
def test_proctor_scenarios_addressable_and_distinct_type():
|
||||
healthy = get_proctor_scenario("proctor-scenario-healthy")
|
||||
flagged = get_proctor_scenario("proctor-scenario-flagged")
|
||||
assert healthy is not None and flagged is not None
|
||||
kinds = lambda s: {e.kind for e in s.events} # noqa: E731
|
||||
assert "tab_switch" in kinds(get_proctor_scenario("proctor-scenario-distracted"))
|
||||
assert "paste_large" in kinds(flagged)
|
||||
assert not kinds(healthy) & {"tab_switch", "paste_large", "focus_lost"}
|
||||
|
||||
|
||||
def test_unknown_proctor_scenario_none():
|
||||
assert get_proctor_scenario("proctor-scenario-ghost") is None
|
||||
|
||||
|
||||
async def test_assess_returns_validated_signals():
|
||||
provider = ScriptedJSONProvider(VALID_ASSESSMENT)
|
||||
proctor = make_proctor(provider)
|
||||
scenario = get_proctor_scenario("proctor-scenario-distracted")
|
||||
result = await proctor.assess(scenario, get_learner_context())
|
||||
assert isinstance(result, ProctorAssessment)
|
||||
assert len(result.signals) == 2
|
||||
assert result.signals[0].severity == "low"
|
||||
assert "break" in result.intervention.lower()
|
||||
|
||||
|
||||
async def test_assess_rejects_invalid_after_retry():
|
||||
proctor = make_proctor(MockProvider()) # non-schema JSON
|
||||
scenario = get_proctor_scenario("proctor-scenario-healthy")
|
||||
with pytest.raises(StructuredOutputError):
|
||||
await proctor.assess(scenario, get_learner_context())
|
||||
|
||||
|
||||
def test_system_prompt_is_coaching_not_punitive():
|
||||
prompt = make_proctor().system_prompt(get_learner_context())
|
||||
assert "never punitive" in prompt.lower()
|
||||
assert "good faith" in prompt.lower()
|
||||
assert "ONLY" in prompt # JSON-only instruction
|
||||
|
||||
|
||||
def test_timeline_summary_carries_events():
|
||||
scenario = get_proctor_scenario("proctor-scenario-flagged")
|
||||
text = summarize_proctor_scenario(scenario)
|
||||
assert "paste_large" in text
|
||||
assert "3,100 chars" in text
|
||||
|
||||
|
||||
def test_all_three_proctor_scenarios_exist():
|
||||
assert set(PROCTOR_SCENARIOS) == {
|
||||
"proctor-scenario-healthy",
|
||||
"proctor-scenario-distracted",
|
||||
"proctor-scenario-flagged",
|
||||
}
|
||||
@@ -83,6 +83,47 @@ def test_lab_and_assessor_resolve_via_registry():
|
||||
assert isinstance(assessor, AssessorAgent)
|
||||
|
||||
|
||||
def test_proctor_and_mentor_resolve_via_registry():
|
||||
"""Phase 5: proctor + mentor registered centrally (Tasks 5-1-02/5-2-01)."""
|
||||
from ai_service.agents.mentor import MentorAgent
|
||||
from ai_service.agents.proctor import ProctorAgent
|
||||
|
||||
registry = AgentRegistry()
|
||||
register_builtin_agents(registry)
|
||||
settings = Settings(provider="mock")
|
||||
proctor = registry.get(MockProvider(), settings, "proctor")
|
||||
mentor = registry.get(MockProvider(), settings, "mentor")
|
||||
assert isinstance(proctor, ProctorAgent)
|
||||
assert isinstance(mentor, MentorAgent)
|
||||
|
||||
|
||||
def test_registry_resolves_all_six_agents():
|
||||
"""Must-Have (Phase 5): the full roster — coach/tutor/lab/assessor/proctor/mentor."""
|
||||
from ai_service.agents.assessor import AssessorAgent
|
||||
from ai_service.agents.coach import CoachAgent
|
||||
from ai_service.agents.lab import LabAgent
|
||||
from ai_service.agents.mentor import MentorAgent
|
||||
from ai_service.agents.proctor import ProctorAgent
|
||||
from ai_service.agents.tutor import TutorAgent
|
||||
|
||||
registry = AgentRegistry()
|
||||
register_builtin_agents(registry)
|
||||
assert registry.names() == ["assessor", "coach", "lab", "mentor", "proctor", "tutor"]
|
||||
settings = Settings(provider="mock")
|
||||
expected = {
|
||||
"coach": CoachAgent,
|
||||
"tutor": TutorAgent,
|
||||
"lab": LabAgent,
|
||||
"assessor": AssessorAgent,
|
||||
"proctor": ProctorAgent,
|
||||
"mentor": MentorAgent,
|
||||
}
|
||||
for name, cls in expected.items():
|
||||
agent = registry.get(MockProvider(), settings, name)
|
||||
assert isinstance(agent, cls), f"{name} resolved to {type(agent).__name__}"
|
||||
assert agent.name == name
|
||||
|
||||
|
||||
def test_builtin_registration_is_idempotent_safe():
|
||||
"""Duplicate registration raises — builtin bootstrap must be called once."""
|
||||
registry = AgentRegistry()
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Mentor narrative endpoint tests — SSE, session-backed (REQ-2-010)."""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def stream_events(client, payload) -> list[dict]:
|
||||
with client.stream("POST", "/v1/mentor/narrative", json=payload) as response:
|
||||
assert response.status_code == 200
|
||||
events = []
|
||||
for line in response.iter_lines():
|
||||
if line.startswith("data:"):
|
||||
d = line.removeprefix("data:").strip()
|
||||
if d == "[DONE]":
|
||||
events.append({"type": "[DONE]"})
|
||||
else:
|
||||
events.append(json.loads(d))
|
||||
return events
|
||||
|
||||
|
||||
def test_narrative_streams_full_envelope(client):
|
||||
events = stream_events(client, {"session_id": "mentor-1"})
|
||||
assert events[0]["type"] == "meta"
|
||||
assert events[0]["agent"] == "mentor"
|
||||
deltas = [e for e in events if e["type"] == "delta"]
|
||||
assert len(deltas) >= 1
|
||||
assert any(e["type"] == "done" for e in events)
|
||||
assert events[-1]["type"] == "[DONE]"
|
||||
|
||||
|
||||
def test_narrative_is_session_backed(client):
|
||||
"""Second call replays history: provider input grows; distinct mock output."""
|
||||
first = stream_events(client, {"session_id": "mentor-2", "prompt": "narrate my path"})
|
||||
second = stream_events(client, {"session_id": "mentor-2", "prompt": "what next?"})
|
||||
first_text = "".join(e["content"] for e in first if e["type"] == "delta")
|
||||
second_text = "".join(e["content"] for e in second if e["type"] == "delta")
|
||||
assert first_text != second_text
|
||||
|
||||
|
||||
def test_narrative_persists_turns(client):
|
||||
import asyncio
|
||||
|
||||
store = client.app.state.session_store
|
||||
|
||||
stream_events(client, {"session_id": "mentor-3", "prompt": "hello trajectory"})
|
||||
|
||||
async def check():
|
||||
return await store.history_window("mentor-3")
|
||||
|
||||
contents = [m.content for m in asyncio.run(check())]
|
||||
assert "hello trajectory" in contents
|
||||
assert len(contents) >= 2 # user + assistant persisted
|
||||
|
||||
|
||||
def test_missing_session_id_422(client):
|
||||
response = client.post("/v1/mentor/narrative", json={"prompt": "hi"})
|
||||
assert response.status_code == 422
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Proctor signals endpoint tests — validated JSON, 404s (REQ-2-009)."""
|
||||
|
||||
from ai_service.agents.proctor import ProctorAssessment
|
||||
from ai_service.llm.mock import ScriptedJSONProvider
|
||||
|
||||
VALID = {
|
||||
"scenario_id": "proctor-scenario-distracted",
|
||||
"signals": [
|
||||
{"signal_type": "context_switch", "severity": "low",
|
||||
"note": "Docs tab at t+120s is normal"},
|
||||
{"signal_type": "idle_gap", "severity": "medium",
|
||||
"note": "5-minute idle at t+300s"},
|
||||
],
|
||||
"intervention": "Offer a short break, then restate the plan",
|
||||
"summary": "Coaching-shaped session note",
|
||||
}
|
||||
|
||||
|
||||
def test_signals_returns_validated_json(client):
|
||||
original = client.app.state.provider
|
||||
client.app.state.provider = ScriptedJSONProvider(VALID)
|
||||
try:
|
||||
response = client.post(
|
||||
"/v1/proctor/signals", json={"scenario_id": "proctor-scenario-distracted"}
|
||||
)
|
||||
finally:
|
||||
client.app.state.provider = original
|
||||
assert response.status_code == 200
|
||||
validated = ProctorAssessment.model_validate(response.json())
|
||||
assert validated.scenario_id == "proctor-scenario-distracted"
|
||||
assert validated.intervention
|
||||
|
||||
|
||||
def test_unknown_scenario_404(client):
|
||||
response = client.post("/v1/proctor/signals", json={"scenario_id": "ghost"})
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_unparseable_provider_502(client):
|
||||
response = client.post(
|
||||
"/v1/proctor/signals", json={"scenario_id": "proctor-scenario-healthy"}
|
||||
)
|
||||
assert response.status_code == 502
|
||||
assert "failed" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_missing_scenario_id_422(client):
|
||||
response = client.post("/v1/proctor/signals", json={})
|
||||
assert response.status_code == 422
|
||||
Reference in New Issue
Block a user