docs(P02): complete agent-framework phase

---ci---
phase: 2
milestone: v0.2
status: complete
---/ci---

REQ-2-004 complete. BaseAgent ABC, agent-scoped sessions (20-msg window,
500-cap LRU), registry, 4-layer structured output defense, 6-module
prompt library, D-021-aligned learner corpus, chat session persistence.
61/61 tests, ruff clean.
This commit is contained in:
CIAgent
2026-09-11 15:52:00 +00:00
parent dda9569b80
commit 4c52d29f91
25 changed files with 1235 additions and 5 deletions
+2 -2
View File
@@ -1,8 +1,8 @@
{
"phase": 1,
"phase": 2,
"stage": "verify",
"milestone": "v0.2",
"phase_role": "execution",
"attempts": 0,
"updated_at": "2026-09-11T16:45:00Z"
"updated_at": "2026-09-11T17:35:00Z"
}
@@ -0,0 +1,18 @@
"""Agent framework — BaseAgent ABC (D-018), registry, sessions, structured outputs.
Boundary rule: agents/ imports from llm/, prompts/, corpus/ — never from api/.
"""
from .base import BaseAgent
from .registry import AgentRegistry
from .session import InMemorySessionStore, SessionStore
from .structured import StructuredOutputError, extract_json_object
__all__ = [
"AgentRegistry",
"BaseAgent",
"InMemorySessionStore",
"SessionStore",
"StructuredOutputError",
"extract_json_object",
]
+77
View File
@@ -0,0 +1,77 @@
"""BaseAgent ABC — the contract all six tutor agents implement (D-018).
Subclasses set `name`, override `system_prompt()`, and rarely `stream_reply()`.
The default pipeline: build_messages() → provider.stream_chat()/chat().
"""
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from pydantic import BaseModel
from ..config import Settings
from ..corpus.learner_context import LearnerContext
from ..llm.base import LLMProvider
from ..llm.types import Message
from .structured import structured_completion
class BaseAgent(ABC):
"""A tutor agent: system prompt + message assembly + provider delegation."""
name: str = "base"
def __init__(self, provider: LLMProvider, settings: Settings) -> None:
self.provider = provider
self.settings = settings
@abstractmethod
def system_prompt(self, learner_context: LearnerContext | None = None) -> str:
"""Return the agent's system prompt, learner-context-aware."""
def build_messages(
self,
history: list[Message] | None = None,
user_input: str = "",
learner_context: LearnerContext | None = None,
) -> list[Message]:
"""Compose the full message list: system prompt + history + user turn."""
messages: list[Message] = [
Message(role="system", content=self.system_prompt(learner_context))
]
for m in history or []:
messages.append(m)
if user_input:
messages.append(Message(role="user", content=user_input))
return messages
async def stream_reply(
self,
history: list[Message] | None = None,
user_input: str = "",
learner_context: LearnerContext | None = None,
response_format: dict | None = None,
) -> AsyncIterator[str]:
"""Stream incremental content deltas for a conversational reply."""
messages = self.build_messages(history, user_input, learner_context)
async for token in self.provider.stream_chat(
messages, model=self.settings.model, response_format=response_format
):
yield token
async def structured_reply(
self,
history: list[Message] | None = None,
user_input: str = "",
learner_context: LearnerContext | None = None,
schema: type[BaseModel] | None = None,
schema_hint: str = "",
) -> BaseModel:
"""Non-streaming completion parsed into a pydantic model (D-020 defense)."""
if schema is None:
raise ValueError("structured_reply requires a schema")
messages = self.build_messages(history, user_input, learner_context)
return await structured_completion(
self.provider, messages, model=self.settings.model,
schema=schema, schema_hint=schema_hint,
)
@@ -0,0 +1,43 @@
"""Agent registry — explicit name → agent factory map (D-018, G-4).
Agents are registered centrally in their own phases (P3-P5) via
`registry.register(name, factory)`. One registration pattern, one registry.
"""
from collections.abc import Callable
from ..config import Settings
from ..llm.base import LLMProvider
from .base import BaseAgent
AgentFactory = Callable[[LLMProvider, Settings], BaseAgent]
class UnknownAgentError(KeyError):
"""Raised when resolving an agent name that was never registered."""
class DuplicateAgentError(ValueError):
"""Raised when registering an agent name that already exists."""
class AgentRegistry:
def __init__(self) -> None:
self._factories: dict[str, AgentFactory] = {}
def register(self, name: str, factory: AgentFactory) -> None:
if name in self._factories:
raise DuplicateAgentError(f"agent {name!r} already registered")
self._factories[name] = factory
def names(self) -> list[str]:
return sorted(self._factories)
def get(self, provider: LLMProvider, settings: Settings, name: str) -> BaseAgent:
try:
factory = self._factories[name]
except KeyError:
raise UnknownAgentError(
f"unknown agent {name!r}; registered: {self.names()}"
) from None
return factory(provider, settings)
@@ -0,0 +1,94 @@
"""SessionStore — protocol + in-memory implementation (D-019).
Protocol is DB-migration-ready (A-003): swap InMemorySessionStore for a
Redis/PG-backed implementation without touching the API layer.
Sessions are agent-scoped: switching agents starts a new session ID (avoids
persona bleed, A-007). History windowing happens here (last N messages),
controlling token growth per session.
"""
import asyncio
from collections import OrderedDict
from dataclasses import dataclass, field
from typing import Protocol
from ..llm.types import Message
DEFAULT_WINDOW = 20
DEFAULT_MAX_SESSIONS = 500
@dataclass
class AgentSession:
session_id: str
agent: str
learner_id: str = "seed-learner-1"
messages: list[Message] = field(default_factory=list)
class SessionStore(Protocol):
def get(self, session_id: str) -> AgentSession | None: ...
def create(
self, session_id: str, agent: str, learner_id: str = "seed-learner-1"
) -> AgentSession: ...
def append(self, session_id: str, message: Message) -> None: ...
def history_window(
self, session_id: str, max_messages: int = DEFAULT_WINDOW
) -> list[Message]: ...
def delete(self, session_id: str) -> None: ...
class InMemorySessionStore:
"""asyncio.Lock-guarded dict with 20-message windows and 500-cap LRU eviction."""
def __init__(
self,
window: int = DEFAULT_WINDOW,
max_sessions: int = DEFAULT_MAX_SESSIONS,
) -> None:
self._sessions: OrderedDict[str, AgentSession] = OrderedDict()
self._lock = asyncio.Lock()
self._window = window
self._max_sessions = max_sessions
async def get(self, session_id: str) -> AgentSession | None:
async with self._lock:
session = self._sessions.get(session_id)
if session is not None:
self._sessions.move_to_end(session_id) # LRU touch
return session
async def create(
self, session_id: str, agent: str, learner_id: str = "seed-learner-1"
) -> AgentSession:
async with self._lock:
session = AgentSession(session_id=session_id, agent=agent, learner_id=learner_id)
self._sessions[session_id] = session
self._evict_locked()
return session
async def append(self, session_id: str, message: Message) -> None:
async with self._lock:
session = self._sessions.get(session_id)
if session is None:
raise KeyError(f"unknown session {session_id!r}")
session.messages.append(message)
self._sessions.move_to_end(session_id)
async def history_window(
self, session_id: str, max_messages: int = DEFAULT_WINDOW
) -> list[Message]:
async with self._lock:
session = self._sessions.get(session_id)
if session is None:
raise KeyError(f"unknown session {session_id!r}")
return list(session.messages[-max_messages:])
async def delete(self, session_id: str) -> None:
async with self._lock:
self._sessions.pop(session_id, None)
def _evict_locked(self) -> None:
while len(self._sessions) > self._max_sessions:
self._sessions.popitem(last=False) # evict least-recently-used
@@ -0,0 +1,117 @@
"""Structured output defense — 4 layers (D-020).
Layer 1: response_format={"type":"json_object"} request (auto-degrades on 400
inside the provider).
Layer 2: prompt-embedded schema hint ("Respond with ONLY valid JSON...").
Layer 3: defensive parse — strip markdown fences, extract first balanced
JSON object, pydantic model_validate.
Layer 4: single bounded retry with the validation error fed back.
"""
from typing import TypeVar
from pydantic import BaseModel, ValidationError
from ..llm.base import LLMProvider
from ..llm.types import Message
T = TypeVar("T", bound=BaseModel)
class StructuredOutputError(Exception):
"""Raised when the model output cannot be validated after one retry."""
def extract_json_object(text: str) -> str:
"""Strip fences and return the first balanced {...} block from text."""
stripped = text.strip()
if stripped.startswith("```"):
first_newline = stripped.find("\n")
if first_newline != -1:
stripped = stripped[first_newline + 1:]
if stripped.rstrip().endswith("```"):
stripped = stripped.rstrip()[:-3]
stripped = stripped.strip()
start = stripped.find("{")
if start == -1:
raise StructuredOutputError("no JSON object found in model output")
depth = 0
in_string = False
escape = False
for i, ch in enumerate(stripped[start:], start=start):
if escape:
escape = False
continue
if ch == "\\":
escape = True
continue
if ch == '"' and not escape:
in_string = not in_string
continue
if in_string:
continue
if ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
return stripped[start:i + 1]
raise StructuredOutputError("unbalanced JSON object in model output")
def parse_structured(text: str, schema: type[T]) -> T:
"""Layer 3: fence-strip + first-balanced-object + pydantic validation."""
candidate = extract_json_object(text)
try:
return schema.model_validate_json(candidate)
except ValidationError as exc:
raise StructuredOutputError(f"schema validation failed: {exc}") from exc
def schema_instruction(schema_hint: str) -> str:
"""Layer 2: prompt-side schema text."""
return (
"Respond with ONLY a valid JSON object matching this schema — "
"no markdown fences, no prose outside the JSON. "
f"Schema: {schema_hint}"
)
async def structured_completion(
provider: LLMProvider,
messages: list[Message],
*,
model: str,
schema: type[T],
schema_hint: str,
retry_feedback: str | None = None,
) -> T:
"""Full 4-layer pipeline. One bounded retry (layer 4), then raise."""
# Build request: append schema instruction to the last user message (layer 2).
request = list(messages)
last_user = next((m for m in reversed(request) if m.role == "user"), None)
if last_user is not None:
request = [
Message(role=m.role, content=(m.content + "\n\n" + schema_instruction(schema_hint)))
if m is last_user else m
for m in request
]
response_format = {"type": "json_object"}
raw = await provider.chat(request, model=model, response_format=response_format)
try:
return parse_structured(raw, schema) # layers 1+2+3
except StructuredOutputError as exc:
# Layer 4: single bounded retry with error feedback
retry_prompt = (
f"Your previous response was invalid: {exc}. "
f"Return ONLY the corrected JSON matching: {schema_hint}"
)
request2 = list(messages)
request2.append(Message(role="user", content=retry_prompt))
raw2 = await provider.chat(request2, model=model, response_format=response_format)
try:
return parse_structured(raw2, schema)
except StructuredOutputError as exc2:
raise StructuredOutputError(
f"structured output failed after retry: {exc2}"
) from exc2
+20 -2
View File
@@ -12,10 +12,11 @@ from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sse_starlette.sse import EventSourceResponse
from ..agents.session import SessionStore
from ..config import Settings
from ..llm.base import LLMProvider
from ..llm.types import Message
from .deps import get_provider, get_settings
from .deps import get_provider, get_session_store, get_settings
router = APIRouter(prefix="/v1")
@@ -31,10 +32,19 @@ async def chat_stream(
body: ChatStreamRequest,
provider: LLMProvider = Depends(get_provider),
settings: Settings = Depends(get_settings),
sessions: SessionStore = Depends(get_session_store),
) -> EventSourceResponse:
if not body.messages:
raise HTTPException(status_code=422, detail="messages must not be empty")
# Agent-scoped session (A-007/G-4): persisted turn history, windowed replay.
session = await sessions.get(body.session_id)
if session is None:
session = await sessions.create(body.session_id, agent=body.agent)
history = await sessions.history_window(body.session_id)
# Persist this turn's user message before streaming.
await sessions.append(body.session_id, body.messages[-1])
async def event_stream() -> AsyncIterator[dict]:
yield {"event": "message", "data": json.dumps({
"type": "meta",
@@ -43,14 +53,22 @@ async def chat_stream(
"model": settings.model,
})}
first_byte = True
reply_parts: list[str] = []
try:
async for token in provider.stream_chat(
body.messages, model=settings.model
body.messages if not history else history + body.messages,
model=settings.model,
):
first_byte = False
reply_parts.append(token)
yield {"event": "message", "data": json.dumps({
"type": "delta", "content": token
})}
full_reply = "".join(reply_parts)
if full_reply:
await sessions.append(
body.session_id, Message(role="assistant", content=full_reply)
)
yield {"event": "message", "data": json.dumps({
"type": "done", "finish_reason": "stop"
})}
+11 -1
View File
@@ -1,7 +1,9 @@
"""FastAPI dependencies — provider and settings via app.state (DI)."""
"""FastAPI dependencies — provider, settings, sessions, agents via app.state (DI)."""
from fastapi import Request
from ..agents.registry import AgentRegistry
from ..agents.session import SessionStore
from ..config import Settings
from ..llm.base import LLMProvider
@@ -12,3 +14,11 @@ def get_settings(request: Request) -> Settings:
def get_provider(request: Request) -> LLMProvider:
return request.app.state.provider
def get_session_store(request: Request) -> SessionStore:
return request.app.state.session_store
def get_agent_registry(request: Request) -> AgentRegistry:
return request.app.state.agent_registry
@@ -0,0 +1,11 @@
"""Mock engine inputs — pydantic-typed corpus (D-021).
Convention-aligned with the TS `packages/mock-data` layer: identical ID
strings (stack-*, comp-*, learner-*, art-*, mc-*), cross-referenced by the
counterpart files. No codegen in v0.2 — alignment is by documented
convention; revisit codegen only if drift bites (v0.3).
"""
from .learner_context import LEARNER_CONTEXTS, LearnerContext, get_learner_context
__all__ = ["LEARNER_CONTEXTS", "LearnerContext", "get_learner_context"]
@@ -0,0 +1,108 @@
"""Learner context corpus — pydantic mirror of TS learner-progress.ts (D-021).
Counterpart: packages/mock-data/src/learner-progress.ts (or learner-progress.ts
at package root). IDs are string-identical: learner-001, stack-orchestration,
stack-safety, stack-orchestration-c00N, art-*, mc-*.
"""
from pydantic import BaseModel
class CompetencyProgress(BaseModel):
competency_id: str
title: str
status: str # "mastered" | "in_progress" | "not_started"
class StackProgress(BaseModel):
stack_id: str
title: str
percent: int
class LearnerContext(BaseModel):
learner_id: str
name: str
active_stacks: list[StackProgress]
active_competencies: list[CompetencyProgress]
microcredential_count: int
recent_artifacts: list[str] # artifact names
_STACK_ORCHESTRATION = StackProgress(
stack_id="stack-orchestration", title="AI Orchestration Engineer", percent=62
)
_STACK_SAFETY = StackProgress(
stack_id="stack-safety", title="AI Safety & Governance Lead", percent=41
)
_LEARNER_1 = LearnerContext(
learner_id="learner-001",
name="Alex Rivera",
active_stacks=[_STACK_ORCHESTRATION, _STACK_SAFETY],
active_competencies=[
CompetencyProgress(
competency_id="stack-orchestration-c001",
title="Agent architecture fundamentals",
status="mastered",
),
CompetencyProgress(
competency_id="stack-orchestration-c002",
title="Multi-agent communication patterns",
status="in_progress",
),
CompetencyProgress(
competency_id="stack-orchestration-c003",
title="Tool use and function calling",
status="in_progress",
),
CompetencyProgress(
competency_id="stack-safety-c021",
title="Red-team basics for agent systems",
status="in_progress",
),
],
microcredential_count=4,
recent_artifacts=[
"Multi-agent research assistant",
"RAG retrieval quality dashboard",
],
)
_LEARNER_2 = LearnerContext(
learner_id="learner-002",
name="Priya Chen",
active_stacks=[
StackProgress(
stack_id="stack-designer", title="Human-AI Product Designer", percent=55
),
],
active_competencies=[
CompetencyProgress(
competency_id="stack-designer-c001",
title="Prompt-to-prototype workflows",
status="mastered",
),
CompetencyProgress(
competency_id="stack-designer-c002",
title="Evaluating AI UX patterns",
status="in_progress",
),
],
microcredential_count=2,
recent_artifacts=["AI onboarding flow concept test"],
)
LEARNER_CONTEXTS: dict[str, LearnerContext] = {
_LEARNER_1.learner_id: _LEARNER_1,
_LEARNER_2.learner_id: _LEARNER_2,
}
DEFAULT_LEARNER_ID = "learner-001"
def get_learner_context(learner_id: str | None = None) -> LearnerContext:
"""Resolve a learner context by ID, falling back to the default seed."""
if learner_id is None:
return LEARNER_CONTEXTS[DEFAULT_LEARNER_ID]
return LEARNER_CONTEXTS.get(learner_id, LEARNER_CONTEXTS[DEFAULT_LEARNER_ID])
+4
View File
@@ -6,6 +6,8 @@ import httpx
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from .agents.registry import AgentRegistry
from .agents.session import InMemorySessionStore
from .api import chat_router
from .config import Settings
from .llm import create_provider
@@ -21,6 +23,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
app.state.http_client = httpx.AsyncClient(timeout=timeout)
app.state.settings = settings
app.state.provider = create_provider(settings, app.state.http_client)
app.state.session_store = InMemorySessionStore()
app.state.agent_registry = AgentRegistry()
yield
await app.state.http_client.aclose()
@@ -0,0 +1,22 @@
"""Prompt library — prompts are code: versioned in git, reviewed like code (D-018).
Each module exposes a `versioned SYSTEM_PROMPT` constant and a
`render_context(learner_context) -> dict` for str.format_map injection.
Final personas land in Phases 3-5; these are the initial drafts.
"""
from .coach import SYSTEM_PROMPT as COACH_PROMPT
from .coach import render_context as render_coach
from .mentor import SYSTEM_PROMPT as MENTOR_PROMPT
from .mentor import render_context as render_mentor
from .tutor import SYSTEM_PROMPT as TUTOR_PROMPT
from .tutor import render_context as render_tutor
__all__ = [
"COACH_PROMPT",
"MENTOR_PROMPT",
"TUTOR_PROMPT",
"render_coach",
"render_mentor",
"render_tutor",
]
@@ -0,0 +1,19 @@
"""Assessor agent prompt — rubric application to artifacts and defenses (REQ-2-008).
Persona: rigorous, fair grader. Applies the rubric to the artifact and defense
transcript, returns structured JSON scores. Versioned: v1 draft (Phase 2);
final persona + rubric models in Phase 4.
"""
SYSTEM_PROMPT = """You are Assessor, the grading agent of an AI-native competency school.
Learner: {learner_name}.
You receive an artifact, its defense transcript, and a rubric. Your job:
score each rubric criterion with evidence from the artifact and transcript.
Be rigorous but fair — cite what the learner did, not what they should have
done. Respond with ONLY valid JSON matching the provided rubric schema."""
PROMPT_VERSION = "assessor-v1-draft"
def render_context(learner_context) -> dict:
return {"learner_name": learner_context.name}
@@ -0,0 +1,29 @@
"""Coach agent prompt — pacing, motivation, retrieval practice (REQ-2-005).
Persona: warm, action-oriented, accountability partner. Asks for commitments,
uses retrieval practice, keeps momentum. Versioned: v1 draft (Phase 2);
final persona in Phase 3.
"""
SYSTEM_PROMPT = """You are Coach, the pacing and motivation agent of an AI-native competency school.
Learner: {learner_name}. Active stack: {stacks}. Progress: {progress}.
Your job: keep the learner moving. Pace their next step, motivate without fluff,
and weave in retrieval practice — ask them to recall or apply something they
already covered before introducing new material. Be concise, warm, and direct.
End with exactly one clear next action."""
PROMPT_VERSION = "coach-v1-draft"
def render_context(learner_context) -> dict:
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
progress = (
f"{learner_context.active_competencies[0].title} in progress"
if learner_context.active_competencies
else "no active competencies"
)
return {
"learner_name": learner_context.name,
"stacks": stacks or "none yet",
"progress": progress,
}
+23
View File
@@ -0,0 +1,23 @@
"""Lab agent prompt — in-flow feedback over sandbox telemetry (REQ-2-007).
Persona: pragmatic build partner. Reads the telemetry timeline and gives
concrete in-flow feedback: what happened, what to adjust, next step.
Versioned: v1 draft (Phase 2); final persona + scenario serialization in Phase 4.
"""
SYSTEM_PROMPT = """You are Lab, the in-flow feedback agent watching a learner build in the sandbox.
Learner: {learner_name}. Active stack: {stacks}.
You receive a telemetry timeline of the learner's build session. Your job:
describe what the telemetry shows, name the single most useful adjustment,
and give one concrete next step. Be specific to the events you see —
no generic advice. Three short paragraphs maximum."""
PROMPT_VERSION = "lab-v1-draft"
def render_context(learner_context) -> dict:
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
return {
"learner_name": learner_context.name,
"stacks": stacks or "none yet",
}
@@ -0,0 +1,30 @@
"""Mentor agent prompt — long-horizon career narrative (REQ-2-010).
Persona: wise career guide. Connects today's competencies to a long-horizon
trajectory in AI-era roles. Versioned: v1 draft (Phase 2); final in Phase 5.
"""
SYSTEM_PROMPT = """You are Mentor, the long-horizon career agent of an AI-native competency school.
Learner: {learner_name}. Active stack: {stacks}. Progress: {progress}.
Microcredentials earned: {microcredentials}. Recent artifacts: {artifacts}.
Your job: narrate the learner's trajectory — where they are now, what their
competency progress unlocks next, and how their artifacts position them in
the AI-era labor market. Two to three paragraphs, forward-looking, concrete."""
PROMPT_VERSION = "mentor-v1-draft"
def render_context(learner_context) -> dict:
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
progress = (
f"{learner_context.active_competencies[0].title} in progress"
if learner_context.active_competencies
else "no active competencies"
)
return {
"learner_name": learner_context.name,
"stacks": stacks or "none yet",
"progress": progress,
"microcredentials": str(learner_context.microcredential_count),
"artifacts": ", ".join(learner_context.recent_artifacts) or "none yet",
}
@@ -0,0 +1,20 @@
"""Proctor agent prompt — integrity signals with coaching interventions (REQ-2-009).
Persona: supportive observer, not punitive. Classifies integrity signals from
telemetry and recommends coaching interventions. Versioned: v1 draft (Phase 2);
final persona + signal models in Phase 5.
"""
SYSTEM_PROMPT = """You are Proctor, the integrity-support agent of an AI-native competency school.
Learner: {learner_name}.
You receive a telemetry timeline of session events (focus, tab switches,
paste events, idle time). Your job: classify each signal by type and severity,
then recommend ONE supportive coaching intervention — never punitive, never
accusatory. Assume good faith; most signals have innocent explanations.
Respond with ONLY valid JSON matching the provided signals schema."""
PROMPT_VERSION = "proctor-v1-draft"
def render_context(learner_context) -> dict:
return {"learner_name": learner_context.name}
@@ -0,0 +1,28 @@
"""Tutor agent prompt — concept delivery, Socratic questioning (REQ-2-006).
Persona: patient expert teacher. Delivers one concept at a time, checks
understanding with Socratic questions, uses worked examples.
Versioned: v1 draft (Phase 2); final persona in Phase 3.
"""
SYSTEM_PROMPT = """You are Tutor, the concept-delivery agent of an AI-native competency school.
Learner: {learner_name}. Active stack: {stacks}. Progress: {progress}.
Your job: teach concepts clearly, one at a time. Prefer Socratic questioning —
guide the learner to the insight with a worked example, then ask one question
that checks understanding before moving on. Never dump long walls of text."""
PROMPT_VERSION = "tutor-v1-draft"
def render_context(learner_context) -> dict:
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
progress = (
f"{learner_context.active_competencies[0].title} in progress"
if learner_context.active_competencies
else "no active competencies"
)
return {
"learner_name": learner_context.name,
"stacks": stacks or "none yet",
"progress": progress,
}
+55
View File
@@ -0,0 +1,55 @@
"""BaseAgent contract tests — stub agent + mock provider."""
from ai_service.agents.base import BaseAgent
from ai_service.config import Settings
from ai_service.corpus.learner_context import get_learner_context
from ai_service.llm.mock import MockProvider
from ai_service.llm.types import Message
class StubAgent(BaseAgent):
name = "stub"
def system_prompt(self, learner_context=None) -> str:
return "You are Stub. Answer briefly."
def make_agent() -> StubAgent:
return StubAgent(MockProvider(), Settings(provider="mock"))
async def test_build_messages_composition():
agent = make_agent()
history = [Message(role="user", content="earlier"), Message(role="assistant", content="reply")]
messages = agent.build_messages(history, "new question")
assert messages[0].role == "system"
assert messages[0].content == "You are Stub. Answer briefly."
assert [m.content for m in messages[1:]] == ["earlier", "reply", "new question"]
async def test_stream_reply_yields_deltas():
agent = make_agent()
tokens = [t async for t in agent.stream_reply(user_input="hello")]
assert len(tokens) >= 1
assert all(isinstance(t, str) for t in tokens)
async def test_stream_reply_with_history_and_context():
agent = make_agent()
ctx = get_learner_context()
history = [Message(role="user", content="earlier"), Message(role="assistant", content="ok")]
tokens = [t async for t in agent.stream_reply(history, "next", ctx)]
assert tokens
async def test_structured_reply_requires_schema():
import pytest
agent = make_agent()
with pytest.raises(ValueError):
await agent.structured_reply(user_input="x", schema=None)
async def test_name_defaults():
assert make_agent().name == "stub"
@@ -0,0 +1,71 @@
"""Learner context corpus tests — D-021 ID alignment + prompt rendering."""
from ai_service.corpus.learner_context import (
LEARNER_CONTEXTS,
get_learner_context,
)
from ai_service.prompts.assessor import render_context as render_assessor
from ai_service.prompts.coach import SYSTEM_PROMPT as COACH_PROMPT
from ai_service.prompts.coach import render_context as render_coach
from ai_service.prompts.lab import SYSTEM_PROMPT as LAB_PROMPT
from ai_service.prompts.lab import render_context as render_lab
from ai_service.prompts.mentor import SYSTEM_PROMPT as MENTOR_PROMPT
from ai_service.prompts.mentor import render_context as render_mentor
from ai_service.prompts.proctor import SYSTEM_PROMPT as PROCTOR_PROMPT
from ai_service.prompts.proctor import render_context as render_proctor
from ai_service.prompts.tutor import SYSTEM_PROMPT as TUTOR_PROMPT
from ai_service.prompts.tutor import render_context as render_tutor
def test_default_learner_resolves():
ctx = get_learner_context()
assert ctx.learner_id == "learner-001"
assert ctx.name == "Alex Rivera"
def test_unknown_learner_falls_back_to_default():
assert get_learner_context("nobody").learner_id == "learner-001"
def test_ids_align_with_ts_mock_data():
# D-021: identical ID strings to packages/mock-data (learner-progress.ts)
ctx = get_learner_context("learner-001")
stack_ids = {s.stack_id for s in ctx.active_stacks}
assert {"stack-orchestration", "stack-safety"} <= stack_ids
competency_ids = {c.competency_id for c in ctx.active_competencies}
assert "stack-orchestration-c001" in competency_ids
def test_all_prompt_modules_render_without_keyerror():
ctx = get_learner_context()
for render in (render_coach, render_tutor, render_mentor, render_assessor):
values = render(ctx)
assert isinstance(values, dict)
assert "learner_name" in values
assert values["learner_name"] == "Alex Rivera"
def test_prompts_format_map_with_rendered_context():
ctx = get_learner_context()
for prompt, render in (
(COACH_PROMPT, render_coach),
(TUTOR_PROMPT, render_tutor),
(MENTOR_PROMPT, render_mentor),
):
rendered = prompt.format_map(render(ctx))
assert "Alex Rivera" in rendered
assert "{" not in rendered # all placeholders filled
def test_lab_and_proctor_prompts_render():
ctx = get_learner_context()
# Each module renders through its OWN render_context (its own placeholders).
assert "Alex Rivera" in LAB_PROMPT.format_map(render_lab(ctx))
assert "Alex Rivera" in PROCTOR_PROMPT.format_map(render_proctor(ctx))
for prompt, render in ((LAB_PROMPT, render_lab), (PROCTOR_PROMPT, render_proctor)):
rendered = prompt.format_map(render(ctx))
assert "{" not in rendered # all placeholders filled
def test_two_seed_learners_exist():
assert set(LEARNER_CONTEXTS) == {"learner-001", "learner-002"}
@@ -0,0 +1,53 @@
"""Agent registry tests — register/get round-trip, error paths (G-4)."""
import pytest
from ai_service.agents.base import BaseAgent
from ai_service.agents.registry import (
AgentRegistry,
DuplicateAgentError,
UnknownAgentError,
)
from ai_service.config import Settings
from ai_service.llm.mock import MockProvider
class DummyAgent(BaseAgent):
name = "dummy"
def system_prompt(self, learner_context=None) -> str:
return "dummy"
def make_factory():
def factory(provider, settings):
return DummyAgent(provider, settings)
return factory
def test_register_and_get():
registry = AgentRegistry()
registry.register("dummy", make_factory())
agent = registry.get(MockProvider(), Settings(provider="mock"), "dummy")
assert isinstance(agent, DummyAgent)
assert agent.name == "dummy"
def test_unknown_agent_raises():
registry = AgentRegistry()
with pytest.raises(UnknownAgentError):
registry.get(MockProvider(), Settings(provider="mock"), "ghost")
def test_duplicate_registration_raises():
registry = AgentRegistry()
registry.register("dummy", make_factory())
with pytest.raises(DuplicateAgentError):
registry.register("dummy", make_factory())
def test_names_sorted():
registry = AgentRegistry()
registry.register("zeta", make_factory())
registry.register("alpha", make_factory())
assert registry.names() == ["alpha", "zeta"]
@@ -0,0 +1,86 @@
"""SessionStore tests — create/append/window/LRU/agent scoping (D-019)."""
import pytest
from ai_service.agents.session import InMemorySessionStore
from ai_service.llm.types import Message
def msg(n: int) -> Message:
return Message(role="user", content=f"m{n}")
async def test_create_and_get():
store = InMemorySessionStore()
session = await store.create("s1", agent="coach")
assert session.agent == "coach"
assert (await store.get("s1")).session_id == "s1"
assert await store.get("missing") is None
async def test_append_and_history_window():
store = InMemorySessionStore()
await store.create("s1", agent="coach")
for i in range(30):
await store.append("s1", msg(i))
window = await store.history_window("s1", max_messages=20)
assert len(window) == 20
assert window[0].content == "m10" # last 20 of m0..m29
assert window[-1].content == "m29"
async def test_default_window_uses_20():
store = InMemorySessionStore()
await store.create("s1", agent="coach")
for i in range(25):
await store.append("s1", msg(i))
window = await store.history_window("s1")
assert len(window) == 20
assert window[0].content == "m5"
async def test_lru_eviction_at_cap():
store = InMemorySessionStore(window=20, max_sessions=3)
for i in range(3):
await store.create(f"s{i}", agent="coach")
# touch s0 so s1 becomes least-recently-used
await store.get("s0")
await store.create("s3", agent="coach") # evicts s1
assert await store.get("s1") is None
assert await store.get("s0") is not None
assert await store.get("s2") is not None
assert await store.get("s3") is not None
async def test_lru_eviction_at_default_500_cap():
store = InMemorySessionStore() # defaults: window=20, max_sessions=500
for i in range(500):
await store.create(f"s{i}", agent="coach")
await store.get("s0") # touch the oldest → s1 becomes least-recently-used
await store.create("s500", agent="coach") # evicts s1
assert await store.get("s1") is None
assert await store.get("s0") is not None
assert await store.get("s499") is not None
assert await store.get("s500") is not None
async def test_append_unknown_session_raises():
store = InMemorySessionStore()
with pytest.raises(KeyError):
await store.append("nope", msg(0))
async def test_delete():
store = InMemorySessionStore()
await store.create("s1", agent="coach")
await store.delete("s1")
assert await store.get("s1") is None
async def test_sessions_are_agent_scoped():
store = InMemorySessionStore()
a = await store.create("coach-session", agent="coach")
b = await store.create("tutor-session", agent="tutor")
assert a.agent == "coach"
assert b.agent == "tutor"
assert a.session_id != b.session_id
@@ -0,0 +1,139 @@
"""Structured output defense tests — 4 layers (D-020), against mock providers."""
import pytest
from pydantic import BaseModel
from ai_service.agents.structured import (
StructuredOutputError,
extract_json_object,
parse_structured,
structured_completion,
)
from ai_service.llm.mock import MockProvider, ScriptedJSONProvider
from ai_service.llm.types import Message
class Score(BaseModel):
score: int
verdict: str
HINT = '{"score": <int 0-100>, "verdict": "<short verdict>"}'
def test_extract_json_plain():
assert extract_json_object('{"a": 1}') == '{"a": 1}'
def test_extract_json_fenced():
text = '```json\n{"a": 1}\n```'
assert extract_json_object(text) == '{"a": 1}'
def test_extract_json_with_prose_around():
text = 'Sure! Here is my answer: {"a": {"b": "x } y"}, "c": 2} hope that helps'
assert extract_json_object(text) == '{"a": {"b": "x } y"}, "c": 2}'
def test_extract_json_no_object_raises():
with pytest.raises(StructuredOutputError):
extract_json_object("no json here")
def test_extract_json_unbalanced_raises():
with pytest.raises(StructuredOutputError):
extract_json_object('{"a": 1')
def test_parse_structured_valid():
result = parse_structured('{"score": 88, "verdict": "solid"}', Score)
assert result.score == 88
def test_parse_structured_invalid_schema_raises():
with pytest.raises(StructuredOutputError):
parse_structured('{"wrong": "shape"}', Score)
async def test_structured_completion_happy_path():
provider = ScriptedJSONProvider({"score": 91, "verdict": "excellent work"})
messages = [Message(role="user", content="grade my artifact")]
result = await structured_completion(
provider, messages, model="m", schema=Score, schema_hint=HINT
)
assert result.score == 91
assert result.verdict == "excellent work"
async def test_structured_completion_retries_then_raises():
# Plain MockProvider returns non-schema JSON for json_object requests →
# both attempts fail validation → StructuredOutputError after ONE retry.
provider = MockProvider()
provider.received_calls = []
messages = [Message(role="user", content="grade me")]
with pytest.raises(StructuredOutputError):
await structured_completion(
provider, messages, model="m", schema=Score, schema_hint=HINT
)
async def test_structured_completion_retry_succeeds_after_invalid_first_response():
# Layer 4 recovery: first reply is wrong-schema fenced JSON, retry is valid.
class FlakyProvider(MockProvider):
def __init__(self) -> None:
super().__init__()
self.n = 0
self.retry_request: list[Message] = []
async def chat(self, messages, *, model, temperature=0.7, response_format=None):
self.n += 1
if self.n == 1:
return '```json\n{"summary": "wrong shape"}\n```'
self.retry_request = list(messages)
return '{"score": 75, "verdict": "recovered"}'
provider = FlakyProvider()
messages = [Message(role="user", content="grade me")]
result = await structured_completion(
provider, messages, model="m", schema=Score, schema_hint=HINT
)
assert result.score == 75
assert result.verdict == "recovered"
assert provider.n == 2
# The retry must feed the validation error back to the model.
retry_contents = " ".join(m.content for m in provider.retry_request)
assert "previous response was invalid" in retry_contents
assert HINT in retry_contents
async def test_structured_completion_is_bounded_to_one_retry():
# Permanently-invalid provider: exactly two provider calls, then raise.
class CountingProvider(MockProvider):
def __init__(self) -> None:
super().__init__()
self.calls = 0
async def chat(self, messages, *, model, temperature=0.7, response_format=None):
self.calls += 1
return await super().chat(
messages, model=model, response_format=response_format
)
provider = CountingProvider()
messages = [Message(role="user", content="grade me")]
with pytest.raises(StructuredOutputError, match="after retry"):
await structured_completion(
provider, messages, model="m", schema=Score, schema_hint=HINT
)
assert provider.calls == 2
async def test_structured_completion_schema_instruction_appended():
provider = ScriptedJSONProvider({"score": 70, "verdict": "passing"})
messages = [Message(role="user", content="grade")]
await structured_completion(provider, messages, model="m", schema=Score, schema_hint=HINT)
# Determinism check: same request yields same reply
again = await structured_completion(
provider, messages, model="m", schema=Score, schema_hint=HINT
)
assert again.score == 70
@@ -0,0 +1,155 @@
"""Chat endpoint session integration tests — history persistence + windowed replay."""
import json
def stream_events(client, payload) -> list[dict]:
with client.stream("POST", "/v1/chat/stream", json=payload) as response:
assert response.status_code == 200
events = []
for line in response.iter_lines():
if line.startswith("data:"):
d = line.removeprefix("data:").strip()
if d == "[DONE]":
events.append({"type": "[DONE]"})
else:
events.append(json.loads(d))
return events
def test_first_turn_creates_session_and_persists(client):
payload = {
"agent": "coach",
"session_id": "sess-1",
"messages": [{"role": "user", "content": "first turn"}],
}
events = stream_events(client, payload)
assert events[0]["type"] == "meta"
assert events[0]["session_id"] == "sess-1"
store = client.app.state.session_store
import asyncio
async def check():
return await store.history_window("sess-1")
contents = [m.content for m in asyncio.run(check())]
assert "first turn" in contents
assert any("Think" in c or "[" in c for c in contents) # mock reply persisted
def test_second_turn_replays_windowed_history(client):
payload1 = {
"agent": "coach",
"session_id": "sess-2",
"messages": [{"role": "user", "content": "turn one"}],
}
stream_events(client, payload1)
# Second turn: the API passes history + new message to the provider.
# With the mock provider we cannot observe provider inputs directly,
# but the session store must now hold both turns.
store = client.app.state.session_store
# The store is async; use the app's internals through a short event loop
import asyncio
result = {}
async def check():
result["window"] = await store.history_window("sess-2")
asyncio.run(check())
contents = [m.content for m in result["window"]]
assert "turn one" in contents
assert any(m.role == "assistant" for m in result["window"])
def test_second_turn_replays_history_to_provider(client):
# Observable provider input: a recording provider wrapper captures what
# the endpoint sends. Turn 2 must include turn 1's persisted messages.
from ai_service.llm.types import Message
provider = client.app.state.provider
original = provider.stream_chat
captured: list[list[Message]] = []
async def recording_stream(messages, *, model, temperature=0.7, response_format=None):
captured.append(list(messages))
async for t in original(
messages, model=model, temperature=temperature, response_format=response_format
):
yield t
provider.stream_chat = recording_stream
try:
stream_events(client, {
"agent": "coach", "session_id": "sess-replay",
"messages": [{"role": "user", "content": "turn one"}],
})
stream_events(client, {
"agent": "coach", "session_id": "sess-replay",
"messages": [{"role": "user", "content": "turn two"}],
})
finally:
provider.stream_chat = original
assert len(captured) == 2
turn1, turn2 = captured
assert [m.content for m in turn1] == ["turn one"]
turn2_contents = [m.content for m in turn2]
assert "turn one" in turn2_contents
assert "turn two" in turn2_contents
assert any(m.role == "assistant" for m in turn2) # persisted reply replayed
assert "turn two" == turn2_contents[-1] # new user turn last
def test_history_replay_is_windowed(client):
# Windowing: only the last 20 stored messages are replayed to the provider.
from ai_service.llm.types import Message
provider = client.app.state.provider
original = provider.stream_chat
captured: list[list[Message]] = []
async def recording_stream(messages, *, model, temperature=0.7, response_format=None):
captured.append(list(messages))
async for t in original(
messages, model=model, temperature=temperature, response_format=response_format
):
yield t
provider.stream_chat = recording_stream
try:
# 15 turns → 30 persisted messages (user + assistant per turn) > 20 window.
for i in range(15):
stream_events(client, {
"agent": "coach", "session_id": "sess-window",
"messages": [{"role": "user", "content": f"turn {i}"}],
})
finally:
provider.stream_chat = original
last_input = captured[-1]
contents = [m.content for m in last_input]
assert len(last_input) == 20 + 1 # window(20) + the new user message
assert "turn 0" not in contents # oldest messages trimmed out of replay
assert "turn 14" in contents
assert contents[-1] == "turn 14"
def test_session_agent_scoped(client):
payload = {
"agent": "tutor",
"session_id": "sess-3",
"messages": [{"role": "user", "content": "hello"}],
}
stream_events(client, payload)
import asyncio
store = client.app.state.session_store
async def check():
return await store.get("sess-3")
session = asyncio.run(check())
assert session.agent == "tutor"