docs(P02): complete agent-framework phase
---ci--- phase: 2 milestone: v0.2 status: complete ---/ci--- REQ-2-004 complete. BaseAgent ABC, agent-scoped sessions (20-msg window, 500-cap LRU), registry, 4-layer structured output defense, 6-module prompt library, D-021-aligned learner corpus, chat session persistence. 61/61 tests, ruff clean.
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"phase": 1,
|
||||
"phase": 2,
|
||||
"stage": "verify",
|
||||
"milestone": "v0.2",
|
||||
"phase_role": "execution",
|
||||
"attempts": 0,
|
||||
"updated_at": "2026-09-11T16:45:00Z"
|
||||
"updated_at": "2026-09-11T17:35:00Z"
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Agent framework — BaseAgent ABC (D-018), registry, sessions, structured outputs.
|
||||
|
||||
Boundary rule: agents/ imports from llm/, prompts/, corpus/ — never from api/.
|
||||
"""
|
||||
|
||||
from .base import BaseAgent
|
||||
from .registry import AgentRegistry
|
||||
from .session import InMemorySessionStore, SessionStore
|
||||
from .structured import StructuredOutputError, extract_json_object
|
||||
|
||||
__all__ = [
|
||||
"AgentRegistry",
|
||||
"BaseAgent",
|
||||
"InMemorySessionStore",
|
||||
"SessionStore",
|
||||
"StructuredOutputError",
|
||||
"extract_json_object",
|
||||
]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""BaseAgent ABC — the contract all six tutor agents implement (D-018).
|
||||
|
||||
Subclasses set `name`, override `system_prompt()`, and rarely `stream_reply()`.
|
||||
The default pipeline: build_messages() → provider.stream_chat()/chat().
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..config import Settings
|
||||
from ..corpus.learner_context import LearnerContext
|
||||
from ..llm.base import LLMProvider
|
||||
from ..llm.types import Message
|
||||
from .structured import structured_completion
|
||||
|
||||
|
||||
class BaseAgent(ABC):
|
||||
"""A tutor agent: system prompt + message assembly + provider delegation."""
|
||||
|
||||
name: str = "base"
|
||||
|
||||
def __init__(self, provider: LLMProvider, settings: Settings) -> None:
|
||||
self.provider = provider
|
||||
self.settings = settings
|
||||
|
||||
@abstractmethod
|
||||
def system_prompt(self, learner_context: LearnerContext | None = None) -> str:
|
||||
"""Return the agent's system prompt, learner-context-aware."""
|
||||
|
||||
def build_messages(
|
||||
self,
|
||||
history: list[Message] | None = None,
|
||||
user_input: str = "",
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> list[Message]:
|
||||
"""Compose the full message list: system prompt + history + user turn."""
|
||||
messages: list[Message] = [
|
||||
Message(role="system", content=self.system_prompt(learner_context))
|
||||
]
|
||||
for m in history or []:
|
||||
messages.append(m)
|
||||
if user_input:
|
||||
messages.append(Message(role="user", content=user_input))
|
||||
return messages
|
||||
|
||||
async def stream_reply(
|
||||
self,
|
||||
history: list[Message] | None = None,
|
||||
user_input: str = "",
|
||||
learner_context: LearnerContext | None = None,
|
||||
response_format: dict | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream incremental content deltas for a conversational reply."""
|
||||
messages = self.build_messages(history, user_input, learner_context)
|
||||
async for token in self.provider.stream_chat(
|
||||
messages, model=self.settings.model, response_format=response_format
|
||||
):
|
||||
yield token
|
||||
|
||||
async def structured_reply(
|
||||
self,
|
||||
history: list[Message] | None = None,
|
||||
user_input: str = "",
|
||||
learner_context: LearnerContext | None = None,
|
||||
schema: type[BaseModel] | None = None,
|
||||
schema_hint: str = "",
|
||||
) -> BaseModel:
|
||||
"""Non-streaming completion parsed into a pydantic model (D-020 defense)."""
|
||||
if schema is None:
|
||||
raise ValueError("structured_reply requires a schema")
|
||||
messages = self.build_messages(history, user_input, learner_context)
|
||||
return await structured_completion(
|
||||
self.provider, messages, model=self.settings.model,
|
||||
schema=schema, schema_hint=schema_hint,
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Agent registry — explicit name → agent factory map (D-018, G-4).
|
||||
|
||||
Agents are registered centrally in their own phases (P3-P5) via
|
||||
`registry.register(name, factory)`. One registration pattern, one registry.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
from ..config import Settings
|
||||
from ..llm.base import LLMProvider
|
||||
from .base import BaseAgent
|
||||
|
||||
AgentFactory = Callable[[LLMProvider, Settings], BaseAgent]
|
||||
|
||||
|
||||
class UnknownAgentError(KeyError):
|
||||
"""Raised when resolving an agent name that was never registered."""
|
||||
|
||||
|
||||
class DuplicateAgentError(ValueError):
|
||||
"""Raised when registering an agent name that already exists."""
|
||||
|
||||
|
||||
class AgentRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._factories: dict[str, AgentFactory] = {}
|
||||
|
||||
def register(self, name: str, factory: AgentFactory) -> None:
|
||||
if name in self._factories:
|
||||
raise DuplicateAgentError(f"agent {name!r} already registered")
|
||||
self._factories[name] = factory
|
||||
|
||||
def names(self) -> list[str]:
|
||||
return sorted(self._factories)
|
||||
|
||||
def get(self, provider: LLMProvider, settings: Settings, name: str) -> BaseAgent:
|
||||
try:
|
||||
factory = self._factories[name]
|
||||
except KeyError:
|
||||
raise UnknownAgentError(
|
||||
f"unknown agent {name!r}; registered: {self.names()}"
|
||||
) from None
|
||||
return factory(provider, settings)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""SessionStore — protocol + in-memory implementation (D-019).
|
||||
|
||||
Protocol is DB-migration-ready (A-003): swap InMemorySessionStore for a
|
||||
Redis/PG-backed implementation without touching the API layer.
|
||||
|
||||
Sessions are agent-scoped: switching agents starts a new session ID (avoids
|
||||
persona bleed, A-007). History windowing happens here (last N messages),
|
||||
controlling token growth per session.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol
|
||||
|
||||
from ..llm.types import Message
|
||||
|
||||
DEFAULT_WINDOW = 20
|
||||
DEFAULT_MAX_SESSIONS = 500
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentSession:
|
||||
session_id: str
|
||||
agent: str
|
||||
learner_id: str = "seed-learner-1"
|
||||
messages: list[Message] = field(default_factory=list)
|
||||
|
||||
|
||||
class SessionStore(Protocol):
|
||||
def get(self, session_id: str) -> AgentSession | None: ...
|
||||
def create(
|
||||
self, session_id: str, agent: str, learner_id: str = "seed-learner-1"
|
||||
) -> AgentSession: ...
|
||||
def append(self, session_id: str, message: Message) -> None: ...
|
||||
def history_window(
|
||||
self, session_id: str, max_messages: int = DEFAULT_WINDOW
|
||||
) -> list[Message]: ...
|
||||
def delete(self, session_id: str) -> None: ...
|
||||
|
||||
|
||||
class InMemorySessionStore:
|
||||
"""asyncio.Lock-guarded dict with 20-message windows and 500-cap LRU eviction."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
window: int = DEFAULT_WINDOW,
|
||||
max_sessions: int = DEFAULT_MAX_SESSIONS,
|
||||
) -> None:
|
||||
self._sessions: OrderedDict[str, AgentSession] = OrderedDict()
|
||||
self._lock = asyncio.Lock()
|
||||
self._window = window
|
||||
self._max_sessions = max_sessions
|
||||
|
||||
async def get(self, session_id: str) -> AgentSession | None:
|
||||
async with self._lock:
|
||||
session = self._sessions.get(session_id)
|
||||
if session is not None:
|
||||
self._sessions.move_to_end(session_id) # LRU touch
|
||||
return session
|
||||
|
||||
async def create(
|
||||
self, session_id: str, agent: str, learner_id: str = "seed-learner-1"
|
||||
) -> AgentSession:
|
||||
async with self._lock:
|
||||
session = AgentSession(session_id=session_id, agent=agent, learner_id=learner_id)
|
||||
self._sessions[session_id] = session
|
||||
self._evict_locked()
|
||||
return session
|
||||
|
||||
async def append(self, session_id: str, message: Message) -> None:
|
||||
async with self._lock:
|
||||
session = self._sessions.get(session_id)
|
||||
if session is None:
|
||||
raise KeyError(f"unknown session {session_id!r}")
|
||||
session.messages.append(message)
|
||||
self._sessions.move_to_end(session_id)
|
||||
|
||||
async def history_window(
|
||||
self, session_id: str, max_messages: int = DEFAULT_WINDOW
|
||||
) -> list[Message]:
|
||||
async with self._lock:
|
||||
session = self._sessions.get(session_id)
|
||||
if session is None:
|
||||
raise KeyError(f"unknown session {session_id!r}")
|
||||
return list(session.messages[-max_messages:])
|
||||
|
||||
async def delete(self, session_id: str) -> None:
|
||||
async with self._lock:
|
||||
self._sessions.pop(session_id, None)
|
||||
|
||||
def _evict_locked(self) -> None:
|
||||
while len(self._sessions) > self._max_sessions:
|
||||
self._sessions.popitem(last=False) # evict least-recently-used
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Structured output defense — 4 layers (D-020).
|
||||
|
||||
Layer 1: response_format={"type":"json_object"} request (auto-degrades on 400
|
||||
inside the provider).
|
||||
Layer 2: prompt-embedded schema hint ("Respond with ONLY valid JSON...").
|
||||
Layer 3: defensive parse — strip markdown fences, extract first balanced
|
||||
JSON object, pydantic model_validate.
|
||||
Layer 4: single bounded retry with the validation error fed back.
|
||||
"""
|
||||
|
||||
from typing import TypeVar
|
||||
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from ..llm.base import LLMProvider
|
||||
from ..llm.types import Message
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
class StructuredOutputError(Exception):
|
||||
"""Raised when the model output cannot be validated after one retry."""
|
||||
|
||||
|
||||
def extract_json_object(text: str) -> str:
|
||||
"""Strip fences and return the first balanced {...} block from text."""
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("```"):
|
||||
first_newline = stripped.find("\n")
|
||||
if first_newline != -1:
|
||||
stripped = stripped[first_newline + 1:]
|
||||
if stripped.rstrip().endswith("```"):
|
||||
stripped = stripped.rstrip()[:-3]
|
||||
stripped = stripped.strip()
|
||||
start = stripped.find("{")
|
||||
if start == -1:
|
||||
raise StructuredOutputError("no JSON object found in model output")
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for i, ch in enumerate(stripped[start:], start=start):
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
continue
|
||||
if ch == '"' and not escape:
|
||||
in_string = not in_string
|
||||
continue
|
||||
if in_string:
|
||||
continue
|
||||
if ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return stripped[start:i + 1]
|
||||
raise StructuredOutputError("unbalanced JSON object in model output")
|
||||
|
||||
|
||||
def parse_structured(text: str, schema: type[T]) -> T:
|
||||
"""Layer 3: fence-strip + first-balanced-object + pydantic validation."""
|
||||
candidate = extract_json_object(text)
|
||||
try:
|
||||
return schema.model_validate_json(candidate)
|
||||
except ValidationError as exc:
|
||||
raise StructuredOutputError(f"schema validation failed: {exc}") from exc
|
||||
|
||||
|
||||
def schema_instruction(schema_hint: str) -> str:
|
||||
"""Layer 2: prompt-side schema text."""
|
||||
return (
|
||||
"Respond with ONLY a valid JSON object matching this schema — "
|
||||
"no markdown fences, no prose outside the JSON. "
|
||||
f"Schema: {schema_hint}"
|
||||
)
|
||||
|
||||
|
||||
async def structured_completion(
|
||||
provider: LLMProvider,
|
||||
messages: list[Message],
|
||||
*,
|
||||
model: str,
|
||||
schema: type[T],
|
||||
schema_hint: str,
|
||||
retry_feedback: str | None = None,
|
||||
) -> T:
|
||||
"""Full 4-layer pipeline. One bounded retry (layer 4), then raise."""
|
||||
# Build request: append schema instruction to the last user message (layer 2).
|
||||
request = list(messages)
|
||||
last_user = next((m for m in reversed(request) if m.role == "user"), None)
|
||||
if last_user is not None:
|
||||
request = [
|
||||
Message(role=m.role, content=(m.content + "\n\n" + schema_instruction(schema_hint)))
|
||||
if m is last_user else m
|
||||
for m in request
|
||||
]
|
||||
response_format = {"type": "json_object"}
|
||||
raw = await provider.chat(request, model=model, response_format=response_format)
|
||||
try:
|
||||
return parse_structured(raw, schema) # layers 1+2+3
|
||||
except StructuredOutputError as exc:
|
||||
# Layer 4: single bounded retry with error feedback
|
||||
retry_prompt = (
|
||||
f"Your previous response was invalid: {exc}. "
|
||||
f"Return ONLY the corrected JSON matching: {schema_hint}"
|
||||
)
|
||||
request2 = list(messages)
|
||||
request2.append(Message(role="user", content=retry_prompt))
|
||||
raw2 = await provider.chat(request2, model=model, response_format=response_format)
|
||||
try:
|
||||
return parse_structured(raw2, schema)
|
||||
except StructuredOutputError as exc2:
|
||||
raise StructuredOutputError(
|
||||
f"structured output failed after retry: {exc2}"
|
||||
) from exc2
|
||||
@@ -12,10 +12,11 @@ from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from ..agents.session import SessionStore
|
||||
from ..config import Settings
|
||||
from ..llm.base import LLMProvider
|
||||
from ..llm.types import Message
|
||||
from .deps import get_provider, get_settings
|
||||
from .deps import get_provider, get_session_store, get_settings
|
||||
|
||||
router = APIRouter(prefix="/v1")
|
||||
|
||||
@@ -31,10 +32,19 @@ async def chat_stream(
|
||||
body: ChatStreamRequest,
|
||||
provider: LLMProvider = Depends(get_provider),
|
||||
settings: Settings = Depends(get_settings),
|
||||
sessions: SessionStore = Depends(get_session_store),
|
||||
) -> EventSourceResponse:
|
||||
if not body.messages:
|
||||
raise HTTPException(status_code=422, detail="messages must not be empty")
|
||||
|
||||
# Agent-scoped session (A-007/G-4): persisted turn history, windowed replay.
|
||||
session = await sessions.get(body.session_id)
|
||||
if session is None:
|
||||
session = await sessions.create(body.session_id, agent=body.agent)
|
||||
history = await sessions.history_window(body.session_id)
|
||||
# Persist this turn's user message before streaming.
|
||||
await sessions.append(body.session_id, body.messages[-1])
|
||||
|
||||
async def event_stream() -> AsyncIterator[dict]:
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "meta",
|
||||
@@ -43,14 +53,22 @@ async def chat_stream(
|
||||
"model": settings.model,
|
||||
})}
|
||||
first_byte = True
|
||||
reply_parts: list[str] = []
|
||||
try:
|
||||
async for token in provider.stream_chat(
|
||||
body.messages, model=settings.model
|
||||
body.messages if not history else history + body.messages,
|
||||
model=settings.model,
|
||||
):
|
||||
first_byte = False
|
||||
reply_parts.append(token)
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "delta", "content": token
|
||||
})}
|
||||
full_reply = "".join(reply_parts)
|
||||
if full_reply:
|
||||
await sessions.append(
|
||||
body.session_id, Message(role="assistant", content=full_reply)
|
||||
)
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "done", "finish_reason": "stop"
|
||||
})}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""FastAPI dependencies — provider and settings via app.state (DI)."""
|
||||
"""FastAPI dependencies — provider, settings, sessions, agents via app.state (DI)."""
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from ..agents.registry import AgentRegistry
|
||||
from ..agents.session import SessionStore
|
||||
from ..config import Settings
|
||||
from ..llm.base import LLMProvider
|
||||
|
||||
@@ -12,3 +14,11 @@ def get_settings(request: Request) -> Settings:
|
||||
|
||||
def get_provider(request: Request) -> LLMProvider:
|
||||
return request.app.state.provider
|
||||
|
||||
|
||||
def get_session_store(request: Request) -> SessionStore:
|
||||
return request.app.state.session_store
|
||||
|
||||
|
||||
def get_agent_registry(request: Request) -> AgentRegistry:
|
||||
return request.app.state.agent_registry
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Mock engine inputs — pydantic-typed corpus (D-021).
|
||||
|
||||
Convention-aligned with the TS `packages/mock-data` layer: identical ID
|
||||
strings (stack-*, comp-*, learner-*, art-*, mc-*), cross-referenced by the
|
||||
counterpart files. No codegen in v0.2 — alignment is by documented
|
||||
convention; revisit codegen only if drift bites (v0.3).
|
||||
"""
|
||||
|
||||
from .learner_context import LEARNER_CONTEXTS, LearnerContext, get_learner_context
|
||||
|
||||
__all__ = ["LEARNER_CONTEXTS", "LearnerContext", "get_learner_context"]
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Learner context corpus — pydantic mirror of TS learner-progress.ts (D-021).
|
||||
|
||||
Counterpart: packages/mock-data/src/learner-progress.ts (or learner-progress.ts
|
||||
at package root). IDs are string-identical: learner-001, stack-orchestration,
|
||||
stack-safety, stack-orchestration-c00N, art-*, mc-*.
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class CompetencyProgress(BaseModel):
|
||||
competency_id: str
|
||||
title: str
|
||||
status: str # "mastered" | "in_progress" | "not_started"
|
||||
|
||||
|
||||
class StackProgress(BaseModel):
|
||||
stack_id: str
|
||||
title: str
|
||||
percent: int
|
||||
|
||||
|
||||
class LearnerContext(BaseModel):
|
||||
learner_id: str
|
||||
name: str
|
||||
active_stacks: list[StackProgress]
|
||||
active_competencies: list[CompetencyProgress]
|
||||
microcredential_count: int
|
||||
recent_artifacts: list[str] # artifact names
|
||||
|
||||
|
||||
_STACK_ORCHESTRATION = StackProgress(
|
||||
stack_id="stack-orchestration", title="AI Orchestration Engineer", percent=62
|
||||
)
|
||||
_STACK_SAFETY = StackProgress(
|
||||
stack_id="stack-safety", title="AI Safety & Governance Lead", percent=41
|
||||
)
|
||||
|
||||
_LEARNER_1 = LearnerContext(
|
||||
learner_id="learner-001",
|
||||
name="Alex Rivera",
|
||||
active_stacks=[_STACK_ORCHESTRATION, _STACK_SAFETY],
|
||||
active_competencies=[
|
||||
CompetencyProgress(
|
||||
competency_id="stack-orchestration-c001",
|
||||
title="Agent architecture fundamentals",
|
||||
status="mastered",
|
||||
),
|
||||
CompetencyProgress(
|
||||
competency_id="stack-orchestration-c002",
|
||||
title="Multi-agent communication patterns",
|
||||
status="in_progress",
|
||||
),
|
||||
CompetencyProgress(
|
||||
competency_id="stack-orchestration-c003",
|
||||
title="Tool use and function calling",
|
||||
status="in_progress",
|
||||
),
|
||||
CompetencyProgress(
|
||||
competency_id="stack-safety-c021",
|
||||
title="Red-team basics for agent systems",
|
||||
status="in_progress",
|
||||
),
|
||||
],
|
||||
microcredential_count=4,
|
||||
recent_artifacts=[
|
||||
"Multi-agent research assistant",
|
||||
"RAG retrieval quality dashboard",
|
||||
],
|
||||
)
|
||||
|
||||
_LEARNER_2 = LearnerContext(
|
||||
learner_id="learner-002",
|
||||
name="Priya Chen",
|
||||
active_stacks=[
|
||||
StackProgress(
|
||||
stack_id="stack-designer", title="Human-AI Product Designer", percent=55
|
||||
),
|
||||
],
|
||||
active_competencies=[
|
||||
CompetencyProgress(
|
||||
competency_id="stack-designer-c001",
|
||||
title="Prompt-to-prototype workflows",
|
||||
status="mastered",
|
||||
),
|
||||
CompetencyProgress(
|
||||
competency_id="stack-designer-c002",
|
||||
title="Evaluating AI UX patterns",
|
||||
status="in_progress",
|
||||
),
|
||||
],
|
||||
microcredential_count=2,
|
||||
recent_artifacts=["AI onboarding flow concept test"],
|
||||
)
|
||||
|
||||
LEARNER_CONTEXTS: dict[str, LearnerContext] = {
|
||||
_LEARNER_1.learner_id: _LEARNER_1,
|
||||
_LEARNER_2.learner_id: _LEARNER_2,
|
||||
}
|
||||
|
||||
DEFAULT_LEARNER_ID = "learner-001"
|
||||
|
||||
|
||||
def get_learner_context(learner_id: str | None = None) -> LearnerContext:
|
||||
"""Resolve a learner context by ID, falling back to the default seed."""
|
||||
if learner_id is None:
|
||||
return LEARNER_CONTEXTS[DEFAULT_LEARNER_ID]
|
||||
return LEARNER_CONTEXTS.get(learner_id, LEARNER_CONTEXTS[DEFAULT_LEARNER_ID])
|
||||
@@ -6,6 +6,8 @@ import httpx
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .agents.registry import AgentRegistry
|
||||
from .agents.session import InMemorySessionStore
|
||||
from .api import chat_router
|
||||
from .config import Settings
|
||||
from .llm import create_provider
|
||||
@@ -21,6 +23,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.state.http_client = httpx.AsyncClient(timeout=timeout)
|
||||
app.state.settings = settings
|
||||
app.state.provider = create_provider(settings, app.state.http_client)
|
||||
app.state.session_store = InMemorySessionStore()
|
||||
app.state.agent_registry = AgentRegistry()
|
||||
yield
|
||||
await app.state.http_client.aclose()
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Prompt library — prompts are code: versioned in git, reviewed like code (D-018).
|
||||
|
||||
Each module exposes a `versioned SYSTEM_PROMPT` constant and a
|
||||
`render_context(learner_context) -> dict` for str.format_map injection.
|
||||
Final personas land in Phases 3-5; these are the initial drafts.
|
||||
"""
|
||||
|
||||
from .coach import SYSTEM_PROMPT as COACH_PROMPT
|
||||
from .coach import render_context as render_coach
|
||||
from .mentor import SYSTEM_PROMPT as MENTOR_PROMPT
|
||||
from .mentor import render_context as render_mentor
|
||||
from .tutor import SYSTEM_PROMPT as TUTOR_PROMPT
|
||||
from .tutor import render_context as render_tutor
|
||||
|
||||
__all__ = [
|
||||
"COACH_PROMPT",
|
||||
"MENTOR_PROMPT",
|
||||
"TUTOR_PROMPT",
|
||||
"render_coach",
|
||||
"render_mentor",
|
||||
"render_tutor",
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Assessor agent prompt — rubric application to artifacts and defenses (REQ-2-008).
|
||||
|
||||
Persona: rigorous, fair grader. Applies the rubric to the artifact and defense
|
||||
transcript, returns structured JSON scores. Versioned: v1 draft (Phase 2);
|
||||
final persona + rubric models in Phase 4.
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Assessor, the grading agent of an AI-native competency school.
|
||||
Learner: {learner_name}.
|
||||
You receive an artifact, its defense transcript, and a rubric. Your job:
|
||||
score each rubric criterion with evidence from the artifact and transcript.
|
||||
Be rigorous but fair — cite what the learner did, not what they should have
|
||||
done. Respond with ONLY valid JSON matching the provided rubric schema."""
|
||||
|
||||
PROMPT_VERSION = "assessor-v1-draft"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
return {"learner_name": learner_context.name}
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Coach agent prompt — pacing, motivation, retrieval practice (REQ-2-005).
|
||||
|
||||
Persona: warm, action-oriented, accountability partner. Asks for commitments,
|
||||
uses retrieval practice, keeps momentum. Versioned: v1 draft (Phase 2);
|
||||
final persona in Phase 3.
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Coach, the pacing and motivation agent of an AI-native competency school.
|
||||
Learner: {learner_name}. Active stack: {stacks}. Progress: {progress}.
|
||||
Your job: keep the learner moving. Pace their next step, motivate without fluff,
|
||||
and weave in retrieval practice — ask them to recall or apply something they
|
||||
already covered before introducing new material. Be concise, warm, and direct.
|
||||
End with exactly one clear next action."""
|
||||
|
||||
PROMPT_VERSION = "coach-v1-draft"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
|
||||
progress = (
|
||||
f"{learner_context.active_competencies[0].title} in progress"
|
||||
if learner_context.active_competencies
|
||||
else "no active competencies"
|
||||
)
|
||||
return {
|
||||
"learner_name": learner_context.name,
|
||||
"stacks": stacks or "none yet",
|
||||
"progress": progress,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Lab agent prompt — in-flow feedback over sandbox telemetry (REQ-2-007).
|
||||
|
||||
Persona: pragmatic build partner. Reads the telemetry timeline and gives
|
||||
concrete in-flow feedback: what happened, what to adjust, next step.
|
||||
Versioned: v1 draft (Phase 2); final persona + scenario serialization in Phase 4.
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Lab, the in-flow feedback agent watching a learner build in the sandbox.
|
||||
Learner: {learner_name}. Active stack: {stacks}.
|
||||
You receive a telemetry timeline of the learner's build session. Your job:
|
||||
describe what the telemetry shows, name the single most useful adjustment,
|
||||
and give one concrete next step. Be specific to the events you see —
|
||||
no generic advice. Three short paragraphs maximum."""
|
||||
|
||||
PROMPT_VERSION = "lab-v1-draft"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
|
||||
return {
|
||||
"learner_name": learner_context.name,
|
||||
"stacks": stacks or "none yet",
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Mentor agent prompt — long-horizon career narrative (REQ-2-010).
|
||||
|
||||
Persona: wise career guide. Connects today's competencies to a long-horizon
|
||||
trajectory in AI-era roles. Versioned: v1 draft (Phase 2); final in Phase 5.
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Mentor, the long-horizon career agent of an AI-native competency school.
|
||||
Learner: {learner_name}. Active stack: {stacks}. Progress: {progress}.
|
||||
Microcredentials earned: {microcredentials}. Recent artifacts: {artifacts}.
|
||||
Your job: narrate the learner's trajectory — where they are now, what their
|
||||
competency progress unlocks next, and how their artifacts position them in
|
||||
the AI-era labor market. Two to three paragraphs, forward-looking, concrete."""
|
||||
|
||||
PROMPT_VERSION = "mentor-v1-draft"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
|
||||
progress = (
|
||||
f"{learner_context.active_competencies[0].title} in progress"
|
||||
if learner_context.active_competencies
|
||||
else "no active competencies"
|
||||
)
|
||||
return {
|
||||
"learner_name": learner_context.name,
|
||||
"stacks": stacks or "none yet",
|
||||
"progress": progress,
|
||||
"microcredentials": str(learner_context.microcredential_count),
|
||||
"artifacts": ", ".join(learner_context.recent_artifacts) or "none yet",
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Proctor agent prompt — integrity signals with coaching interventions (REQ-2-009).
|
||||
|
||||
Persona: supportive observer, not punitive. Classifies integrity signals from
|
||||
telemetry and recommends coaching interventions. Versioned: v1 draft (Phase 2);
|
||||
final persona + signal models in Phase 5.
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Proctor, the integrity-support agent of an AI-native competency school.
|
||||
Learner: {learner_name}.
|
||||
You receive a telemetry timeline of session events (focus, tab switches,
|
||||
paste events, idle time). Your job: classify each signal by type and severity,
|
||||
then recommend ONE supportive coaching intervention — never punitive, never
|
||||
accusatory. Assume good faith; most signals have innocent explanations.
|
||||
Respond with ONLY valid JSON matching the provided signals schema."""
|
||||
|
||||
PROMPT_VERSION = "proctor-v1-draft"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
return {"learner_name": learner_context.name}
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Tutor agent prompt — concept delivery, Socratic questioning (REQ-2-006).
|
||||
|
||||
Persona: patient expert teacher. Delivers one concept at a time, checks
|
||||
understanding with Socratic questions, uses worked examples.
|
||||
Versioned: v1 draft (Phase 2); final persona in Phase 3.
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Tutor, the concept-delivery agent of an AI-native competency school.
|
||||
Learner: {learner_name}. Active stack: {stacks}. Progress: {progress}.
|
||||
Your job: teach concepts clearly, one at a time. Prefer Socratic questioning —
|
||||
guide the learner to the insight with a worked example, then ask one question
|
||||
that checks understanding before moving on. Never dump long walls of text."""
|
||||
|
||||
PROMPT_VERSION = "tutor-v1-draft"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
stacks = ", ".join(f"{s.title} ({s.percent}%)" for s in learner_context.active_stacks)
|
||||
progress = (
|
||||
f"{learner_context.active_competencies[0].title} in progress"
|
||||
if learner_context.active_competencies
|
||||
else "no active competencies"
|
||||
)
|
||||
return {
|
||||
"learner_name": learner_context.name,
|
||||
"stacks": stacks or "none yet",
|
||||
"progress": progress,
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
"""BaseAgent contract tests — stub agent + mock provider."""
|
||||
|
||||
|
||||
from ai_service.agents.base import BaseAgent
|
||||
from ai_service.config import Settings
|
||||
from ai_service.corpus.learner_context import get_learner_context
|
||||
from ai_service.llm.mock import MockProvider
|
||||
from ai_service.llm.types import Message
|
||||
|
||||
|
||||
class StubAgent(BaseAgent):
|
||||
name = "stub"
|
||||
|
||||
def system_prompt(self, learner_context=None) -> str:
|
||||
return "You are Stub. Answer briefly."
|
||||
|
||||
|
||||
def make_agent() -> StubAgent:
|
||||
return StubAgent(MockProvider(), Settings(provider="mock"))
|
||||
|
||||
|
||||
async def test_build_messages_composition():
|
||||
agent = make_agent()
|
||||
history = [Message(role="user", content="earlier"), Message(role="assistant", content="reply")]
|
||||
messages = agent.build_messages(history, "new question")
|
||||
assert messages[0].role == "system"
|
||||
assert messages[0].content == "You are Stub. Answer briefly."
|
||||
assert [m.content for m in messages[1:]] == ["earlier", "reply", "new question"]
|
||||
|
||||
|
||||
async def test_stream_reply_yields_deltas():
|
||||
agent = make_agent()
|
||||
tokens = [t async for t in agent.stream_reply(user_input="hello")]
|
||||
assert len(tokens) >= 1
|
||||
assert all(isinstance(t, str) for t in tokens)
|
||||
|
||||
|
||||
async def test_stream_reply_with_history_and_context():
|
||||
agent = make_agent()
|
||||
ctx = get_learner_context()
|
||||
history = [Message(role="user", content="earlier"), Message(role="assistant", content="ok")]
|
||||
tokens = [t async for t in agent.stream_reply(history, "next", ctx)]
|
||||
assert tokens
|
||||
|
||||
|
||||
async def test_structured_reply_requires_schema():
|
||||
import pytest
|
||||
|
||||
agent = make_agent()
|
||||
with pytest.raises(ValueError):
|
||||
await agent.structured_reply(user_input="x", schema=None)
|
||||
|
||||
|
||||
async def test_name_defaults():
|
||||
assert make_agent().name == "stub"
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Learner context corpus tests — D-021 ID alignment + prompt rendering."""
|
||||
|
||||
from ai_service.corpus.learner_context import (
|
||||
LEARNER_CONTEXTS,
|
||||
get_learner_context,
|
||||
)
|
||||
from ai_service.prompts.assessor import render_context as render_assessor
|
||||
from ai_service.prompts.coach import SYSTEM_PROMPT as COACH_PROMPT
|
||||
from ai_service.prompts.coach import render_context as render_coach
|
||||
from ai_service.prompts.lab import SYSTEM_PROMPT as LAB_PROMPT
|
||||
from ai_service.prompts.lab import render_context as render_lab
|
||||
from ai_service.prompts.mentor import SYSTEM_PROMPT as MENTOR_PROMPT
|
||||
from ai_service.prompts.mentor import render_context as render_mentor
|
||||
from ai_service.prompts.proctor import SYSTEM_PROMPT as PROCTOR_PROMPT
|
||||
from ai_service.prompts.proctor import render_context as render_proctor
|
||||
from ai_service.prompts.tutor import SYSTEM_PROMPT as TUTOR_PROMPT
|
||||
from ai_service.prompts.tutor import render_context as render_tutor
|
||||
|
||||
|
||||
def test_default_learner_resolves():
|
||||
ctx = get_learner_context()
|
||||
assert ctx.learner_id == "learner-001"
|
||||
assert ctx.name == "Alex Rivera"
|
||||
|
||||
|
||||
def test_unknown_learner_falls_back_to_default():
|
||||
assert get_learner_context("nobody").learner_id == "learner-001"
|
||||
|
||||
|
||||
def test_ids_align_with_ts_mock_data():
|
||||
# D-021: identical ID strings to packages/mock-data (learner-progress.ts)
|
||||
ctx = get_learner_context("learner-001")
|
||||
stack_ids = {s.stack_id for s in ctx.active_stacks}
|
||||
assert {"stack-orchestration", "stack-safety"} <= stack_ids
|
||||
competency_ids = {c.competency_id for c in ctx.active_competencies}
|
||||
assert "stack-orchestration-c001" in competency_ids
|
||||
|
||||
|
||||
def test_all_prompt_modules_render_without_keyerror():
|
||||
ctx = get_learner_context()
|
||||
for render in (render_coach, render_tutor, render_mentor, render_assessor):
|
||||
values = render(ctx)
|
||||
assert isinstance(values, dict)
|
||||
assert "learner_name" in values
|
||||
assert values["learner_name"] == "Alex Rivera"
|
||||
|
||||
|
||||
def test_prompts_format_map_with_rendered_context():
|
||||
ctx = get_learner_context()
|
||||
for prompt, render in (
|
||||
(COACH_PROMPT, render_coach),
|
||||
(TUTOR_PROMPT, render_tutor),
|
||||
(MENTOR_PROMPT, render_mentor),
|
||||
):
|
||||
rendered = prompt.format_map(render(ctx))
|
||||
assert "Alex Rivera" in rendered
|
||||
assert "{" not in rendered # all placeholders filled
|
||||
|
||||
|
||||
def test_lab_and_proctor_prompts_render():
|
||||
ctx = get_learner_context()
|
||||
# Each module renders through its OWN render_context (its own placeholders).
|
||||
assert "Alex Rivera" in LAB_PROMPT.format_map(render_lab(ctx))
|
||||
assert "Alex Rivera" in PROCTOR_PROMPT.format_map(render_proctor(ctx))
|
||||
for prompt, render in ((LAB_PROMPT, render_lab), (PROCTOR_PROMPT, render_proctor)):
|
||||
rendered = prompt.format_map(render(ctx))
|
||||
assert "{" not in rendered # all placeholders filled
|
||||
|
||||
|
||||
def test_two_seed_learners_exist():
|
||||
assert set(LEARNER_CONTEXTS) == {"learner-001", "learner-002"}
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Agent registry tests — register/get round-trip, error paths (G-4)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from ai_service.agents.base import BaseAgent
|
||||
from ai_service.agents.registry import (
|
||||
AgentRegistry,
|
||||
DuplicateAgentError,
|
||||
UnknownAgentError,
|
||||
)
|
||||
from ai_service.config import Settings
|
||||
from ai_service.llm.mock import MockProvider
|
||||
|
||||
|
||||
class DummyAgent(BaseAgent):
|
||||
name = "dummy"
|
||||
|
||||
def system_prompt(self, learner_context=None) -> str:
|
||||
return "dummy"
|
||||
|
||||
|
||||
def make_factory():
|
||||
def factory(provider, settings):
|
||||
return DummyAgent(provider, settings)
|
||||
return factory
|
||||
|
||||
|
||||
def test_register_and_get():
|
||||
registry = AgentRegistry()
|
||||
registry.register("dummy", make_factory())
|
||||
agent = registry.get(MockProvider(), Settings(provider="mock"), "dummy")
|
||||
assert isinstance(agent, DummyAgent)
|
||||
assert agent.name == "dummy"
|
||||
|
||||
|
||||
def test_unknown_agent_raises():
|
||||
registry = AgentRegistry()
|
||||
with pytest.raises(UnknownAgentError):
|
||||
registry.get(MockProvider(), Settings(provider="mock"), "ghost")
|
||||
|
||||
|
||||
def test_duplicate_registration_raises():
|
||||
registry = AgentRegistry()
|
||||
registry.register("dummy", make_factory())
|
||||
with pytest.raises(DuplicateAgentError):
|
||||
registry.register("dummy", make_factory())
|
||||
|
||||
|
||||
def test_names_sorted():
|
||||
registry = AgentRegistry()
|
||||
registry.register("zeta", make_factory())
|
||||
registry.register("alpha", make_factory())
|
||||
assert registry.names() == ["alpha", "zeta"]
|
||||
@@ -0,0 +1,86 @@
|
||||
"""SessionStore tests — create/append/window/LRU/agent scoping (D-019)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from ai_service.agents.session import InMemorySessionStore
|
||||
from ai_service.llm.types import Message
|
||||
|
||||
|
||||
def msg(n: int) -> Message:
|
||||
return Message(role="user", content=f"m{n}")
|
||||
|
||||
|
||||
async def test_create_and_get():
|
||||
store = InMemorySessionStore()
|
||||
session = await store.create("s1", agent="coach")
|
||||
assert session.agent == "coach"
|
||||
assert (await store.get("s1")).session_id == "s1"
|
||||
assert await store.get("missing") is None
|
||||
|
||||
|
||||
async def test_append_and_history_window():
|
||||
store = InMemorySessionStore()
|
||||
await store.create("s1", agent="coach")
|
||||
for i in range(30):
|
||||
await store.append("s1", msg(i))
|
||||
window = await store.history_window("s1", max_messages=20)
|
||||
assert len(window) == 20
|
||||
assert window[0].content == "m10" # last 20 of m0..m29
|
||||
assert window[-1].content == "m29"
|
||||
|
||||
|
||||
async def test_default_window_uses_20():
|
||||
store = InMemorySessionStore()
|
||||
await store.create("s1", agent="coach")
|
||||
for i in range(25):
|
||||
await store.append("s1", msg(i))
|
||||
window = await store.history_window("s1")
|
||||
assert len(window) == 20
|
||||
assert window[0].content == "m5"
|
||||
|
||||
|
||||
async def test_lru_eviction_at_cap():
|
||||
store = InMemorySessionStore(window=20, max_sessions=3)
|
||||
for i in range(3):
|
||||
await store.create(f"s{i}", agent="coach")
|
||||
# touch s0 so s1 becomes least-recently-used
|
||||
await store.get("s0")
|
||||
await store.create("s3", agent="coach") # evicts s1
|
||||
assert await store.get("s1") is None
|
||||
assert await store.get("s0") is not None
|
||||
assert await store.get("s2") is not None
|
||||
assert await store.get("s3") is not None
|
||||
|
||||
|
||||
async def test_lru_eviction_at_default_500_cap():
|
||||
store = InMemorySessionStore() # defaults: window=20, max_sessions=500
|
||||
for i in range(500):
|
||||
await store.create(f"s{i}", agent="coach")
|
||||
await store.get("s0") # touch the oldest → s1 becomes least-recently-used
|
||||
await store.create("s500", agent="coach") # evicts s1
|
||||
assert await store.get("s1") is None
|
||||
assert await store.get("s0") is not None
|
||||
assert await store.get("s499") is not None
|
||||
assert await store.get("s500") is not None
|
||||
|
||||
|
||||
async def test_append_unknown_session_raises():
|
||||
store = InMemorySessionStore()
|
||||
with pytest.raises(KeyError):
|
||||
await store.append("nope", msg(0))
|
||||
|
||||
|
||||
async def test_delete():
|
||||
store = InMemorySessionStore()
|
||||
await store.create("s1", agent="coach")
|
||||
await store.delete("s1")
|
||||
assert await store.get("s1") is None
|
||||
|
||||
|
||||
async def test_sessions_are_agent_scoped():
|
||||
store = InMemorySessionStore()
|
||||
a = await store.create("coach-session", agent="coach")
|
||||
b = await store.create("tutor-session", agent="tutor")
|
||||
assert a.agent == "coach"
|
||||
assert b.agent == "tutor"
|
||||
assert a.session_id != b.session_id
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Structured output defense tests — 4 layers (D-020), against mock providers."""
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ai_service.agents.structured import (
|
||||
StructuredOutputError,
|
||||
extract_json_object,
|
||||
parse_structured,
|
||||
structured_completion,
|
||||
)
|
||||
from ai_service.llm.mock import MockProvider, ScriptedJSONProvider
|
||||
from ai_service.llm.types import Message
|
||||
|
||||
|
||||
class Score(BaseModel):
|
||||
score: int
|
||||
verdict: str
|
||||
|
||||
|
||||
HINT = '{"score": <int 0-100>, "verdict": "<short verdict>"}'
|
||||
|
||||
|
||||
def test_extract_json_plain():
|
||||
assert extract_json_object('{"a": 1}') == '{"a": 1}'
|
||||
|
||||
|
||||
def test_extract_json_fenced():
|
||||
text = '```json\n{"a": 1}\n```'
|
||||
assert extract_json_object(text) == '{"a": 1}'
|
||||
|
||||
|
||||
def test_extract_json_with_prose_around():
|
||||
text = 'Sure! Here is my answer: {"a": {"b": "x } y"}, "c": 2} hope that helps'
|
||||
assert extract_json_object(text) == '{"a": {"b": "x } y"}, "c": 2}'
|
||||
|
||||
|
||||
def test_extract_json_no_object_raises():
|
||||
with pytest.raises(StructuredOutputError):
|
||||
extract_json_object("no json here")
|
||||
|
||||
|
||||
def test_extract_json_unbalanced_raises():
|
||||
with pytest.raises(StructuredOutputError):
|
||||
extract_json_object('{"a": 1')
|
||||
|
||||
|
||||
def test_parse_structured_valid():
|
||||
result = parse_structured('{"score": 88, "verdict": "solid"}', Score)
|
||||
assert result.score == 88
|
||||
|
||||
|
||||
def test_parse_structured_invalid_schema_raises():
|
||||
with pytest.raises(StructuredOutputError):
|
||||
parse_structured('{"wrong": "shape"}', Score)
|
||||
|
||||
|
||||
async def test_structured_completion_happy_path():
|
||||
provider = ScriptedJSONProvider({"score": 91, "verdict": "excellent work"})
|
||||
messages = [Message(role="user", content="grade my artifact")]
|
||||
result = await structured_completion(
|
||||
provider, messages, model="m", schema=Score, schema_hint=HINT
|
||||
)
|
||||
assert result.score == 91
|
||||
assert result.verdict == "excellent work"
|
||||
|
||||
|
||||
async def test_structured_completion_retries_then_raises():
|
||||
# Plain MockProvider returns non-schema JSON for json_object requests →
|
||||
# both attempts fail validation → StructuredOutputError after ONE retry.
|
||||
provider = MockProvider()
|
||||
provider.received_calls = []
|
||||
messages = [Message(role="user", content="grade me")]
|
||||
with pytest.raises(StructuredOutputError):
|
||||
await structured_completion(
|
||||
provider, messages, model="m", schema=Score, schema_hint=HINT
|
||||
)
|
||||
|
||||
|
||||
async def test_structured_completion_retry_succeeds_after_invalid_first_response():
|
||||
# Layer 4 recovery: first reply is wrong-schema fenced JSON, retry is valid.
|
||||
class FlakyProvider(MockProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.n = 0
|
||||
self.retry_request: list[Message] = []
|
||||
|
||||
async def chat(self, messages, *, model, temperature=0.7, response_format=None):
|
||||
self.n += 1
|
||||
if self.n == 1:
|
||||
return '```json\n{"summary": "wrong shape"}\n```'
|
||||
self.retry_request = list(messages)
|
||||
return '{"score": 75, "verdict": "recovered"}'
|
||||
|
||||
provider = FlakyProvider()
|
||||
messages = [Message(role="user", content="grade me")]
|
||||
result = await structured_completion(
|
||||
provider, messages, model="m", schema=Score, schema_hint=HINT
|
||||
)
|
||||
assert result.score == 75
|
||||
assert result.verdict == "recovered"
|
||||
assert provider.n == 2
|
||||
# The retry must feed the validation error back to the model.
|
||||
retry_contents = " ".join(m.content for m in provider.retry_request)
|
||||
assert "previous response was invalid" in retry_contents
|
||||
assert HINT in retry_contents
|
||||
|
||||
|
||||
async def test_structured_completion_is_bounded_to_one_retry():
|
||||
# Permanently-invalid provider: exactly two provider calls, then raise.
|
||||
class CountingProvider(MockProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
async def chat(self, messages, *, model, temperature=0.7, response_format=None):
|
||||
self.calls += 1
|
||||
return await super().chat(
|
||||
messages, model=model, response_format=response_format
|
||||
)
|
||||
|
||||
provider = CountingProvider()
|
||||
messages = [Message(role="user", content="grade me")]
|
||||
with pytest.raises(StructuredOutputError, match="after retry"):
|
||||
await structured_completion(
|
||||
provider, messages, model="m", schema=Score, schema_hint=HINT
|
||||
)
|
||||
assert provider.calls == 2
|
||||
|
||||
|
||||
async def test_structured_completion_schema_instruction_appended():
|
||||
provider = ScriptedJSONProvider({"score": 70, "verdict": "passing"})
|
||||
messages = [Message(role="user", content="grade")]
|
||||
await structured_completion(provider, messages, model="m", schema=Score, schema_hint=HINT)
|
||||
# Determinism check: same request yields same reply
|
||||
again = await structured_completion(
|
||||
provider, messages, model="m", schema=Score, schema_hint=HINT
|
||||
)
|
||||
assert again.score == 70
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Chat endpoint session integration tests — history persistence + windowed replay."""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def stream_events(client, payload) -> list[dict]:
|
||||
with client.stream("POST", "/v1/chat/stream", json=payload) as response:
|
||||
assert response.status_code == 200
|
||||
events = []
|
||||
for line in response.iter_lines():
|
||||
if line.startswith("data:"):
|
||||
d = line.removeprefix("data:").strip()
|
||||
if d == "[DONE]":
|
||||
events.append({"type": "[DONE]"})
|
||||
else:
|
||||
events.append(json.loads(d))
|
||||
return events
|
||||
|
||||
|
||||
def test_first_turn_creates_session_and_persists(client):
|
||||
payload = {
|
||||
"agent": "coach",
|
||||
"session_id": "sess-1",
|
||||
"messages": [{"role": "user", "content": "first turn"}],
|
||||
}
|
||||
events = stream_events(client, payload)
|
||||
assert events[0]["type"] == "meta"
|
||||
assert events[0]["session_id"] == "sess-1"
|
||||
store = client.app.state.session_store
|
||||
|
||||
import asyncio
|
||||
|
||||
async def check():
|
||||
return await store.history_window("sess-1")
|
||||
|
||||
contents = [m.content for m in asyncio.run(check())]
|
||||
assert "first turn" in contents
|
||||
assert any("Think" in c or "[" in c for c in contents) # mock reply persisted
|
||||
|
||||
|
||||
def test_second_turn_replays_windowed_history(client):
|
||||
payload1 = {
|
||||
"agent": "coach",
|
||||
"session_id": "sess-2",
|
||||
"messages": [{"role": "user", "content": "turn one"}],
|
||||
}
|
||||
stream_events(client, payload1)
|
||||
# Second turn: the API passes history + new message to the provider.
|
||||
# With the mock provider we cannot observe provider inputs directly,
|
||||
# but the session store must now hold both turns.
|
||||
store = client.app.state.session_store
|
||||
# The store is async; use the app's internals through a short event loop
|
||||
import asyncio
|
||||
result = {}
|
||||
|
||||
async def check():
|
||||
result["window"] = await store.history_window("sess-2")
|
||||
|
||||
asyncio.run(check())
|
||||
contents = [m.content for m in result["window"]]
|
||||
assert "turn one" in contents
|
||||
assert any(m.role == "assistant" for m in result["window"])
|
||||
|
||||
|
||||
def test_second_turn_replays_history_to_provider(client):
|
||||
# Observable provider input: a recording provider wrapper captures what
|
||||
# the endpoint sends. Turn 2 must include turn 1's persisted messages.
|
||||
|
||||
from ai_service.llm.types import Message
|
||||
|
||||
provider = client.app.state.provider
|
||||
original = provider.stream_chat
|
||||
captured: list[list[Message]] = []
|
||||
|
||||
async def recording_stream(messages, *, model, temperature=0.7, response_format=None):
|
||||
captured.append(list(messages))
|
||||
async for t in original(
|
||||
messages, model=model, temperature=temperature, response_format=response_format
|
||||
):
|
||||
yield t
|
||||
|
||||
provider.stream_chat = recording_stream
|
||||
try:
|
||||
stream_events(client, {
|
||||
"agent": "coach", "session_id": "sess-replay",
|
||||
"messages": [{"role": "user", "content": "turn one"}],
|
||||
})
|
||||
stream_events(client, {
|
||||
"agent": "coach", "session_id": "sess-replay",
|
||||
"messages": [{"role": "user", "content": "turn two"}],
|
||||
})
|
||||
finally:
|
||||
provider.stream_chat = original
|
||||
|
||||
assert len(captured) == 2
|
||||
turn1, turn2 = captured
|
||||
assert [m.content for m in turn1] == ["turn one"]
|
||||
turn2_contents = [m.content for m in turn2]
|
||||
assert "turn one" in turn2_contents
|
||||
assert "turn two" in turn2_contents
|
||||
assert any(m.role == "assistant" for m in turn2) # persisted reply replayed
|
||||
assert "turn two" == turn2_contents[-1] # new user turn last
|
||||
|
||||
|
||||
def test_history_replay_is_windowed(client):
|
||||
# Windowing: only the last 20 stored messages are replayed to the provider.
|
||||
|
||||
from ai_service.llm.types import Message
|
||||
|
||||
provider = client.app.state.provider
|
||||
original = provider.stream_chat
|
||||
captured: list[list[Message]] = []
|
||||
|
||||
async def recording_stream(messages, *, model, temperature=0.7, response_format=None):
|
||||
captured.append(list(messages))
|
||||
async for t in original(
|
||||
messages, model=model, temperature=temperature, response_format=response_format
|
||||
):
|
||||
yield t
|
||||
|
||||
provider.stream_chat = recording_stream
|
||||
try:
|
||||
# 15 turns → 30 persisted messages (user + assistant per turn) > 20 window.
|
||||
for i in range(15):
|
||||
stream_events(client, {
|
||||
"agent": "coach", "session_id": "sess-window",
|
||||
"messages": [{"role": "user", "content": f"turn {i}"}],
|
||||
})
|
||||
finally:
|
||||
provider.stream_chat = original
|
||||
|
||||
last_input = captured[-1]
|
||||
contents = [m.content for m in last_input]
|
||||
assert len(last_input) == 20 + 1 # window(20) + the new user message
|
||||
assert "turn 0" not in contents # oldest messages trimmed out of replay
|
||||
assert "turn 14" in contents
|
||||
assert contents[-1] == "turn 14"
|
||||
|
||||
|
||||
def test_session_agent_scoped(client):
|
||||
payload = {
|
||||
"agent": "tutor",
|
||||
"session_id": "sess-3",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
}
|
||||
stream_events(client, payload)
|
||||
import asyncio
|
||||
|
||||
store = client.app.state.session_store
|
||||
|
||||
async def check():
|
||||
return await store.get("sess-3")
|
||||
|
||||
session = asyncio.run(check())
|
||||
assert session.agent == "tutor"
|
||||
Reference in New Issue
Block a user