docs(P04): complete lab-assessor-agents phase
---ci--- phase: 4 milestone: v0.2 status: complete ---/ci--- REQ-2-007/008 complete. Lab streams scenario-driven in-flow feedback over mock telemetry; Assessor returns pydantic-validated rubric scores via 4-layer defense. Endpoints /v1/lab/feedback + /v1/assessment/ evaluate. D-021-aligned corpora both sides. 110/110 tests, ruff + tsc green, lockfile integrity restored.
This commit is contained in:
CIAgent
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"phase": 3,
|
||||
"phase": 4,
|
||||
"stage": "verify",
|
||||
"milestone": "v0.2",
|
||||
"phase_role": "execution",
|
||||
"attempts": 0,
|
||||
"updated_at": "2026-09-11T18:40:00Z"
|
||||
"updated_at": "2026-09-11T19:50:00Z"
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
"""AssessorAgent — rubric application to pre-baked artifacts (REQ-2-008).
|
||||
|
||||
Structured-output showcase: applies the 4-layer defense (D-020) to return
|
||||
a pydantic-validated rubric score. Mock engine inputs (corpus artifacts +
|
||||
transcripts); real process-trace grading is v0.3+.
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..corpus.artifacts import (
|
||||
ArtifactSubmission,
|
||||
AssessmentRubric,
|
||||
DefenseTranscript,
|
||||
render_rubric,
|
||||
render_transcript,
|
||||
)
|
||||
from ..corpus.learner_context import LearnerContext, get_learner_context
|
||||
from ..llm.types import Message
|
||||
from ..prompts.assessor import SYSTEM_PROMPT, render_context
|
||||
from .base import BaseAgent
|
||||
|
||||
|
||||
class CriterionScore(BaseModel):
|
||||
criterion_id: str
|
||||
name: str
|
||||
score: int = Field(ge=0, le=100)
|
||||
evidence: str
|
||||
|
||||
|
||||
class RubricScore(BaseModel):
|
||||
rubric_id: str
|
||||
artifact_id: str
|
||||
competency_id: str
|
||||
scores: list[CriterionScore]
|
||||
strengths: list[str] = Field(min_length=1, max_length=2)
|
||||
gaps: list[str] = Field(min_length=1, max_length=2)
|
||||
verdict: str # "mastered" | "developing" | "not_yet"
|
||||
|
||||
def weighted_total(self, rubric: AssessmentRubric) -> float:
|
||||
by_id = {c.criterion_id: c for c in rubric.criteria}
|
||||
total = 0.0
|
||||
for s in self.scores:
|
||||
total += s.score * by_id[s.criterion_id].weight
|
||||
return total
|
||||
|
||||
|
||||
RUBRIC_SCORE_SCHEMA_HINT = (
|
||||
'{"rubric_id": "<id>", "artifact_id": "<id>", "competency_id": "<id>", '
|
||||
'"scores": [{"criterion_id": "<id>", "name": "<name>", "score": <0-100>, '
|
||||
'"evidence": "<one sentence>"}], "strengths": ["<one sentence>"], '
|
||||
'"gaps": ["<one sentence>"], "verdict": "mastered"|"developing"|"not_yet"}'
|
||||
)
|
||||
|
||||
|
||||
class AssessorAgent(BaseAgent):
|
||||
name = "assessor"
|
||||
|
||||
def system_prompt(self, learner_context: LearnerContext | None = None) -> str:
|
||||
ctx = learner_context or get_learner_context()
|
||||
return SYSTEM_PROMPT.format_map(render_context(ctx))
|
||||
|
||||
def build_evaluation_input(
|
||||
self,
|
||||
artifact: ArtifactSubmission,
|
||||
rubric: AssessmentRubric,
|
||||
transcript: DefenseTranscript | None,
|
||||
) -> str:
|
||||
parts = [
|
||||
f"ARTIFACT: {artifact.name} ({artifact.artifact_id})",
|
||||
f"Evidence excerpt: {artifact.evidence_excerpt}",
|
||||
"",
|
||||
render_rubric(rubric),
|
||||
]
|
||||
if transcript is not None:
|
||||
parts += ["", render_transcript(transcript)]
|
||||
return "\n".join(parts)
|
||||
|
||||
async def evaluate(
|
||||
self,
|
||||
artifact: ArtifactSubmission,
|
||||
rubric: AssessmentRubric,
|
||||
transcript: DefenseTranscript | None,
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> RubricScore:
|
||||
evaluation_input = self.build_evaluation_input(artifact, rubric, transcript)
|
||||
score: RubricScore = await self.structured_reply(
|
||||
history=None,
|
||||
user_input=evaluation_input,
|
||||
learner_context=learner_context,
|
||||
schema=RubricScore,
|
||||
schema_hint=RUBRIC_SCORE_SCHEMA_HINT,
|
||||
)
|
||||
return score
|
||||
|
||||
def build_messages(
|
||||
self,
|
||||
history: list[Message] | None = None,
|
||||
user_input: str = "",
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> list[Message]:
|
||||
return super().build_messages(history, user_input, learner_context)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""LabAgent — in-flow feedback over simulated sandbox telemetry (REQ-2-007).
|
||||
|
||||
Scenario-driven: consumes a LabTelemetryScenario from the corpus, renders
|
||||
the event timeline into the conversation, streams concrete feedback.
|
||||
No session chat — each request is one scenario read.
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..config import Settings
|
||||
from ..corpus.learner_context import LearnerContext, get_learner_context
|
||||
from ..corpus.telemetry import LabTelemetryScenario, summarize_scenario
|
||||
from ..llm.base import LLMProvider
|
||||
from ..llm.types import Message
|
||||
from ..prompts.lab import SYSTEM_PROMPT, render_context
|
||||
from .base import BaseAgent
|
||||
|
||||
|
||||
class LabFeedbackRequest(BaseModel):
|
||||
scenario_id: str
|
||||
|
||||
|
||||
class LabAgent(BaseAgent):
|
||||
name = "lab"
|
||||
|
||||
def __init__(self, provider: LLMProvider, settings: Settings) -> None:
|
||||
super().__init__(provider, settings)
|
||||
|
||||
def system_prompt(self, learner_context: LearnerContext | None = None) -> str:
|
||||
ctx = learner_context or get_learner_context()
|
||||
return SYSTEM_PROMPT.format_map(render_context(ctx))
|
||||
|
||||
def build_messages(
|
||||
self,
|
||||
history: list[Message] | None = None,
|
||||
user_input: str = "",
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> list[Message]:
|
||||
"""user_input carries the rendered telemetry timeline."""
|
||||
return super().build_messages(history, user_input, learner_context)
|
||||
|
||||
async def stream_feedback(
|
||||
self,
|
||||
scenario: LabTelemetryScenario,
|
||||
learner_context: LearnerContext | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
timeline = summarize_scenario(scenario)
|
||||
async for token in self.stream_reply(
|
||||
history=None, user_input=timeline, learner_context=learner_context
|
||||
):
|
||||
yield token
|
||||
@@ -18,11 +18,17 @@ def register_builtin_agents(registry: "AgentRegistry") -> None:
|
||||
|
||||
Phases 3-5 add their agents here as they land.
|
||||
"""
|
||||
from .assessor import AssessorAgent
|
||||
from .coach import CoachAgent
|
||||
from .lab import LabAgent
|
||||
from .tutor import TutorAgent
|
||||
|
||||
registry.register("coach", lambda provider, settings: CoachAgent(provider, settings))
|
||||
registry.register("tutor", lambda provider, settings: TutorAgent(provider, settings))
|
||||
registry.register("lab", lambda provider, settings: LabAgent(provider, settings))
|
||||
registry.register(
|
||||
"assessor", lambda provider, settings: AssessorAgent(provider, settings)
|
||||
)
|
||||
|
||||
|
||||
class UnknownAgentError(KeyError):
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
Boundary rule: api/ composes agents/ and llm/; they never import api/.
|
||||
"""
|
||||
|
||||
from .assessment import router as assessment_router
|
||||
from .chat import router as chat_router
|
||||
from .lab import router as lab_router
|
||||
|
||||
__all__ = ["chat_router"]
|
||||
__all__ = ["assessment_router", "chat_router", "lab_router"]
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""POST /v1/assessment/evaluate — structured rubric scores (REQ-2-008).
|
||||
|
||||
JSON response (not SSE): a pydantic-validated RubricScore. Unknown
|
||||
artifact → 404. The Assessor's structured output IS the payload.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..agents.assessor import RubricScore
|
||||
from ..agents.registry import AgentRegistry
|
||||
from ..config import Settings
|
||||
from ..corpus.artifacts import get_artifact_bundle, get_transcript_for_artifact
|
||||
from ..corpus.learner_context import get_learner_context
|
||||
from .deps import get_agent_registry, get_provider, get_settings
|
||||
|
||||
router = APIRouter(prefix="/v1")
|
||||
|
||||
|
||||
class AssessmentRequest(BaseModel):
|
||||
artifact_id: str = Field(min_length=1)
|
||||
learner_id: str | None = None
|
||||
|
||||
|
||||
@router.post("/assessment/evaluate", response_model=RubricScore)
|
||||
async def assessment_evaluate(
|
||||
body: AssessmentRequest,
|
||||
registry: AgentRegistry = Depends(get_agent_registry),
|
||||
settings: Settings = Depends(get_settings),
|
||||
provider=Depends(get_provider),
|
||||
) -> RubricScore:
|
||||
bundle = get_artifact_bundle(body.artifact_id)
|
||||
if bundle is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"unknown artifact {body.artifact_id!r}"
|
||||
)
|
||||
artifact, rubric = bundle
|
||||
transcript = get_transcript_for_artifact(artifact.artifact_id)
|
||||
agent = registry.get(provider, settings, "assessor")
|
||||
learner_context = get_learner_context(body.learner_id)
|
||||
try:
|
||||
return await agent.evaluate(artifact, rubric, transcript, learner_context)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"assessment evaluation failed: {exc}",
|
||||
) from exc
|
||||
@@ -0,0 +1,70 @@
|
||||
"""POST /v1/lab/feedback — SSE stream of Lab in-flow feedback (REQ-2-007).
|
||||
|
||||
D-016 envelope with agent=lab. Unknown scenario → 404 before streaming.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from ..agents.registry import AgentRegistry
|
||||
from ..config import Settings
|
||||
from ..corpus.learner_context import get_learner_context
|
||||
from ..corpus.telemetry import get_lab_scenario
|
||||
from .deps import get_agent_registry, get_provider, get_settings
|
||||
|
||||
router = APIRouter(prefix="/v1")
|
||||
|
||||
|
||||
class LabFeedbackRequest(BaseModel):
|
||||
scenario_id: str = Field(min_length=1)
|
||||
learner_id: str | None = None
|
||||
|
||||
|
||||
@router.post("/lab/feedback")
|
||||
async def lab_feedback(
|
||||
body: LabFeedbackRequest,
|
||||
registry: AgentRegistry = Depends(get_agent_registry),
|
||||
settings: Settings = Depends(get_settings),
|
||||
provider=Depends(get_provider),
|
||||
) -> EventSourceResponse:
|
||||
scenario = get_lab_scenario(body.scenario_id)
|
||||
if scenario is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"unknown scenario {body.scenario_id!r}"
|
||||
)
|
||||
agent = registry.get(provider, settings, "lab")
|
||||
learner_context = get_learner_context(body.learner_id)
|
||||
|
||||
async def event_stream() -> AsyncIterator[dict]:
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "meta",
|
||||
"agent": "lab",
|
||||
"scenario_id": body.scenario_id,
|
||||
"model": settings.model,
|
||||
})}
|
||||
first_byte = True
|
||||
try:
|
||||
async for token in agent.stream_feedback(scenario, learner_context):
|
||||
first_byte = False
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "delta", "content": token
|
||||
})}
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "done", "finish_reason": "stop"
|
||||
})}
|
||||
except Exception as exc:
|
||||
code = "provider_unavailable" if first_byte else "provider_error"
|
||||
yield {"event": "message", "data": json.dumps({
|
||||
"type": "error", "code": code, "message": str(exc)
|
||||
})}
|
||||
finally:
|
||||
yield {"event": "message", "data": "[DONE]"}
|
||||
|
||||
return EventSourceResponse(
|
||||
event_stream(),
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
@@ -8,4 +8,8 @@ convention; revisit codegen only if drift bites (v0.3).
|
||||
|
||||
from .learner_context import LEARNER_CONTEXTS, LearnerContext, get_learner_context
|
||||
|
||||
__all__ = ["LEARNER_CONTEXTS", "LearnerContext", "get_learner_context"]
|
||||
__all__ = [
|
||||
"LEARNER_CONTEXTS",
|
||||
"LearnerContext",
|
||||
"get_learner_context",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Pre-baked artifacts + rubrics + defense transcripts — Assessor mock inputs (REQ-2-008).
|
||||
|
||||
Counterpart: packages/mock-data/ai-scenarios.ts (artifact IDs string-identical,
|
||||
D-021). Real process-trace grading is a v0.3+ engine (assessment engine);
|
||||
these pre-baked submissions stand in for artifact + defense evaluation.
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class RubricCriterion(BaseModel):
|
||||
criterion_id: str
|
||||
name: str
|
||||
weight: float
|
||||
description: str
|
||||
|
||||
|
||||
class AssessmentRubric(BaseModel):
|
||||
rubric_id: str
|
||||
competency_id: str
|
||||
criteria: list[RubricCriterion]
|
||||
|
||||
|
||||
class ArtifactSubmission(BaseModel):
|
||||
artifact_id: str
|
||||
name: str
|
||||
artifact_type: str # "code" | "design" | "simulation"
|
||||
competency_id: str
|
||||
description: str
|
||||
evidence_excerpt: str # what the grader sees of the artifact itself
|
||||
|
||||
|
||||
class DefenseTranscript(BaseModel):
|
||||
transcript_id: str
|
||||
artifact_id: str
|
||||
turns: list[dict] # {"speaker": "examiner"|"learner", "text": "..."}
|
||||
|
||||
|
||||
_RUBRIC_ORCHESTRATION = AssessmentRubric(
|
||||
rubric_id="rubric-orchestration-c002",
|
||||
competency_id="stack-orchestration-c002",
|
||||
criteria=[
|
||||
RubricCriterion(
|
||||
criterion_id="rc-architecture",
|
||||
name="Agent architecture soundness",
|
||||
weight=0.3,
|
||||
description="State boundaries and responsibilities are clearly separated",
|
||||
),
|
||||
RubricCriterion(
|
||||
criterion_id="rc-communication",
|
||||
name="Inter-agent communication design",
|
||||
weight=0.3,
|
||||
description="Message contracts are explicit, typed, and failure-aware",
|
||||
),
|
||||
RubricCriterion(
|
||||
criterion_id="rc-reliability",
|
||||
name="Reliability engineering",
|
||||
weight=0.25,
|
||||
description="Retries, timeouts, and degradation paths handled",
|
||||
),
|
||||
RubricCriterion(
|
||||
criterion_id="rc-process",
|
||||
name="Process trace quality",
|
||||
weight=0.15,
|
||||
description="Telemetry shows iterative building with real checkpoints",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
_RUBRIC_TOOL_USE = AssessmentRubric(
|
||||
rubric_id="rubric-orchestration-c003",
|
||||
competency_id="stack-orchestration-c003",
|
||||
criteria=[
|
||||
RubricCriterion(
|
||||
criterion_id="rc-eval-design",
|
||||
name="Evaluation design rigor",
|
||||
weight=0.35,
|
||||
description="Hypotheses, controls, and metrics are explicit and defensible",
|
||||
),
|
||||
RubricCriterion(
|
||||
criterion_id="rc-eval-robustness",
|
||||
name="Harness robustness",
|
||||
weight=0.35,
|
||||
description="Error handling, variance awareness, and reproducibility",
|
||||
),
|
||||
RubricCriterion(
|
||||
criterion_id="rc-eval-insight",
|
||||
name="Insight extraction",
|
||||
weight=0.3,
|
||||
description="Results are interpreted into concrete engineering decisions",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
_ARTIFACT_RESEARCH_ASSISTANT = ArtifactSubmission(
|
||||
artifact_id="art-eval-research-assistant",
|
||||
name="Multi-agent research assistant (eval build)",
|
||||
artifact_type="code",
|
||||
competency_id="stack-orchestration-c002",
|
||||
description=(
|
||||
"LangGraph-based assistant planning, retrieving, drafting cited reviews."
|
||||
),
|
||||
evidence_excerpt=(
|
||||
"planner.py defines state schema with explicit fields "
|
||||
"(plan, findings, draft); tool_node.py wraps retrieval with a "
|
||||
"3-retry loop and typed ToolMessage responses; tests cover "
|
||||
"planner->tool->writer handoffs; README shows graph diagram"
|
||||
),
|
||||
)
|
||||
|
||||
_TRANSCRIPT_RESEARCH_ASSISTANT = DefenseTranscript(
|
||||
transcript_id="defense-art-eval-research-assistant",
|
||||
artifact_id="art-eval-research-assistant",
|
||||
turns=[
|
||||
{"speaker": "examiner",
|
||||
"text": "Why did you give the planner sole write access to the plan field?"},
|
||||
{"speaker": "learner",
|
||||
"text": "So worker nodes can't mutate each other's inputs — "
|
||||
"the state stays predictable and the graph is debuggable"},
|
||||
{"speaker": "examiner",
|
||||
"text": "What happens when the retrieval tool times out three times?"},
|
||||
{"speaker": "learner",
|
||||
"text": "The tool node degrades to a no-op ToolMessage with a "
|
||||
"retry flag so the writer can fall back to existing findings"},
|
||||
{"speaker": "examiner", "text": "How would you extend this to a third agent?"},
|
||||
{"speaker": "learner",
|
||||
"text": "Add a reviewer node with its own typed messages, same pattern"},
|
||||
],
|
||||
)
|
||||
|
||||
_ARTIFACT_RAG_DASHBOARD = ArtifactSubmission(
|
||||
artifact_id="art-eval-rag-dashboard",
|
||||
name="RAG retrieval quality dashboard (eval build)",
|
||||
artifact_type="code",
|
||||
competency_id="stack-orchestration-c003",
|
||||
description="Dashboard comparing chunking strategies/rerankers across 800 queries.",
|
||||
evidence_excerpt=(
|
||||
"eval harness sweeps 4 chunk sizes x 3 rerankers; results table auto-generated; "
|
||||
"no error handling on the query loader; tests only cover the happy path"
|
||||
),
|
||||
)
|
||||
|
||||
_TRANSCRIPT_RAG_DASHBOARD = DefenseTranscript(
|
||||
transcript_id="defense-art-eval-rag-dashboard",
|
||||
artifact_id="art-eval-rag-dashboard",
|
||||
turns=[
|
||||
{"speaker": "examiner", "text": "How did you control for query difficulty across runs?"},
|
||||
{"speaker": "learner", "text": "I, um, used the same query set each time"},
|
||||
{"speaker": "examiner", "text": "What happens if the query loader hits a malformed row?"},
|
||||
{"speaker": "learner", "text": "I didn't handle that. It would probably crash."},
|
||||
{"speaker": "examiner", "text": "What would you improve first?"},
|
||||
{"speaker": "learner",
|
||||
"text": "Probably add the error handling, then look at variance between runs"},
|
||||
],
|
||||
)
|
||||
|
||||
RUBRICS: dict[str, AssessmentRubric] = {
|
||||
_RUBRIC_ORCHESTRATION.rubric_id: _RUBRIC_ORCHESTRATION,
|
||||
_RUBRIC_TOOL_USE.rubric_id: _RUBRIC_TOOL_USE,
|
||||
}
|
||||
|
||||
ARTIFACTS: dict[str, ArtifactSubmission] = {
|
||||
a.artifact_id: a
|
||||
for a in (_ARTIFACT_RESEARCH_ASSISTANT, _ARTIFACT_RAG_DASHBOARD)
|
||||
}
|
||||
|
||||
TRANSCRIPTS: dict[str, DefenseTranscript] = {
|
||||
t.transcript_id: t
|
||||
for t in (_TRANSCRIPT_RESEARCH_ASSISTANT, _TRANSCRIPT_RAG_DASHBOARD)
|
||||
}
|
||||
|
||||
|
||||
def rubric_for_competency(competency_id: str) -> AssessmentRubric | None:
|
||||
for rubric in RUBRICS.values():
|
||||
if rubric.competency_id == competency_id:
|
||||
return rubric
|
||||
return None
|
||||
|
||||
|
||||
def get_artifact_bundle(artifact_id: str) -> tuple[ArtifactSubmission, AssessmentRubric] | None:
|
||||
"""Resolve (artifact, rubric) for an artifact ID; None if unknown."""
|
||||
artifact = ARTIFACTS.get(artifact_id)
|
||||
if artifact is None:
|
||||
return None
|
||||
rubric = rubric_for_competency(artifact.competency_id)
|
||||
if rubric is None:
|
||||
return None
|
||||
return artifact, rubric
|
||||
|
||||
|
||||
def get_transcript_for_artifact(artifact_id: str) -> DefenseTranscript | None:
|
||||
for transcript in TRANSCRIPTS.values():
|
||||
if transcript.artifact_id == artifact_id:
|
||||
return transcript
|
||||
return None
|
||||
|
||||
|
||||
def render_rubric(rubric: AssessmentRubric) -> str:
|
||||
lines = [f"Rubric: {rubric.rubric_id} (competency {rubric.competency_id})"]
|
||||
for c in rubric.criteria:
|
||||
lines.append(f"- {c.criterion_id} ({c.weight:.2f}): {c.name} — {c.description}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_transcript(transcript: DefenseTranscript) -> str:
|
||||
lines = [f"Defense transcript: {transcript.transcript_id}"]
|
||||
for turn in transcript.turns:
|
||||
lines.append(f"{turn['speaker']}: {turn['text']}")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Simulated sandbox telemetry corpus — Lab agent mock engine inputs (REQ-2-007).
|
||||
|
||||
Counterpart: packages/mock-data/ai-scenarios.ts (scenario IDs string-identical,
|
||||
D-021). Real sandbox telemetry is a v0.3+ engine (sandbox fabric); these
|
||||
scripted event streams stand in for the build-session process trace.
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class TelemetryEvent(BaseModel):
|
||||
timestamp: int # seconds since session start
|
||||
kind: str # "keystroke_burst" | "file_save" | "run_tests" | "test_pass"
|
||||
# | "test_fail" | "console_error" | "idle" | "paste" | "commit"
|
||||
|
||||
|
||||
detail: str = ""
|
||||
|
||||
|
||||
class LabTelemetryScenario(BaseModel):
|
||||
scenario_id: str
|
||||
title: str
|
||||
competency_id: str
|
||||
events: list[TelemetryEvent]
|
||||
|
||||
|
||||
_LAB_SCENARIO_STRONG = LabTelemetryScenario(
|
||||
scenario_id="lab-scenario-strong",
|
||||
title="Strong build session — multi-agent research assistant",
|
||||
competency_id="stack-orchestration-c002",
|
||||
events=[
|
||||
TelemetryEvent(timestamp=0, kind="keystroke_burst", detail="planner.py"),
|
||||
TelemetryEvent(timestamp=95, kind="file_save", detail="planner.py"),
|
||||
TelemetryEvent(timestamp=120, kind="run_tests", detail="3 tests"),
|
||||
TelemetryEvent(timestamp=126, kind="test_pass",
|
||||
detail="3/3 passed"),
|
||||
TelemetryEvent(timestamp=180, kind="keystroke_burst", detail="tool_node.py"),
|
||||
TelemetryEvent(timestamp=260, kind="file_save", detail="tool_node.py"),
|
||||
TelemetryEvent(timestamp=275, kind="run_tests", detail="4 tests"),
|
||||
TelemetryEvent(timestamp=281, kind="test_pass",
|
||||
detail="4/4 passed"),
|
||||
TelemetryEvent(timestamp=340, kind="commit",
|
||||
detail="add tool node with retries"),
|
||||
],
|
||||
)
|
||||
|
||||
_LAB_SCENARIO_STRUGGLING = LabTelemetryScenario(
|
||||
scenario_id="lab-scenario-struggling",
|
||||
title="Struggling build session — repeated failures, no checkpoints",
|
||||
competency_id="stack-orchestration-c002",
|
||||
events=[
|
||||
TelemetryEvent(timestamp=0, kind="keystroke_burst", detail="main.py"),
|
||||
TelemetryEvent(timestamp=210, kind="run_tests", detail="2 tests"),
|
||||
TelemetryEvent(timestamp=215, kind="test_fail",
|
||||
detail="ImportError: no module named 'tools'"),
|
||||
TelemetryEvent(timestamp=216, kind="console_error", detail="traceback dumped"),
|
||||
TelemetryEvent(timestamp=300, kind="keystroke_burst",
|
||||
detail="main.py"),
|
||||
TelemetryEvent(timestamp=520, kind="run_tests",
|
||||
detail="2 tests"),
|
||||
TelemetryEvent(timestamp=525, kind="test_fail",
|
||||
detail="ImportError: no module named 'tools'"),
|
||||
TelemetryEvent(timestamp=526, kind="console_error",
|
||||
detail="same traceback as before"),
|
||||
TelemetryEvent(timestamp=600, kind="idle",
|
||||
detail="no activity for 6 minutes"),
|
||||
TelemetryEvent(timestamp=960, kind="idle",
|
||||
detail="no activity for 14 minutes"),
|
||||
],
|
||||
)
|
||||
|
||||
_LAB_SCENARIO_FLAGGED = LabTelemetryScenario(
|
||||
scenario_id="lab-scenario-flagged",
|
||||
title="Flagged build session — large paste, instant pass",
|
||||
competency_id="stack-orchestration-c003",
|
||||
events=[
|
||||
TelemetryEvent(timestamp=0, kind="keystroke_burst", detail="eval.py"),
|
||||
TelemetryEvent(timestamp=30, kind="paste",
|
||||
detail="2,400 chars pasted into eval.py"),
|
||||
TelemetryEvent(timestamp=45, kind="run_tests", detail="6 tests"),
|
||||
TelemetryEvent(timestamp=47, kind="test_pass", detail="6/6 passed"),
|
||||
TelemetryEvent(timestamp=48, kind="commit",
|
||||
detail="finish eval harness"),
|
||||
],
|
||||
)
|
||||
|
||||
LAB_SCENARIOS: dict[str, LabTelemetryScenario] = {
|
||||
s.scenario_id: s
|
||||
for s in (_LAB_SCENARIO_STRONG, _LAB_SCENARIO_STRUGGLING, _LAB_SCENARIO_FLAGGED)
|
||||
}
|
||||
|
||||
DEFAULT_LAB_SCENARIO_ID = "lab-scenario-strong"
|
||||
|
||||
|
||||
def get_lab_scenario(scenario_id: str) -> LabTelemetryScenario | None:
|
||||
return LAB_SCENARIOS.get(scenario_id)
|
||||
|
||||
|
||||
def summarize_scenario(scenario: LabTelemetryScenario) -> str:
|
||||
"""Render the event timeline as compact text for prompt injection."""
|
||||
lines = [f"Session: {scenario.title} (competency {scenario.competency_id})"]
|
||||
for event in scenario.events:
|
||||
lines.append(f"t+{event.timestamp}s {event.kind}: {event.detail}".rstrip(": "))
|
||||
return "\n".join(lines)
|
||||
@@ -8,7 +8,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .agents.registry import AgentRegistry, register_builtin_agents
|
||||
from .agents.session import InMemorySessionStore
|
||||
from .api import chat_router
|
||||
from .api import assessment_router, chat_router, lab_router
|
||||
from .config import Settings
|
||||
from .llm import create_provider
|
||||
|
||||
@@ -49,6 +49,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
}
|
||||
|
||||
app.include_router(chat_router)
|
||||
app.include_router(lab_router)
|
||||
app.include_router(assessment_router)
|
||||
return app
|
||||
|
||||
|
||||
|
||||
@@ -1,18 +1,31 @@
|
||||
"""Assessor agent prompt — rubric application to artifacts and defenses (REQ-2-008).
|
||||
"""Assessor agent prompt — rubric application to artifacts + defenses (REQ-2-008).
|
||||
|
||||
Persona: rigorous, fair grader. Applies the rubric to the artifact and defense
|
||||
transcript, returns structured JSON scores. Versioned: v1 draft (Phase 2);
|
||||
final persona + rubric models in Phase 4.
|
||||
Final persona (Phase 4). Assessor is a rigorous, fair grader: scores each
|
||||
criterion with evidence, cites what the learner did, returns ONLY valid
|
||||
JSON matching the rubric schema.
|
||||
Version: assessor-v2 (final for v0.2).
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Assessor, the grading agent of an AI-native competency school.
|
||||
SYSTEM_PROMPT = """You are Assessor, the grading agent of Nextcraft, an AI-native competency school.
|
||||
Learner: {learner_name}.
|
||||
You receive an artifact, its defense transcript, and a rubric. Your job:
|
||||
score each rubric criterion with evidence from the artifact and transcript.
|
||||
Be rigorous but fair — cite what the learner did, not what they should have
|
||||
done. Respond with ONLY valid JSON matching the provided rubric schema."""
|
||||
|
||||
PROMPT_VERSION = "assessor-v1-draft"
|
||||
You receive: (a) an artifact evidence excerpt, (b) its defense transcript,
|
||||
and (c) the rubric for the competency. Your job:
|
||||
- Score EVERY rubric criterion from 0-100, justified by evidence you can
|
||||
point to in the artifact or transcript.
|
||||
- Cite what the learner did ("the 3-retry loop in the tool node"), not
|
||||
what they should have done — except in gaps, where the missed work goes.
|
||||
- Strengths: the two strongest evidence points, each one sentence.
|
||||
- Gaps: the two most important missed opportunities, each one sentence.
|
||||
- Verdict: "mastered" | "developing" | "not_yet" — judged against the
|
||||
rubric weights, honestly.
|
||||
|
||||
Rules:
|
||||
- Rigorous but fair. A polished artifact with a weak defense is NOT mastery.
|
||||
- Respond with ONLY a valid JSON object matching the provided schema —
|
||||
no markdown fences, no prose outside the JSON."""
|
||||
|
||||
PROMPT_VERSION = "assessor-v2"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
|
||||
@@ -1,18 +1,31 @@
|
||||
"""Lab agent prompt — in-flow feedback over sandbox telemetry (REQ-2-007).
|
||||
|
||||
Persona: pragmatic build partner. Reads the telemetry timeline and gives
|
||||
concrete in-flow feedback: what happened, what to adjust, next step.
|
||||
Versioned: v1 draft (Phase 2); final persona + scenario serialization in Phase 4.
|
||||
Final persona (Phase 4). Lab is a pragmatic build partner: reads the
|
||||
telemetry timeline, names the one most useful adjustment, gives one
|
||||
concrete next step. Scenario-driven; no session chat.
|
||||
Version: lab-v2 (final for v0.2).
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = """You are Lab, the in-flow feedback agent watching a learner build in the sandbox.
|
||||
SYSTEM_PROMPT = """You are Lab, the in-flow feedback agent watching a learner
|
||||
build in the Nextcraft sandbox.
|
||||
Learner: {learner_name}. Active stack: {stacks}.
|
||||
You receive a telemetry timeline of the learner's build session. Your job:
|
||||
describe what the telemetry shows, name the single most useful adjustment,
|
||||
and give one concrete next step. Be specific to the events you see —
|
||||
no generic advice. Three short paragraphs maximum."""
|
||||
|
||||
PROMPT_VERSION = "lab-v1-draft"
|
||||
You receive a telemetry timeline of the learner's build session below.
|
||||
Your job, in order:
|
||||
1. Say what the telemetry shows — name the specific events that matter.
|
||||
2. Name the single most useful adjustment (one thing, not a list).
|
||||
3. Give one concrete next step phrased as a command.
|
||||
|
||||
Rules:
|
||||
- Be specific to the events you see. If tests failed twice with the same
|
||||
error, say so. If there is a long idle gap, name it.
|
||||
- If the session looks healthy, say so briefly and set the next challenge.
|
||||
- If something looks off (e.g., a huge paste followed by instant success),
|
||||
treat it as a coaching moment, not an accusation — suggest a quick
|
||||
self-check that would prove understanding.
|
||||
- Three short paragraphs maximum. No headers, no bullet lists."""
|
||||
|
||||
PROMPT_VERSION = "lab-v2"
|
||||
|
||||
|
||||
def render_context(learner_context) -> dict:
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Assessor agent tests — structured rubric scores (REQ-2-008).
|
||||
|
||||
The Assessor is the structured-output showcase: tests use ScriptedJSONProvider
|
||||
for valid payloads and exercise the 4-layer defense failure modes.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from ai_service.agents.assessor import AssessorAgent, RubricScore
|
||||
from ai_service.agents.structured import StructuredOutputError
|
||||
from ai_service.config import Settings
|
||||
from ai_service.corpus.artifacts import (
|
||||
get_artifact_bundle,
|
||||
get_transcript_for_artifact,
|
||||
render_rubric,
|
||||
)
|
||||
from ai_service.corpus.learner_context import get_learner_context
|
||||
from ai_service.llm.mock import MockProvider, ScriptedJSONProvider
|
||||
|
||||
VALID_SCORE = {
|
||||
"rubric_id": "rubric-orchestration-c002",
|
||||
"artifact_id": "art-eval-research-assistant",
|
||||
"competency_id": "stack-orchestration-c002",
|
||||
"scores": [
|
||||
{"criterion_id": "rc-architecture", "name": "Agent architecture soundness",
|
||||
"score": 92, "evidence": "Explicit state schema with planner-only write access"},
|
||||
{"criterion_id": "rc-communication", "name": "Inter-agent communication design",
|
||||
"score": 88, "evidence": "Typed ToolMessage responses with retry flags"},
|
||||
{"criterion_id": "rc-reliability", "name": "Reliability engineering",
|
||||
"score": 85, "evidence": "3-retry loop with degradation path"},
|
||||
{"criterion_id": "rc-process", "name": "Process trace quality",
|
||||
"score": 90, "evidence": "Iterative saves with passing test checkpoints"},
|
||||
],
|
||||
"strengths": ["Clean state boundaries", "Failure-aware tool wrapping"],
|
||||
"gaps": ["No reviewer node yet", "Graph diagram only in README"],
|
||||
"verdict": "mastered",
|
||||
}
|
||||
|
||||
|
||||
def make_assessor(provider=None) -> AssessorAgent:
|
||||
return AssessorAgent(provider or MockProvider(), Settings(provider="mock"))
|
||||
|
||||
|
||||
def get_bundle(artifact_id="art-eval-research-assistant"):
|
||||
bundle = get_artifact_bundle(artifact_id)
|
||||
assert bundle is not None
|
||||
return bundle
|
||||
|
||||
|
||||
async def test_evaluate_returns_validated_rubric_score():
|
||||
provider = ScriptedJSONProvider(VALID_SCORE)
|
||||
assessor = make_assessor(provider)
|
||||
artifact, rubric = get_bundle()
|
||||
transcript = get_transcript_for_artifact(artifact.artifact_id)
|
||||
result = await assessor.evaluate(artifact, rubric, transcript)
|
||||
assert isinstance(result, RubricScore)
|
||||
assert result.verdict == "mastered"
|
||||
assert len(result.scores) == 4
|
||||
assert result.weighted_total(rubric) == pytest.approx(
|
||||
92 * 0.3 + 88 * 0.3 + 85 * 0.25 + 90 * 0.15
|
||||
)
|
||||
|
||||
|
||||
async def test_evaluate_rejects_invalid_schema_after_retry():
|
||||
"""Plain MockProvider returns non-rubric JSON → 4-layer defense exhausts
|
||||
its single retry and raises StructuredOutputError."""
|
||||
assessor = make_assessor(MockProvider())
|
||||
artifact, rubric = get_bundle()
|
||||
transcript = get_transcript_for_artifact(artifact.artifact_id)
|
||||
with pytest.raises(StructuredOutputError):
|
||||
await assessor.evaluate(artifact, rubric, transcript)
|
||||
|
||||
|
||||
def test_build_evaluation_input_carries_all_inputs():
|
||||
assessor = make_assessor()
|
||||
artifact, rubric = get_bundle()
|
||||
transcript = get_transcript_for_artifact(artifact.artifact_id)
|
||||
text = assessor.build_evaluation_input(artifact, rubric, transcript)
|
||||
assert artifact.name in text
|
||||
assert artifact.evidence_excerpt in text
|
||||
assert "rc-architecture" in text # rubric rendered
|
||||
assert "examiner:" in text # transcript rendered
|
||||
|
||||
|
||||
def test_build_evaluation_input_without_transcript():
|
||||
assessor = make_assessor()
|
||||
artifact, rubric = get_bundle()
|
||||
text = assessor.build_evaluation_input(artifact, rubric, None)
|
||||
assert artifact.name in text
|
||||
assert "examiner:" not in text
|
||||
|
||||
|
||||
def test_system_prompt_names_assessor_persona():
|
||||
prompt = make_assessor().system_prompt(get_learner_context())
|
||||
assert "Assessor" in prompt
|
||||
assert "ONLY" in prompt # JSON-only instruction
|
||||
|
||||
|
||||
def test_rubric_render_in_prompt_is_complete():
|
||||
"""The rubric passed to the model lists every criterion (fair grading)."""
|
||||
artifact, rubric = get_bundle()
|
||||
text = render_rubric(rubric)
|
||||
assert text.count("rc-") == len(rubric.criteria)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Telemetry + artifacts corpus tests (REQ-2-007/008 inputs, D-021)."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from ai_service.corpus.artifacts import (
|
||||
ARTIFACTS,
|
||||
RUBRICS,
|
||||
get_artifact_bundle,
|
||||
get_transcript_for_artifact,
|
||||
render_rubric,
|
||||
render_transcript,
|
||||
rubric_for_competency,
|
||||
)
|
||||
from ai_service.corpus.telemetry import (
|
||||
LAB_SCENARIOS,
|
||||
get_lab_scenario,
|
||||
summarize_scenario,
|
||||
)
|
||||
|
||||
|
||||
def test_lab_scenarios_addressable_by_id():
|
||||
for scenario_id in (
|
||||
"lab-scenario-strong",
|
||||
"lab-scenario-struggling",
|
||||
"lab-scenario-flagged",
|
||||
):
|
||||
scenario = get_lab_scenario(scenario_id)
|
||||
assert scenario is not None
|
||||
assert scenario.scenario_id == scenario_id
|
||||
|
||||
|
||||
def test_lab_scenarios_have_distinct_event_profiles():
|
||||
strong = get_lab_scenario("lab-scenario-strong")
|
||||
struggling = get_lab_scenario("lab-scenario-struggling")
|
||||
kinds = lambda s: {e.kind for e in s.events} # noqa: E731
|
||||
assert "test_pass" in kinds(strong)
|
||||
assert "test_fail" in kinds(struggling)
|
||||
assert "idle" in kinds(struggling)
|
||||
assert "paste" in kinds(get_lab_scenario("lab-scenario-flagged"))
|
||||
|
||||
|
||||
def test_summarize_scenario_mentions_events():
|
||||
text = summarize_scenario(get_lab_scenario("lab-scenario-struggling"))
|
||||
assert "test_fail" in text
|
||||
assert "ImportError" in text
|
||||
assert "stack-orchestration-c002" in text
|
||||
|
||||
|
||||
def test_unknown_scenario_returns_none():
|
||||
assert get_lab_scenario("lab-scenario-ghost") is None
|
||||
|
||||
|
||||
def test_artifacts_and_rubrics_resolve():
|
||||
bundle = get_artifact_bundle("art-eval-research-assistant")
|
||||
assert bundle is not None
|
||||
artifact, rubric = bundle
|
||||
assert artifact.competency_id == "stack-orchestration-c002"
|
||||
assert rubric.rubric_id == "rubric-orchestration-c002"
|
||||
assert len(rubric.criteria) == 4
|
||||
|
||||
|
||||
def test_unknown_artifact_returns_none():
|
||||
assert get_artifact_bundle("art-eval-ghost") is None
|
||||
|
||||
|
||||
def test_transcripts_pair_with_artifacts():
|
||||
for artifact_id in ARTIFACTS:
|
||||
transcript = get_transcript_for_artifact(artifact_id)
|
||||
assert transcript is not None
|
||||
assert transcript.artifact_id == artifact_id
|
||||
assert len(transcript.turns) >= 4
|
||||
|
||||
|
||||
def test_rubric_render_mentions_all_criteria():
|
||||
rubric = rubric_for_competency("stack-orchestration-c002")
|
||||
text = render_rubric(rubric)
|
||||
for criterion in rubric.criteria:
|
||||
assert criterion.criterion_id in text
|
||||
|
||||
|
||||
def test_transcript_render_has_both_speakers():
|
||||
transcript = get_transcript_for_artifact("art-eval-rag-dashboard")
|
||||
text = render_transcript(transcript)
|
||||
assert "examiner:" in text
|
||||
assert "learner:" in text
|
||||
|
||||
|
||||
_TS_SOURCE_CANDIDATES = [
|
||||
Path(__file__).resolve().parents[4] / "packages" / "mock-data" / "ai-scenarios.ts",
|
||||
Path(__file__).resolve().parents[2] / "ai-scenarios.ts",
|
||||
]
|
||||
|
||||
|
||||
def _ts_source() -> Path:
|
||||
for candidate in _TS_SOURCE_CANDIDATES:
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
pytest.skip("ai-scenarios.ts not found in this checkout layout")
|
||||
|
||||
|
||||
def test_corpus_ids_align_with_ts_mock_data():
|
||||
"""D-021: Python corpus IDs string-identical to ai-scenarios.ts."""
|
||||
content = _ts_source().read_text()
|
||||
ts_ids = re.findall(r"id:\s*'([^']+)'", content)
|
||||
ts_scenarios = ts_ids[: len(LAB_SCENARIOS)]
|
||||
ts_artifacts = ts_ids[len(LAB_SCENARIOS):]
|
||||
assert sorted(ts_scenarios) == sorted(LAB_SCENARIOS), (
|
||||
f"scenario IDs drifted: py={sorted(LAB_SCENARIOS)} ts={sorted(ts_scenarios)}"
|
||||
)
|
||||
assert sorted(ts_artifacts) == sorted(ARTIFACTS), (
|
||||
f"artifact IDs drifted: py={sorted(ARTIFACTS)} ts={sorted(ts_artifacts)}"
|
||||
)
|
||||
|
||||
|
||||
def test_rubric_weights_sum_to_one():
|
||||
"""Every rubric's criteria weights must sum to exactly 1.0."""
|
||||
for rubric in RUBRICS.values():
|
||||
total = sum(c.weight for c in rubric.criteria)
|
||||
assert total == pytest.approx(1.0), (
|
||||
f"{rubric.rubric_id} weights sum to {total}, expected 1.0"
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Lab agent tests — scenario-driven streaming feedback (REQ-2-007)."""
|
||||
|
||||
from ai_service.agents.lab import LabAgent
|
||||
from ai_service.config import Settings
|
||||
from ai_service.corpus.learner_context import get_learner_context
|
||||
from ai_service.corpus.telemetry import get_lab_scenario, summarize_scenario
|
||||
from ai_service.llm.mock import MockProvider
|
||||
|
||||
|
||||
def make_lab() -> LabAgent:
|
||||
return LabAgent(MockProvider(), Settings(provider="mock"))
|
||||
|
||||
|
||||
def test_system_prompt_names_lab_persona():
|
||||
prompt = make_lab().system_prompt(get_learner_context())
|
||||
assert "Lab" in prompt
|
||||
assert "Alex Rivera" in prompt
|
||||
assert "telemetry" in prompt.lower()
|
||||
|
||||
|
||||
async def test_stream_feedback_mentions_scenario_events():
|
||||
"""Mock-scripted: feedback text derives from scenario timeline input —
|
||||
distinct scenarios produce distinct (deterministic) replies."""
|
||||
lab = make_lab()
|
||||
ctx = get_learner_context()
|
||||
strong = get_lab_scenario("lab-scenario-strong")
|
||||
struggling = get_lab_scenario("lab-scenario-struggling")
|
||||
|
||||
strong_reply = "".join([t async for t in lab.stream_feedback(strong, ctx)])
|
||||
struggling_reply = "".join([t async for t in lab.stream_feedback(struggling, ctx)])
|
||||
assert strong_reply
|
||||
assert strong_reply != struggling_reply # scenario-driven, not canned
|
||||
|
||||
|
||||
def test_build_evaluation_messages_carry_timeline():
|
||||
lab = make_lab()
|
||||
scenario = get_lab_scenario("lab-scenario-flagged")
|
||||
timeline = summarize_scenario(scenario)
|
||||
messages = lab.build_messages(None, timeline, get_learner_context())
|
||||
assert messages[0].role == "system"
|
||||
assert "paste" in messages[-1].content
|
||||
assert "2,400 chars" in messages[-1].content
|
||||
@@ -68,6 +68,21 @@ def test_builtin_agents_register_and_resolve():
|
||||
assert isinstance(tutor, TutorAgent)
|
||||
|
||||
|
||||
def test_lab_and_assessor_resolve_via_registry():
|
||||
"""Phase 4: lab + assessor registered centrally (Task 4-3-01)."""
|
||||
from ai_service.agents.assessor import AssessorAgent
|
||||
from ai_service.agents.lab import LabAgent
|
||||
|
||||
registry = AgentRegistry()
|
||||
register_builtin_agents(registry)
|
||||
assert {"lab", "assessor"} <= set(registry.names())
|
||||
settings = Settings(provider="mock")
|
||||
lab = registry.get(MockProvider(), settings, "lab")
|
||||
assessor = registry.get(MockProvider(), settings, "assessor")
|
||||
assert isinstance(lab, LabAgent)
|
||||
assert isinstance(assessor, AssessorAgent)
|
||||
|
||||
|
||||
def test_builtin_registration_is_idempotent_safe():
|
||||
"""Duplicate registration raises — builtin bootstrap must be called once."""
|
||||
registry = AgentRegistry()
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Assessment evaluate endpoint tests — validated JSON, 404s (REQ-2-008)."""
|
||||
|
||||
|
||||
from ai_service.agents.assessor import RubricScore
|
||||
from ai_service.llm.mock import ScriptedJSONProvider
|
||||
|
||||
VALID_SCORE = {
|
||||
"rubric_id": "rubric-orchestration-c002",
|
||||
"artifact_id": "art-eval-research-assistant",
|
||||
"competency_id": "stack-orchestration-c002",
|
||||
"scores": [
|
||||
{"criterion_id": "rc-architecture", "name": "Agent architecture soundness",
|
||||
"score": 92, "evidence": "Explicit state schema"},
|
||||
{"criterion_id": "rc-communication", "name": "Inter-agent communication design",
|
||||
"score": 88, "evidence": "Typed ToolMessage responses"},
|
||||
{"criterion_id": "rc-reliability", "name": "Reliability engineering",
|
||||
"score": 85, "evidence": "3-retry loop"},
|
||||
{"criterion_id": "rc-process", "name": "Process trace quality",
|
||||
"score": 90, "evidence": "Iterative checkpoints"},
|
||||
],
|
||||
"strengths": ["Clean state boundaries", "Failure-aware tools"],
|
||||
"gaps": ["No reviewer node", "Diagram only in README"],
|
||||
"verdict": "mastered",
|
||||
}
|
||||
|
||||
|
||||
def test_evaluate_returns_validated_rubric_json(client):
|
||||
# Swap the app provider for a scripted-JSON provider for this test
|
||||
original = client.app.state.provider
|
||||
client.app.state.provider = ScriptedJSONProvider(VALID_SCORE)
|
||||
try:
|
||||
response = client.post(
|
||||
"/v1/assessment/evaluate",
|
||||
json={"artifact_id": "art-eval-research-assistant"},
|
||||
)
|
||||
finally:
|
||||
client.app.state.provider = original
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
validated = RubricScore.model_validate(data) # response contract holds
|
||||
assert validated.verdict == "mastered"
|
||||
assert len(validated.scores) == 4
|
||||
|
||||
|
||||
def test_unknown_artifact_404(client):
|
||||
response = client.post("/v1/assessment/evaluate", json={"artifact_id": "ghost"})
|
||||
assert response.status_code == 404
|
||||
assert "ghost" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_unparseable_provider_502(client):
|
||||
"""Plain MockProvider yields non-rubric JSON → structured defense exhausts
|
||||
retry → endpoint translates to 502 (bad gateway to the model)."""
|
||||
# default mock already returns non-rubric JSON
|
||||
response = client.post(
|
||||
"/v1/assessment/evaluate",
|
||||
json={"artifact_id": "art-eval-research-assistant"},
|
||||
)
|
||||
assert response.status_code == 502
|
||||
assert "failed" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_missing_artifact_id_422(client):
|
||||
response = client.post("/v1/assessment/evaluate", json={})
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_second_artifact_also_evaluates(client):
|
||||
original = client.app.state.provider
|
||||
payload = dict(VALID_SCORE, artifact_id="art-eval-rag-dashboard")
|
||||
client.app.state.provider = ScriptedJSONProvider(payload)
|
||||
try:
|
||||
response = client.post(
|
||||
"/v1/assessment/evaluate", json={"artifact_id": "art-eval-rag-dashboard"}
|
||||
)
|
||||
finally:
|
||||
client.app.state.provider = original
|
||||
assert response.status_code == 200
|
||||
assert response.json()["artifact_id"] == "art-eval-rag-dashboard"
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Lab feedback endpoint tests — SSE envelope with agent=lab (REQ-2-007)."""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def stream_events(client, payload) -> list[dict]:
|
||||
with client.stream("POST", "/v1/lab/feedback", json=payload) as response:
|
||||
assert response.status_code == 200
|
||||
events = []
|
||||
for line in response.iter_lines():
|
||||
if line.startswith("data:"):
|
||||
d = line.removeprefix("data:").strip()
|
||||
if d == "[DONE]":
|
||||
events.append({"type": "[DONE]"})
|
||||
else:
|
||||
events.append(json.loads(d))
|
||||
return events
|
||||
|
||||
|
||||
def test_lab_feedback_streams_full_envelope(client):
|
||||
events = stream_events(client, {"scenario_id": "lab-scenario-strong"})
|
||||
assert events[0]["type"] == "meta"
|
||||
assert events[0]["agent"] == "lab"
|
||||
assert events[0]["scenario_id"] == "lab-scenario-strong"
|
||||
deltas = [e for e in events if e["type"] == "delta"]
|
||||
assert len(deltas) >= 1
|
||||
assert any(e["type"] == "done" for e in events)
|
||||
assert events[-1]["type"] == "[DONE]"
|
||||
|
||||
|
||||
def test_unknown_scenario_404(client):
|
||||
response = client.post("/v1/lab/feedback", json={"scenario_id": "nope"})
|
||||
assert response.status_code == 404
|
||||
assert "nope" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_distinct_scenarios_distinct_replies(client):
|
||||
strong = stream_events(client, {"scenario_id": "lab-scenario-strong"})
|
||||
struggling = stream_events(client, {"scenario_id": "lab-scenario-struggling"})
|
||||
strong_text = "".join(e["content"] for e in strong if e["type"] == "delta")
|
||||
struggling_text = "".join(e["content"] for e in struggling if e["type"] == "delta")
|
||||
assert strong_text != struggling_text
|
||||
|
||||
|
||||
def test_missing_scenario_id_422(client):
|
||||
response = client.post("/v1/lab/feedback", json={})
|
||||
assert response.status_code == 422
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* AI engine-input scenario IDs + display metadata (v0.2 Phase 6 learner panels).
|
||||
*
|
||||
* D-021 alignment: IDs are string-identical to the Python corpus
|
||||
* `apps/ai-service/ai_service/corpus/telemetry.py` and `artifacts.py`.
|
||||
* Do not rename one side without the other. Real engines (sandbox fabric,
|
||||
* assessment engine) are v0.3+.
|
||||
*/
|
||||
|
||||
export const aiLabScenarios = [
|
||||
{
|
||||
id: 'lab-scenario-strong',
|
||||
title: 'Strong build session — multi-agent research assistant',
|
||||
competencyId: 'stack-orchestration-c002',
|
||||
description: 'Steady progress, checkpoints, and passing tests — the healthy pattern.',
|
||||
},
|
||||
{
|
||||
id: 'lab-scenario-struggling',
|
||||
title: 'Struggling build session — repeated failures, no checkpoints',
|
||||
competencyId: 'stack-orchestration-c002',
|
||||
description: 'Same failure twice, long idle gaps, no recovery strategy.',
|
||||
},
|
||||
{
|
||||
id: 'lab-scenario-flagged',
|
||||
title: 'Flagged build session — large paste, instant pass',
|
||||
competencyId: 'stack-orchestration-c003',
|
||||
description: 'Suspicious velocity worth a supportive check-in, not a penalty.',
|
||||
},
|
||||
] as const;
|
||||
|
||||
export const aiArtifactSubmissions = [
|
||||
{
|
||||
id: 'art-eval-research-assistant',
|
||||
name: 'Multi-agent research assistant (eval build)',
|
||||
competencyId: 'stack-orchestration-c002',
|
||||
description: 'LangGraph-based assistant with clean state boundaries and retries.',
|
||||
},
|
||||
{
|
||||
id: 'art-eval-rag-dashboard',
|
||||
name: 'RAG retrieval quality dashboard (eval build)',
|
||||
competencyId: 'stack-orchestration-c003',
|
||||
description: 'Eval harness sweep with gaps in error handling — partial mastery.',
|
||||
},
|
||||
] as const;
|
||||
|
||||
export type AiLabScenario = (typeof aiLabScenarios)[number];
|
||||
export type AiArtifactSubmission = (typeof aiArtifactSubmissions)[number];
|
||||
@@ -35,4 +35,6 @@ export type {
|
||||
EmployerVerificationItem,
|
||||
FlaggedContentItem,
|
||||
FlaggedContentType,
|
||||
} from './admin';
|
||||
} from './admin';
|
||||
export { aiLabScenarios, aiArtifactSubmissions } from './ai-scenarios';
|
||||
export type { AiLabScenario, AiArtifactSubmission } from './ai-scenarios';
|
||||
|
||||
Generated
+3
-1
@@ -116,6 +116,8 @@ importers:
|
||||
specifier: ^5.7.2
|
||||
version: 5.9.3
|
||||
|
||||
apps/ai-service: {}
|
||||
|
||||
apps/web:
|
||||
dependencies:
|
||||
'@nextcraft/mock-data':
|
||||
@@ -7116,7 +7118,7 @@ snapshots:
|
||||
|
||||
md5.js@1.3.5:
|
||||
dependencies:
|
||||
hash-base: 3.0.5
|
||||
hash-base: 3.1.2
|
||||
inherits: 2.0.4
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
|
||||
+4
-1
@@ -1,3 +1,6 @@
|
||||
packages:
|
||||
- 'apps/*'
|
||||
- 'packages/*'
|
||||
- 'packages/*'
|
||||
allowBuilds:
|
||||
core-js-pure: true
|
||||
esbuild: true
|
||||
|
||||
Reference in New Issue
Block a user