CIAgent
88a1dab810
---ci--- phase: 7 milestone: v0.2 status: complete requirements: covered: [REQ-2-001, REQ-2-002, REQ-2-003, REQ-2-004, REQ-2-005, REQ-2-006, REQ-2-007, REQ-2-008, REQ-2-009, REQ-2-010, REQ-2-011, REQ-2-012] partial: [] ---/ci--- Milestone v0.2 (ai-tutor-architecture) merged to main. Escalation record (audit remediation, durable): P1 executor delegation failed twice (empty subagent results, zero files created); auto-resolved at full autonomy to inline execution with identical plan fidelity (commit 3271373, reflog-only after phase branch squash-delete).
80 lines
3.0 KiB
Python
80 lines
3.0 KiB
Python
"""Assessment evaluate endpoint tests — validated JSON, 404s (REQ-2-008)."""
|
|
|
|
|
|
from ai_service.agents.assessor import RubricScore
|
|
from ai_service.llm.mock import ScriptedJSONProvider
|
|
|
|
VALID_SCORE = {
|
|
"rubric_id": "rubric-orchestration-c002",
|
|
"artifact_id": "art-eval-research-assistant",
|
|
"competency_id": "stack-orchestration-c002",
|
|
"scores": [
|
|
{"criterion_id": "rc-architecture", "name": "Agent architecture soundness",
|
|
"score": 92, "evidence": "Explicit state schema"},
|
|
{"criterion_id": "rc-communication", "name": "Inter-agent communication design",
|
|
"score": 88, "evidence": "Typed ToolMessage responses"},
|
|
{"criterion_id": "rc-reliability", "name": "Reliability engineering",
|
|
"score": 85, "evidence": "3-retry loop"},
|
|
{"criterion_id": "rc-process", "name": "Process trace quality",
|
|
"score": 90, "evidence": "Iterative checkpoints"},
|
|
],
|
|
"strengths": ["Clean state boundaries", "Failure-aware tools"],
|
|
"gaps": ["No reviewer node", "Diagram only in README"],
|
|
"verdict": "mastered",
|
|
}
|
|
|
|
|
|
def test_evaluate_returns_validated_rubric_json(client):
|
|
# Swap the app provider for a scripted-JSON provider for this test
|
|
original = client.app.state.provider
|
|
client.app.state.provider = ScriptedJSONProvider(VALID_SCORE)
|
|
try:
|
|
response = client.post(
|
|
"/v1/assessment/evaluate",
|
|
json={"artifact_id": "art-eval-research-assistant"},
|
|
)
|
|
finally:
|
|
client.app.state.provider = original
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
validated = RubricScore.model_validate(data) # response contract holds
|
|
assert validated.verdict == "mastered"
|
|
assert len(validated.scores) == 4
|
|
|
|
|
|
def test_unknown_artifact_404(client):
|
|
response = client.post("/v1/assessment/evaluate", json={"artifact_id": "ghost"})
|
|
assert response.status_code == 404
|
|
assert "ghost" in response.json()["detail"]
|
|
|
|
|
|
def test_unparseable_provider_502(client):
|
|
"""Plain MockProvider yields non-rubric JSON → structured defense exhausts
|
|
retry → endpoint translates to 502 (bad gateway to the model)."""
|
|
# default mock already returns non-rubric JSON
|
|
response = client.post(
|
|
"/v1/assessment/evaluate",
|
|
json={"artifact_id": "art-eval-research-assistant"},
|
|
)
|
|
assert response.status_code == 502
|
|
assert "failed" in response.json()["detail"].lower()
|
|
|
|
|
|
def test_missing_artifact_id_422(client):
|
|
response = client.post("/v1/assessment/evaluate", json={})
|
|
assert response.status_code == 422
|
|
|
|
|
|
def test_second_artifact_also_evaluates(client):
|
|
original = client.app.state.provider
|
|
payload = dict(VALID_SCORE, artifact_id="art-eval-rag-dashboard")
|
|
client.app.state.provider = ScriptedJSONProvider(payload)
|
|
try:
|
|
response = client.post(
|
|
"/v1/assessment/evaluate", json={"artifact_id": "art-eval-rag-dashboard"}
|
|
)
|
|
finally:
|
|
client.app.state.provider = original
|
|
assert response.status_code == 200
|
|
assert response.json()["artifact_id"] == "art-eval-rag-dashboard"
|