CIAgent
007865a5a1
Task 5-4-01: tests/voice/test_latency.py — instrumentation presence + population
(stt_ms/llm_ms/tts_ms in every answer response; latency_ms on every persisted turn);
DEFENSE_TURN_BUDGET_MS=4s named; mock turns within budget. README: voice mock-first
section — real server STT/TTS deferred to v0.4 (CUT-1/G-7), AI_VOICE_PROVIDER modes,
the v0.4 wall-clock acceptance probe (manual, keys in .ciagent/.env.secrets only).
Suite 383 green; ruff clean.
---ci---
phase: 5
milestone: v0.3
status: execute
requirements: {covered: [REQ-3-006], partial: []}
---/ci---
102 lines
4.3 KiB
Python
102 lines
4.3 KiB
Python
"""Per-turn latency instrumentation tests (Task 5-4-01, REQ-3-006, A-109).
|
|
|
|
Mock-based: asserts instrumentation PRESENCE and population (stt_ms / llm_ms /
|
|
tts_ms fields, per-turn latency_ms persisted, the budget constant defined) —
|
|
wall-clock against a real voice endpoint is a v0.4 acceptance criterion
|
|
(real STT/TTS deferred per GRILL CUT-1 / G-7).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from ai_service.agents.examiner import ExaminerAgent
|
|
from ai_service.config import Settings
|
|
from ai_service.grading.store import SQLiteGradeStore
|
|
from ai_service.llm.mock import MockProvider
|
|
from ai_service.main import create_app
|
|
from ai_service.telemetry.ingest import TraceIntegrityMap
|
|
from ai_service.telemetry.store import SQLiteTraceStore
|
|
from ai_service.variants.store import SQLiteVariantStore
|
|
from ai_service.voice.defense_store import SQLiteDefenseStore
|
|
from ai_service.voice.mock import MockVoiceProvider
|
|
|
|
#: A-109: the documented conversational budget (acceptance criterion for the
|
|
#: v0.4 real-voice probe; mock turns are near-instant so v0.3 asserts
|
|
#: instrumentation, not wall-clock).
|
|
DEFENSE_TURN_BUDGET_MS = 4_000
|
|
|
|
|
|
@pytest.fixture()
|
|
def client(tmp_path: Path) -> TestClient:
|
|
llm = MockProvider()
|
|
app = create_app(Settings(provider="mock", voice_provider="mock"))
|
|
app.state.provider = llm
|
|
app.state.trace_store = SQLiteTraceStore(db_path=tmp_path / "t.db")
|
|
app.state.variant_store = SQLiteVariantStore(db_path=tmp_path / "v.db")
|
|
app.state.grade_store = SQLiteGradeStore(db_path=tmp_path / "g.db")
|
|
app.state.trace_integrity = TraceIntegrityMap()
|
|
app.state.defense_store = SQLiteDefenseStore(db_path=tmp_path / "d.db")
|
|
app.state.voice_provider = MockVoiceProvider(["my answer"])
|
|
app.state.examiner_agent = ExaminerAgent(llm, Settings(provider="mock"))
|
|
with TestClient(app) as c:
|
|
yield c
|
|
|
|
|
|
class TestLatencyInstrumentation:
|
|
def test_budget_constant_defined(self) -> None:
|
|
"""The conversational budget is a named, documented constant (A-109)."""
|
|
assert DEFENSE_TURN_BUDGET_MS > 0
|
|
assert DEFENSE_TURN_BUDGET_MS <= 5_000 # conversational feel target
|
|
|
|
def test_answer_reports_per_phase_latency(self, client: TestClient) -> None:
|
|
start = client.post(
|
|
"/v1/defense/start",
|
|
json={"learner_id": "lat-learner", "task_id": "lat-task"},
|
|
).json()
|
|
resp = client.post(
|
|
f"/v1/defense/{start['defense_id']}/answer", data={"text": "answer"}
|
|
)
|
|
assert resp.status_code == 200
|
|
latency = resp.json()["turn_latency"]
|
|
assert latency["llm_ms"] is not None and latency["llm_ms"] >= 0
|
|
assert "stt_ms" in latency and "tts_ms" in latency
|
|
|
|
def test_audio_answer_populates_stt_ms(self, client: TestClient) -> None:
|
|
start = client.post(
|
|
"/v1/defense/start",
|
|
json={"learner_id": "lat-learner", "task_id": "lat-task"},
|
|
).json()
|
|
resp = client.post(
|
|
f"/v1/defense/{start['defense_id']}/answer",
|
|
files={"audio": ("a.wav", b"RIFF" + b"\x00" * 32, "audio/wav")},
|
|
)
|
|
latency = resp.json()["turn_latency"]
|
|
assert latency["stt_ms"] is not None and latency["stt_ms"] >= 0
|
|
|
|
def test_every_turn_persists_latency_ms(self, client: TestClient) -> None:
|
|
start = client.post(
|
|
"/v1/defense/start",
|
|
json={"learner_id": "lat-learner", "task_id": "lat-task"},
|
|
).json()
|
|
client.post(f"/v1/defense/{start['defense_id']}/answer", data={"text": "a"})
|
|
transcript = client.get(f"/v1/defense/{start['defense_id']}").json()
|
|
assert transcript["turns"]
|
|
for turn in transcript["turns"]:
|
|
assert "latency_ms" in turn
|
|
assert turn["latency_ms"] is not None or turn["role"] == "learner"
|
|
|
|
def test_mock_turns_within_budget(self, client: TestClient) -> None:
|
|
"""Mock turns must be near-instant — the budget holds trivially."""
|
|
start = client.post(
|
|
"/v1/defense/start",
|
|
json={"learner_id": "lat-learner", "task_id": "lat-task"},
|
|
).json()
|
|
resp = client.post(
|
|
f"/v1/defense/{start['defense_id']}/answer", data={"text": "a"}
|
|
).json()
|
|
assert resp["turn_latency"]["llm_ms"] < DEFENSE_TURN_BUDGET_MS
|