Praxis CI
bdcf793db2
Phase 2 (Integration + Tech-Debt + NFR Measurement) complete. 4 slices, 2 waves, 9 tasks. 4 REQs covered. 60 new tests (469 total). 8 v0.4 P1+ tech-debt findings addressed. Verify: APPROVE_WITH_NOTES. NFR measurement (p95 latency + guardrail FP/FN), cohort aggregation assist metrics (5 new metrics, no schema change), assist cost tracking + C-3 budget check, tech-debt wave (argon2id offload, cookie-secret validation, credential enum, f-string SQL, cache persistence, zoneinfo, audit log, 429 mock). ---ci--- project: praxis phase: 2 milestone: v0.5 status: complete requirements: covered: [REQ-NFR-ASSIST-01, REQ-IDEATE-04, REQ-IDEATE-06, REQ-IDEATE-07] partial: [] ---/ci---
131 lines
4.9 KiB
Python
131 lines
4.9 KiB
Python
"""AssistLatencyMetrics — p95 assist-turn latency measurement (TASK-09-01, D-072, REQ-IDEATE-04).
|
|
|
|
Collects per-turn LatencyRecord objects (from the LatencyObserver — server/latency.py)
|
|
and computes the 95th percentile of `e2e_asr_to_tts_ms` (ASR transcript-ready → TTS
|
|
first-audio — the C-8 latency budget).
|
|
|
|
D-072 binding (pilot tolerance):
|
|
- target_ms = 600 (C-8 < 600ms — the hard target; v0.6 hardening)
|
|
- pilot_tolerance_ms = 650 (≤ 650ms acceptable for pilot per D-072)
|
|
- within_target = (p95 < 600) — the v0.6 hardening goal
|
|
- within_pilot = (p95 <= 650) — the pilot acceptance gate
|
|
|
|
The metrics are collected per shift (one AssistLatencyMetrics instance per
|
|
AssistSession) and reported at shift-end in the `session_outcome` dict, which
|
|
flows to the cohort aggregation (SLICE-10 — `assist_p95_latency_ms` metric).
|
|
|
|
This module does NOT assert that the actual latency is under budget — that is a
|
|
Phase-1 live measurement, not a CI test. This module provides the measurement
|
|
infrastructure (collect → percentile → summary). The test (TASK-09-03) asserts
|
|
the infrastructure works against mock records.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import statistics
|
|
from typing import Any
|
|
|
|
from server.latency import LatencyRecord
|
|
|
|
# D-072 binding thresholds (pilot tolerance).
|
|
TARGET_MS = 600 # C-8 hard target (< 600ms — v0.6 hardening goal)
|
|
PILOT_TOLERANCE_MS = 650 # D-072 pilot acceptance (≤ 650ms)
|
|
|
|
|
|
def _percentile(values: list[float], pct: float) -> float | None:
|
|
"""Compute the `pct`-th percentile (0..100) of `values` using nearest-rank.
|
|
|
|
Returns None if `values` is empty. Uses the nearest-rank method (the same
|
|
method used by numpy's default 'linear' interpolation for integer ranks):
|
|
rank = ceil(pct/100 * N), 1-indexed; index = rank - 1 (clamped to [0, N-1]).
|
|
This is the standard p95 computation for latency SLOs (Google SRE book §6).
|
|
"""
|
|
if not values:
|
|
return None
|
|
s = sorted(values)
|
|
n = len(s)
|
|
if n == 1:
|
|
return s[0]
|
|
# Nearest-rank: rank = ceil(pct/100 * n), then index = rank - 1.
|
|
import math
|
|
rank = max(1, math.ceil((pct / 100.0) * n))
|
|
idx = min(rank - 1, n - 1)
|
|
return s[idx]
|
|
|
|
|
|
class AssistLatencyMetrics:
|
|
"""Collects per-turn latency records + computes p95/p50/p99 (TASK-09-01).
|
|
|
|
One instance per assist shift. The LatencyObserver (server/latency.py) holds
|
|
the live per-turn records; at shift-end the session code calls `record()` for
|
|
each completed turn, then `summary()` to get the aggregate dict.
|
|
|
|
D-072: the summary reports both `within_target` (p95 < 600ms — the v0.6 goal)
|
|
and `within_pilot` (p95 ≤ 650ms — the pilot acceptance gate). If
|
|
`within_pilot` is False, the shift is flagged for the operator via the cohort
|
|
aggregation (`assist_p95_latency_ms` metric — SLICE-10).
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self._records: list[LatencyRecord] = []
|
|
|
|
def record(self, record: LatencyRecord) -> None:
|
|
"""Append a latency record (one per completed assist turn)."""
|
|
self._records.append(record)
|
|
|
|
@property
|
|
def count(self) -> int:
|
|
return len(self._records)
|
|
|
|
def _e2e_values(self) -> list[float]:
|
|
"""The non-None e2e_asr_to_tts_ms values across all records."""
|
|
out: list[float] = []
|
|
for r in self._records:
|
|
v = r.e2e_asr_to_tts_ms
|
|
if v is not None:
|
|
out.append(float(v))
|
|
return out
|
|
|
|
def p50(self) -> float | None:
|
|
"""The median e2e latency (ms), or None if no records."""
|
|
return _percentile(self._e2e_values(), 50.0)
|
|
|
|
def p95(self) -> float | None:
|
|
"""The 95th percentile e2e latency (ms), or None if no records."""
|
|
return _percentile(self._e2e_values(), 95.0)
|
|
|
|
def p99(self) -> float | None:
|
|
"""The 99th percentile e2e latency (ms), or None if no records."""
|
|
return _percentile(self._e2e_values(), 99.0)
|
|
|
|
def summary(self) -> dict[str, Any]:
|
|
"""Return the shift-end latency summary dict (D-072).
|
|
|
|
Fields:
|
|
p50, p95, p99: the percentiles (ms) or None if no records.
|
|
count: number of recorded turns.
|
|
target_ms: 600 (C-8 hard target).
|
|
pilot_tolerance_ms: 650 (D-072 pilot acceptance).
|
|
within_target: p95 < 600 (the v0.6 hardening goal).
|
|
within_pilot: p95 <= 650 (the pilot acceptance gate).
|
|
"""
|
|
p50 = self.p50()
|
|
p95 = self.p95()
|
|
p99 = self.p99()
|
|
return {
|
|
"p50": p50,
|
|
"p95": p95,
|
|
"p99": p99,
|
|
"count": self.count,
|
|
"target_ms": TARGET_MS,
|
|
"pilot_tolerance_ms": PILOT_TOLERANCE_MS,
|
|
"within_target": (p95 is not None and p95 < TARGET_MS),
|
|
"within_pilot": (p95 is not None and p95 <= PILOT_TOLERANCE_MS),
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"AssistLatencyMetrics",
|
|
"TARGET_MS",
|
|
"PILOT_TOLERANCE_MS",
|
|
] |