Praxis CI
ec397f2c65
v0.5 (Live Assist — on-the-job voice companion) milestone complete. 4 phases: P0 (pre-execution, v0.1.10) → P1 (assist core + guardrail, v0.1.11) → P2 (integration + tech-debt + NFR, v0.1.12) → P3 (final review + ship, v0.1.13 = milestone release). 16/16 REQs covered (3 ASSIST + 4 NFR + 9 IDEATE). 4 v0.6 backlog. 469 tests passed, 0 failed. 1 P0 fixed (guardrail processor safety). 8 P1+ flagged for v0.6. 8 v0.4 P1+ tech-debt addressed. G-049 + G-067 grill MUSTs resolved. ESCALATION-01 (PIPEDA) OPEN for human legal review before assist surface go-live. ---ci--- project: praxis phase: 3 milestone: v0.5 status: complete requirements: covered: [REQ-ASSIST-01, REQ-ASSIST-02, REQ-ASSIST-03, REQ-NFR-ASSIST-01, REQ-NFR-ASSIST-02, REQ-NFR-ASSIST-03, REQ-NFR-ASSIST-04, REQ-IDEATE-01, REQ-IDEATE-02, REQ-IDEATE-03, REQ-IDEATE-04, REQ-IDEATE-05, REQ-IDEATE-06, REQ-IDEATE-07, REQ-IDEATE-08, REQ-IDEATE-09] partial: [] ---/ci---
274 lines
11 KiB
Python
274 lines
11 KiB
Python
"""GuardrailMetrics — false-positive / false-negative measurement (TASK-09-02, REQ-IDEATE-04).
|
|
|
|
Measures the two guardrail NFR targets from REQ-IDEATE-04:
|
|
- false_positive_rate: the FP rate on the tuning corpus (coaching responses
|
|
blocked). Target < 5% (REQ-IDEATE-04). Measured at test time
|
|
(test_guardrail_tuning.py) + reported here for the P2 verification.
|
|
- false_negative_rate: the FN rate on the direct-answer + adversarial corpus
|
|
(direct answers allowed). Measured at test time + trended nightly.
|
|
|
|
The nightly trend (`nightly_trend`) samples the last 24h of assist turns from
|
|
the local SQLite turns table, re-runs the LiveAssistGuardrail on the `tts_text`
|
|
(the LLM response that was actually played to the learner), and reports any
|
|
`fn_candidates` — turns where the guardrail allowed the text but the text
|
|
contains direct-answer patterns (a heuristic re-check, not a full LLM-as-judge
|
|
which is v0.6 per REQ-IDEATE-10).
|
|
|
|
D-068 mitigation: the regex is the first line, not the only line. The nightly
|
|
trend + the v0.6 LLM-as-judge (REQ-IDEATE-10) are the defense-in-depth. This
|
|
nightly trend is a diagnostic (logged, not stored in Postgres — it's not a
|
|
cohort metric). The operator can review the log to spot guardrail regressions.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import datetime as _dt
|
|
import json
|
|
import logging
|
|
from typing import Any
|
|
|
|
from server.guardrails.live_assist import LiveAssistGuardrail
|
|
from server.services.base import GuardrailContext
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
# REQ-IDEATE-04 targets.
|
|
FP_TARGET = 0.05 # < 5% false-positive rate on coaching corpus
|
|
FN_TARGET = 0.05 # < 5% false-negative rate on direct-answer corpus
|
|
|
|
|
|
class GuardrailMetrics:
|
|
"""Measures the LiveAssistGuardrail FP/FN rates (TASK-09-02, REQ-IDEATE-04).
|
|
|
|
Constructed with the tuning corpus (tests/guardrail_corpus.py) for the
|
|
FP/FN rate computation. The nightly_trend() method takes a PraxisStore
|
|
(SQLite) to sample recent assist turns.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
coaching_corpus: list[dict] | None = None,
|
|
direct_corpus: list[dict] | None = None,
|
|
adversarial_corpus: list[dict] | None = None,
|
|
) -> None:
|
|
# Lazy-import the corpus to avoid a circular import at module load
|
|
# (tests/guardrail_corpus.py is a test fixture).
|
|
if coaching_corpus is None or direct_corpus is None:
|
|
from tests.guardrail_corpus import (
|
|
ADVERSARIAL_RESPONSES,
|
|
COACHING_RESPONSES,
|
|
DIRECT_ANSWER_RESPONSES,
|
|
)
|
|
self._coaching = coaching_corpus or COACHING_RESPONSES
|
|
self._direct = direct_corpus or DIRECT_ANSWER_RESPONSES
|
|
self._adversarial = adversarial_corpus or ADVERSARIAL_RESPONSES
|
|
else:
|
|
self._coaching = coaching_corpus
|
|
self._direct = direct_corpus
|
|
self._adversarial = adversarial_corpus or []
|
|
self._guardrail = LiveAssistGuardrail()
|
|
self._ctx = GuardrailContext(role="assist")
|
|
|
|
async def _check(self, text: str) -> bool:
|
|
"""Return True if the guardrail allows `text` (allowed=True)."""
|
|
verdict = await self._guardrail.check(text, self._ctx)
|
|
return bool(verdict.allowed)
|
|
|
|
async def false_positive_rate(self) -> tuple[float, int, int]:
|
|
"""FP rate on the coaching corpus (coaching responses blocked).
|
|
|
|
A false positive = a coaching response that the guardrail blocked
|
|
(allowed=False when it should have been allowed=True). Target < 5%
|
|
(REQ-IDEATE-04).
|
|
"""
|
|
misclassified = 0
|
|
total = 0
|
|
for entry in self._coaching:
|
|
total += 1
|
|
allowed = await self._check(entry["text"])
|
|
if not allowed: # blocked a coaching response → FP
|
|
misclassified += 1
|
|
rate = misclassified / total if total else 0.0
|
|
return rate, misclassified, total
|
|
|
|
async def false_negative_rate(self) -> tuple[float, int, int]:
|
|
"""FN rate on the direct-answer corpus (direct answers allowed).
|
|
|
|
A false negative = a direct-answer response that the guardrail allowed
|
|
(allowed=True when it should have been allowed=False). Target < 5%
|
|
(REQ-IDEATE-04).
|
|
"""
|
|
misclassified = 0
|
|
total = 0
|
|
for entry in self._direct:
|
|
total += 1
|
|
allowed = await self._check(entry["text"])
|
|
if allowed: # allowed a direct answer → FN
|
|
misclassified += 1
|
|
rate = misclassified / total if total else 0.0
|
|
return rate, misclassified, total
|
|
|
|
async def adversarial_false_negative_rate(self) -> tuple[float, int, int]:
|
|
"""FN rate on the adversarial corpus (paraphrased direct answers).
|
|
|
|
This is the G-067 residual-risk set. The threshold is ≤ 20% for pilot
|
|
(documented in test_guardrail_tuning.py). Reported here for the P2
|
|
verification matrix; NOT asserted against the 5% target (the adversarial
|
|
set is explicitly the residual-risk set, not the tuning target).
|
|
"""
|
|
misclassified = 0
|
|
total = 0
|
|
for entry in self._adversarial:
|
|
total += 1
|
|
allowed = await self._check(entry["text"])
|
|
if allowed: # allowed a paraphrased direct answer → FN
|
|
misclassified += 1
|
|
rate = misclassified / total if total else 0.0
|
|
return rate, misclassified, total
|
|
|
|
async def nightly_trend(self, store: Any) -> dict[str, Any]:
|
|
"""Sample the last 24h of assist turns + re-run the guardrail (TASK-09-02).
|
|
|
|
Reads assist turns from the local SQLite turns table (joined to sessions
|
|
on session_type='assist'), re-runs the LiveAssistGuardrail on each
|
|
`tts_text`, and reports `fn_candidates` — turns where the guardrail
|
|
allowed the text but the text contains direct-answer heuristic patterns.
|
|
|
|
This is the "trended nightly" part of REQ-IDEATE-04. It's a diagnostic
|
|
(logged, not stored in Postgres — not a cohort metric). The heuristic
|
|
re-check is a simple direct-answer pattern match (not a full LLM-as-judge
|
|
— that's v0.6 per REQ-IDEATE-10).
|
|
|
|
Returns:
|
|
{total_turns, blocked, allowed_coaching, allowed_neutral,
|
|
fn_candidates: [{turn_seq, tts_text, reason}], window_hours: 24}
|
|
"""
|
|
cutoff = (_dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=24)).isoformat()
|
|
# Query assist turns from the last 24h. The PraxisStore (SQLite) holds
|
|
# the turns table; we read directly via aiosqlite to avoid adding a
|
|
# method to the store surface for a diagnostic.
|
|
rows: list[dict[str, Any]] = []
|
|
try:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(store.db_path) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
cur = await db.execute(
|
|
"SELECT t.id, t.seq, t.tts_text, t.guardrail_verdict_json, "
|
|
"t.created_at, s.session_type "
|
|
"FROM turns t JOIN sessions s ON t.session_id = s.id "
|
|
"WHERE s.session_type = 'assist' "
|
|
"AND t.tts_text IS NOT NULL "
|
|
"AND t.created_at >= ? "
|
|
"ORDER BY t.seq",
|
|
(cutoff,),
|
|
)
|
|
async for r in cur:
|
|
rows.append(dict(r))
|
|
except Exception:
|
|
log.exception("nightly_trend: failed to read assist turns from %s",
|
|
getattr(store, "db_path", "?"))
|
|
return {
|
|
"total_turns": 0, "blocked": 0, "allowed_coaching": 0,
|
|
"allowed_neutral": 0, "fn_candidates": [], "window_hours": 24,
|
|
"error": "failed to read turns",
|
|
}
|
|
|
|
total = len(rows)
|
|
blocked = 0
|
|
allowed_coaching = 0
|
|
allowed_neutral = 0
|
|
fn_candidates: list[dict[str, Any]] = []
|
|
|
|
for r in rows:
|
|
tts_text = r.get("tts_text") or ""
|
|
verdict_json = r.get("guardrail_verdict_json")
|
|
try:
|
|
verdict = json.loads(verdict_json) if verdict_json else {}
|
|
except Exception:
|
|
verdict = {}
|
|
allowed = bool(verdict.get("allowed", True))
|
|
if not allowed:
|
|
blocked += 1
|
|
continue
|
|
# The guardrail allowed this text. Re-run the guardrail to confirm
|
|
# (regression detection) + apply a heuristic direct-answer check.
|
|
re_allowed = await self._check(tts_text)
|
|
if not re_allowed:
|
|
# The guardrail now blocks what it previously allowed → a
|
|
# regression (or the corpus tuning changed). Flag it.
|
|
fn_candidates.append({
|
|
"turn_seq": r.get("seq"),
|
|
"tts_text": tts_text[:200], # truncate for the log
|
|
"reason": "guardrail regression: previously allowed, now blocked",
|
|
})
|
|
continue
|
|
# Heuristic direct-answer check (defense-in-depth — not the LLM-as-judge).
|
|
if _heuristic_direct_answer(tts_text):
|
|
fn_candidates.append({
|
|
"turn_seq": r.get("seq"),
|
|
"tts_text": tts_text[:200],
|
|
"reason": "heuristic direct-answer pattern detected",
|
|
})
|
|
continue
|
|
# Classify allowed responses as coaching or neutral.
|
|
if _looks_like_coaching_question(tts_text):
|
|
allowed_coaching += 1
|
|
else:
|
|
allowed_neutral += 1
|
|
|
|
result = {
|
|
"total_turns": total,
|
|
"blocked": blocked,
|
|
"allowed_coaching": allowed_coaching,
|
|
"allowed_neutral": allowed_neutral,
|
|
"fn_candidates": fn_candidates,
|
|
"window_hours": 24,
|
|
}
|
|
log.info(
|
|
"guardrail nightly trend: %d turns, %d blocked, %d allowed_coaching, "
|
|
"%d allowed_neutral, %d fn_candidates",
|
|
total, blocked, allowed_coaching, allowed_neutral, len(fn_candidates),
|
|
)
|
|
return result
|
|
|
|
|
|
# ── Heuristic direct-answer detection (nightly trend defense-in-depth) ──────
|
|
# A simple pattern check for the nightly trend. This is NOT the guardrail itself
|
|
# (the guardrail is the 6-regex LiveAssistGuardrail). This is a secondary
|
|
# heuristic to catch direct-answer patterns the guardrail may have allowed —
|
|
# it's the "trended nightly" detection surface per REQ-IDEATE-04. The v0.6
|
|
# LLM-as-judge (REQ-IDEATE-10) will replace this with a semantic classifier.
|
|
|
|
_DIRECT_ANSWER_HEURISTIC_PATTERNS = (
|
|
"you should say",
|
|
"tell the customer",
|
|
"the answer is",
|
|
"here's what to say",
|
|
"what you should do is",
|
|
"say this:",
|
|
"respond with:",
|
|
)
|
|
|
|
|
|
def _heuristic_direct_answer(text: str) -> bool:
|
|
"""Heuristic check for direct-answer patterns (nightly trend only)."""
|
|
lower = text.lower()
|
|
return any(p in lower for p in _DIRECT_ANSWER_HEURISTIC_PATTERNS)
|
|
|
|
|
|
def _looks_like_coaching_question(text: str) -> bool:
|
|
"""Heuristic: does the text look like a coaching question?"""
|
|
stripped = text.strip()
|
|
if stripped.endswith("?"):
|
|
return True
|
|
coaching_starters = ("what ", "how ", "why ", "have you ", "can you ", "could you ")
|
|
lower = stripped.lower()
|
|
return any(lower.startswith(s) for s in coaching_starters)
|
|
|
|
|
|
__all__ = [
|
|
"GuardrailMetrics",
|
|
"FP_TARGET",
|
|
"FN_TARGET",
|
|
] |