"""Tests for build_assist_pipeline + LiveAssistGuardrailProcessor (TASK-05-03, REQ-IDEATE-02). Verifies: - The pipeline structure is correct (Piper TTS default, guardrail processor between llm and tts, no opening line). - The LLM context is the ≤150-token assist prompt. - The LiveAssistGuardrailProcessor passes allowed text through. - The processor blocks direct-answer text → CANNED_FALLBACK. - The processor retries on a retry-eligible block. - The processor does NOT retry on false-authority (hard violation). - The verdict is logged to the session. These tests mock the WebRTC connection + transport so no live keys are needed. The pipeline structure is verified by inspecting the Pipeline's processors list. """ from __future__ import annotations import asyncio import os from unittest.mock import AsyncMock, MagicMock, patch import pytest from server.assist.context import AssistContext, COACHING_INSTRUCTION from server.assist.guardrail_processor import LiveAssistGuardrailProcessor from server.guardrails.live_assist import ( CANNED_FALLBACK, LiveAssistGuardrail, ) from server.services.base import GuardrailContext def _make_context() -> AssistContext: return AssistContext( system_prompt=f"{COACHING_INSTRUCTION}\n\nWeek 1, damaged-product refund.\n\nBe brief.", current_week=1, scenario_tag="damaged-product refund", theta=0.0, coaching_focus="empathy", path_slug="customer_service", ) def test_assist_system_prompt_under_word_budget(): """D-066: the assist system prompt is ≤200 words (≈150 tokens).""" ctx = _make_context() assert len(ctx.system_prompt.split()) <= 200 def test_build_assist_pipeline_structure(): """TASK-05-01: build_assist_pipeline returns a valid pipeline with the right structure. Mocks the WebRTC connection + services so no live keys are needed. Verifies the pipeline contains the guardrail processor + uses Piper TTS by default. """ # Mock the Pipecat services + aggregators + runner so no live keys/event loop needed. with patch("server.pipeline._build_transport") as mock_transport, \ patch("server.pipeline._build_stt") as mock_stt, \ patch("server.pipeline._build_llm") as mock_llm, \ patch("server.assist.pipeline._build_tts_piper") as mock_tts, \ patch("pipecat.processors.aggregators.llm_response_universal.LLMContextAggregator") as mock_agg, \ patch("pipecat.pipeline.runner.PipelineRunner") as mock_runner_cls: mock_transport.return_value = MagicMock(name="transport") mock_stt.return_value = MagicMock(name="stt") mock_llm.return_value = MagicMock(name="llm") mock_tts.return_value = MagicMock(name="piper_tts") mock_agg.return_value = MagicMock(name="aggregator") mock_runner_cls.return_value = MagicMock(name="runner") from server.assist.pipeline import build_assist_pipeline ctx = _make_context() webrtc_conn = MagicMock(name="webrtc_connection") pipeline, task, runner, transport = build_assist_pipeline( webrtc_conn, context=ctx ) # The pipeline has processors; verify the guardrail processor is present. processors = list(pipeline.processors) assert any(isinstance(p, LiveAssistGuardrailProcessor) for p in processors), ( "LiveAssistGuardrailProcessor must be in the pipeline (D-060 layer 2)" ) # Piper TTS was used (D-065 default). mock_tts.assert_called_once() # No opening line is played (assist is invoked mid-shift). def test_build_assist_pipeline_uses_cartesia_when_env_set(): """TASK-05-01: PRAXIS_ASSIST_TTS=cartesia falls back to Cartesia (for testing).""" with patch.dict(os.environ, {"PRAXIS_ASSIST_TTS": "cartesia"}), \ patch("server.pipeline._build_transport") as mock_transport, \ patch("server.pipeline._build_stt") as mock_stt, \ patch("server.pipeline._build_llm") as mock_llm, \ patch("server.pipeline._build_tts") as mock_cartesia, \ patch("pipecat.processors.aggregators.llm_response_universal.LLMContextAggregator") as mock_agg, \ patch("pipecat.pipeline.runner.PipelineRunner") as mock_runner_cls: mock_transport.return_value = MagicMock() mock_stt.return_value = MagicMock() mock_llm.return_value = MagicMock() mock_cartesia.return_value = MagicMock(name="cartesia_tts") mock_agg.return_value = MagicMock(name="aggregator") mock_runner_cls.return_value = MagicMock(name="runner") from server.assist.pipeline import build_assist_pipeline ctx = _make_context() pipeline, task, runner, transport = build_assist_pipeline( MagicMock(), context=ctx ) mock_cartesia.assert_called_once() # ── LiveAssistGuardrailProcessor behavior ──────────────────────────────────── def _make_processor(session=None, llm_context=None) -> LiveAssistGuardrailProcessor: """Build a processor with a mock frame pusher for isolated testing.""" proc = LiveAssistGuardrailProcessor( guardrail=LiveAssistGuardrail(), session=session, llm_context=llm_context, ) proc.push_frame = AsyncMock() return proc def test_processor_passes_allowed_text_through(): """Allowed coaching text → buffered, then pushed to TTS as one frame (REQ-ASSIST-03). The guardrail MUST complete its check before any text reaches TTS. This means text is buffered (not streamed) + pushed as a single TextFrame on LLMFullResponseEndFrame after the guardrail allows it. This is the safety-critical behavior: a blocked response never reaches TTS. """ proc = _make_processor() async def _run(): from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame # Simulate LLM text chunks. await proc.process_frame(TextFrame(text="What do you think "), direction=1) await proc.process_frame(TextFrame(text="the customer needs?"), direction=1) # End of LLM response. end_frame = LLMFullResponseEndFrame() await proc.process_frame(end_frame, direction=1) asyncio.run(_run()) # The buffered text was pushed as a single TextFrame (not streamed chunk-by-chunk). pushed_texts = [ call.args[0].text for call in proc.push_frame.await_args_list if hasattr(call.args[0], "text") ] assert "What do you think the customer needs?" in pushed_texts # The LLMFullResponseEndFrame was also pushed (to signal TTS the response is done). assert proc.push_frame.await_count >= 2 # 1 buffered text + 1 end frame def test_processor_blocks_direct_answer(): """Direct-answer text → CANNED_FALLBACK emitted (no pass-through of the blocked text).""" proc = _make_processor() async def _run(): from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame await proc.process_frame(TextFrame(text="You should say sorry."), direction=1) end_frame = LLMFullResponseEndFrame() await proc.process_frame(end_frame, direction=1) asyncio.run(_run()) # A TextFrame with CANNED_FALLBACK was pushed. pushed_texts = [ call.args[0].text for call in proc.push_frame.await_args_list if hasattr(call.args[0], "text") ] assert CANNED_FALLBACK in pushed_texts def test_processor_retries_on_retry_eligible_block(): """Retry-eligible block (direct-answer) → inject RETRY_INSTRUCTION + retry.""" from pipecat.processors.aggregators.llm_context import LLMContext llm_context = LLMContext() proc = _make_processor(llm_context=llm_context) messages_before = len(llm_context.get_messages()) async def _run(): from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame await proc.process_frame(TextFrame(text="You should say sorry."), direction=1) end_frame = LLMFullResponseEndFrame() await proc.process_frame(end_frame, direction=1) asyncio.run(_run()) # The RETRY_INSTRUCTION was injected into the context (G-049 validated). messages_after = len(llm_context.get_messages()) assert messages_after == messages_before + 1 injected = llm_context.get_messages()[-1] assert "coaching question" in (injected.get("content") or "").lower() # The retry flag is set (no second retry). assert proc._retry_used is True def test_processor_no_retry_on_false_authority(): """Hard violation (false-authority) → CANNED_FALLBACK immediately, no retry.""" from pipecat.processors.aggregators.llm_context import LLMContext llm_context = LLMContext() proc = _make_processor(llm_context=llm_context) messages_before = len(llm_context.get_messages()) async def _run(): from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame await proc.process_frame(TextFrame(text="I am your manager."), direction=1) end_frame = LLMFullResponseEndFrame() await proc.process_frame(end_frame, direction=1) asyncio.run(_run()) # No retry message was injected (hard violation). messages_after = len(llm_context.get_messages()) assert messages_after == messages_before # CANNED_FALLBACK was emitted. pushed_texts = [ call.args[0].text for call in proc.push_frame.await_args_list if hasattr(call.args[0], "text") ] assert CANNED_FALLBACK in pushed_texts def test_processor_logs_verdict_to_session(): """The verdict is logged to the session (D-060 layer 3).""" session = MagicMock() session.log_assist_turn_partial = AsyncMock(return_value=0) session.log_assist_turn_complete = AsyncMock() session.guardrail_block_count = 0 proc = _make_processor(session=session) async def _run(): from pipecat.frames.frames import ( LLMFullResponseEndFrame, TextFrame, TranscriptionFrame, ) # ASR transcript (partial turn write — REQ-IDEATE-09). await proc.process_frame( TranscriptionFrame(text="Customer wants refund", user_id="u", timestamp=""), direction=1, ) # LLM response (direct answer → blocked). await proc.process_frame(TextFrame(text="You should say sorry."), direction=1) await proc.process_frame(LLMFullResponseEndFrame(), direction=1) asyncio.run(_run()) # The partial turn was written (REQ-IDEATE-09). session.log_assist_turn_partial.assert_awaited_once_with("Customer wants refund") # The complete turn was written with the verdict. session.log_assist_turn_complete.assert_awaited_once() # The verdict passed to log_assist_turn_complete has allowed=False (block). complete_call = session.log_assist_turn_complete.await_args verdict_arg = complete_call.kwargs.get("guardrail_verdict") or complete_call.args[2] assert verdict_arg["allowed"] is False # (The real AssistSession.log_assist_turn_complete increments guardrail_block_count # when the verdict has allowed=False — verified in test_p1_guardrail_e2e.py.) def test_processor_incremental_audit_log_partial_turn(): """REQ-IDEATE-09: a partial turn (ASR only) is written before the LLM response.""" session = MagicMock() session.log_assist_turn_partial = AsyncMock(return_value=0) session.log_assist_turn_complete = AsyncMock() session.guardrail_block_count = 0 proc = _make_processor(session=session) async def _run_partial_only(): from pipecat.frames.frames import TranscriptionFrame # ASR transcript arrives but the LLM never responds (simulated abrupt termination). await proc.process_frame( TranscriptionFrame(text="Customer is upset", user_id="u", timestamp=""), direction=1, ) asyncio.run(_run_partial_only()) # The partial turn was written even though the LLM never responded. session.log_assist_turn_partial.assert_awaited_once_with("Customer is upset") session.log_assist_turn_complete.assert_not_awaited()