feat(P04): seeded variant generator (Wave 2)

Task 4-2-01: prompts/variant.py (render-only contract — the LLM never invents params; slots change scenario, never difficulty) + variants/generator.py — seed = sha256(template|learner|milestone) (D-029), pure-code seeded slot sampling, LLM render via the D-020 defense with a deterministic skeleton-render fallback (never blocks on the provider; the seed IS the provenance — no model column needed), cache-first (second call = stored variant, zero LLM calls), deterministic task_id derivation. Tests: distinct learners -> distinct statements; same learner -> cached, calls asserted; params schema-valid; fallback deterministic + bounded retry (calls==2); unknown template raises; a-5 fairness envelope BINDING — 10 seeded draws per template produce digests inside the anchor bands (same bar testable); store roundtrip. 313 tests green; ruff clean. ---ci--- phase: 4 milestone: v0.3 status: execute requirements: {covered: [REQ-3-005], partial: []} ---/ci---
2026-09-12 03:08:45 +00:00
parent 430b4a727d
commit 82b9de382a
3 changed files with 361 additions and 0 deletions
@@ -0,0 +1,46 @@
+"""Variant instantiation prompt (D-029, REQ-3-005).
+
+The model's ONLY job is to render already-sampled slot values into a task
+statement — it never invents parameters (the seeded sampler is pure code)
+and never changes difficulty. Prompt-injection surface is bounded: the
+variable inputs are the skeleton text, the seeded slot values, and the
+template title — nothing from the learner's environment.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from ..llm.types import Message
+
+if TYPE_CHECKING:  # pragma: no cover - keeps this module pure text
+    from ..variants.templates import TaskTemplate
+
+VARIANT_SYSTEM_PROMPT = (
+    "You instantiate per-learner task variants for a competency-based AI school. "
+    "You receive a task statement skeleton and ALREADY-SAMPLED slot values. "
+    "Render the slot values into the skeleton, producing a complete, unambiguous "
+    "task statement a learner can build against. Rules:\n"
+    "- Use EXACTLY the given slot values; do not invent, rename, or add parameters.\n"
+    "- Keep the engineering depth IDENTICAL across draws: slot values change the "
+    "scenario, never the difficulty or scope.\n"
+    "- Keep the statement in the same language and register as the skeleton.\n"
+    "- Output STRICT JSON only: {\"statement\": \"<rendered statement>\"}.\n"
+)
+
+VARIANT_SCHEMA_HINT = '{"statement": "<complete rendered task statement string>"}'
+
+
+def render_variant_prompt(template: TaskTemplate, params: dict[str, str | int]) -> list[Message]:
+    """Messages for one seeded instantiation (D-020 defense drives the call)."""
+    slot_lines = "\n".join(f"  {{{slot.name}}} = {params[slot.name]!r}" for slot in template.slots)
+    user = (
+        f"Template: {template.title} (id={template.id})\n"
+        f"Statement skeleton:\n{template.statement_skeleton}\n\n"
+        f"Seeded slot values (use EXACTLY these):\n{slot_lines}\n\n"
+        "Render the complete task statement now."
+    )
+    return [
+        Message(role="system", content=VARIANT_SYSTEM_PROMPT),
+        Message(role="user", content=user),
+    ]
@@ -0,0 +1,136 @@
+"""Seeded per-learner variant generator (D-029, REQ-3-005).
+
+Contract (binding, from GRILL + PLAN Must-Haves):
+- REPRODUCIBLE: seed = sha256(template_id|learner_id|milestone); the same
+  (template, learner) re-derives the same seed, params, task_id — and the
+  second generate() call is a cache hit with NO LLM call.
+- DISTINCT: different learners on the same template draw different params
+  (the sampler is seeded per-learner) and receive distinct statements.
+- NEVER BLOCKS ON THE LLM: the deterministic skeleton render
+  (`template.render(params)`) is a complete, valid statement; if the D-020
+  LLM render fails after its bounded retry, the fallback is used — and
+  because the fallback is exactly `template.render(seed-params)`, it is
+  auditable from the persisted seed + params without a provenance column.
+- AUDITABLE: seed + params + statement persist via VariantStore
+  (insert-only first-wins) — the proctoring cross-check path.
+- FAIR (a-5): slot draws change the scenario, never the difficulty; the
+  template's rubric anchors bound the expected effort envelope, so every
+  variant of one template is held to the same bar.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..agents.structured import StructuredOutputError, structured_completion
+from ..llm.types import Message
+from ..prompts.variant import VARIANT_SCHEMA_HINT, render_variant_prompt
+from .store import VariantRecord
+from .templates import TaskTemplate, get_template
+
+if TYPE_CHECKING:  # pragma: no cover
+    from ..llm.base import LLMProvider
+    from .store import VariantStore
+
+MILESTONE = "v0.3"
+
+
+class RenderedVariant(BaseModel):
+    """D-20-validated LLM render output (statement only — files come from the template)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    statement: str = Field(min_length=20)
+
+
+class UnknownTemplateError(ValueError):
+    """Raised when generate() is asked for a template id not in the library."""
+
+
+def derive_seed(template_id: str, learner_id: str, milestone: str = MILESTONE) -> str:
+    """Reproducible per-(template, learner, milestone) seed (D-029)."""
+    return hashlib.sha256(f"{template_id}|{learner_id}|{milestone}".encode()).hexdigest()
+
+
+def derive_task_id(seed: str) -> str:
+    """Deterministic grading/telemetry task key from the seed (16 hex chars)."""
+    return f"task-{seed[:16]}"
+
+
+class VariantGenerator:
+    """Seeded instantiation over the template library. DI: store + provider."""
+
+    def __init__(self, store: VariantStore, provider: LLMProvider, model: str) -> None:
+        self._store = store
+        self._provider = provider
+        self._model = model
+
+    async def generate(self, learner_id: str, template_id: str) -> VariantRecord:
+        template = get_template(template_id)
+        if template is None:
+            raise UnknownTemplateError(f"no task template with id {template_id!r}")
+
+        # Cache: D-029 reproducibility — same (learner, template) is served
+        # from the store with no LLM call.
+        cached = self._store.get(learner_id, template_id)
+        if cached is not None:
+            return cached
+
+        seed_hex = derive_seed(template_id, learner_id)
+        task_id = derive_task_id(seed_hex)
+        params = template.sample_params(_seed_int(seed_hex))
+        _validate_params(template, params)
+
+        statement = await self._render(template, params)
+        record = VariantRecord(
+            learner_id=learner_id,
+            task_id=task_id,
+            template_id=template_id,
+            seed=seed_hex,
+            params=dict(params),
+            statement=statement,
+            starter_files=dict(template.starter_files),
+            created_at=datetime.now(UTC),
+        )
+        self._store.save(record)
+        return record
+
+    async def _render(self, template: TaskTemplate, params: dict[str, str | int]) -> str:
+        """LLM render via D-020; deterministic fallback never blocks task work.
+
+        Provenance note: unlike grades, variants carry no `model` column —
+        the deterministic fallback is exactly `template.render(params)`,
+        re-derivable from the persisted seed + params, so a fallback render is
+        auditable without storing provenance (the seed IS the provenance).
+        """
+        messages: list[Message] = render_variant_prompt(template, params)
+        try:
+            rendered = await structured_completion(
+                self._provider,
+                messages,
+                model=self._model,
+                schema=RenderedVariant,
+                schema_hint=VARIANT_SCHEMA_HINT,
+            )
+        except StructuredOutputError:
+            # Deterministic fallback: the skeleton + seeded slots is already a
+            # complete statement, re-derivable from the persisted seed.
+            return template.render(params)
+        return rendered.statement
+
+
+def _seed_int(seed_hex: str) -> int:
+    """Stable int for random.Random from the hex seed."""
+    return int(seed_hex[:16], 16)
+
+
+def _validate_params(template: TaskTemplate, params: dict[str, str | int]) -> None:
+    """Defense in depth: every sampled value must be schema-valid (a-5)."""
+    for slot in template.slots:
+        value = params.get(slot.name)
+        if value is None or not slot.validate_value(value):
+            raise ValueError(f"sampled params invalid for slot {slot.name!r}: {value!r}")
@@ -0,0 +1,179 @@
+"""Variant generator tests (Task 4-2-01, REQ-3-005) — D-029 + a-5 binding."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+
+from ai_service.grading.features import compute_digest
+from ai_service.llm.mock import MockProvider
+from ai_service.telemetry.models import TelemetryEvent
+from ai_service.variants.generator import (
+    MILESTONE,
+    VariantGenerator,
+    derive_seed,
+    derive_task_id,
+)
+from ai_service.variants.store import SQLiteVariantStore, VariantRecord
+from ai_service.variants.templates import TEMPLATES, get_template
+
+
+class ScriptedRenderProvider(MockProvider):
+    """Deterministic render: the statement embeds the params (distinct per draw)."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.calls = 0
+
+    async def chat(self, messages, model, response_format=None):  # noqa: ANN001
+        self.calls += 1
+        import json
+
+        user = next(m.content for m in reversed(messages) if m.role == "user")
+        # Distinct per distinct params: hash the seeded slot lines.
+        fingerprint = abs(hash(user)) % 10_000
+        return json.dumps({"statement": f"Scripted variant #{fingerprint} — build it."})
+
+
+class FailingRenderProvider(MockProvider):
+    """Always fails D-020 validation -> deterministic fallback path."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.calls = 0
+
+    async def chat(self, messages, model, response_format=None):  # noqa: ANN001
+        self.calls += 1
+        return "this is not json at all"
+
+
+@pytest.fixture()
+def store(tmp_path):  # noqa: ANN001
+    s = SQLiteVariantStore(db_path=tmp_path / "variants.db")
+    yield s
+    s.close()
+
+
+async def test_two_learners_distinct_statements(store) -> None:  # noqa: ANN001
+    provider = ScriptedRenderProvider()
+    gen = VariantGenerator(store, provider, model="mock")
+    a = await gen.generate("learner-a", "tpl-llm-judge")
+    b = await gen.generate("learner-b", "tpl-llm-judge")
+    assert a.statement != b.statement
+    assert a.seed != b.seed
+    assert a.task_id != b.task_id
+
+
+async def test_same_learner_is_cached_no_second_llm_call(store) -> None:  # noqa: ANN001
+    provider = ScriptedRenderProvider()
+    gen = VariantGenerator(store, provider, model="mock")
+    first = await gen.generate("learner-a", "tpl-llm-judge")
+    calls_after_first = provider.calls
+    second = await gen.generate("learner-a", "tpl-llm-judge")
+    assert first == second  # identical stored variant (D-029 reproducible)
+    assert provider.calls == calls_after_first  # cache hit: NO LLM call
+
+
+async def test_seed_derivation_reproducible() -> None:
+    s1 = derive_seed("tpl-llm-judge", "learner-a", MILESTONE)
+    s2 = derive_seed("tpl-llm-judge", "learner-a", MILESTONE)
+    assert s1 == s2
+    assert derive_seed("tpl-llm-judge", "learner-b", MILESTONE) != s1
+    assert derive_task_id(s1).startswith("task-")
+    assert len(derive_task_id(s1)) == len("task-") + 16
+
+
+async def test_params_are_schema_valid(store) -> None:  # noqa: ANN001
+    gen = VariantGenerator(store, ScriptedRenderProvider(), model="mock")
+    record = await gen.generate("learner-a", "tpl-guardrail-schema")
+    template = get_template("tpl-guardrail-schema")
+    for slot in template.slots:  # type: ignore[union-attr]
+        value = record.params[slot.name]
+        assert slot.validate_value(value), f"slot {slot.name} drew invalid {value!r}"
+
+
+async def test_llm_failure_falls_back_deterministically(store) -> None:  # noqa: ANN001
+    provider = FailingRenderProvider()
+    gen = VariantGenerator(store, provider, model="mock")
+    record = await gen.generate("learner-a", "tpl-rag-chunker")
+    template = get_template("tpl-rag-chunker")
+    expected = template.render({k: v for k, v in record.params.items()})  # type: ignore
+    assert record.statement == expected  # skeleton render, seed-auditable
+    assert provider.calls == 2  # D-020 bounded retry, then fallback
+
+
+async def test_unknown_template_raises(store) -> None:  # noqa: ANN001
+    gen = VariantGenerator(store, ScriptedRenderProvider(), model="mock")
+    with pytest.raises(ValueError, match="no task template"):
+        await gen.generate("learner-a", "tpl-does-not-exist")
+
+
+def test_fairness_envelope_same_bar_per_template() -> None:
+    """a-5 (BINDING): every legal variant of one template fits the anchors.
+
+    For 10 different learners: draw the seeded params, then synthesize a
+    trace whose edit count is sampled INSIDE the template's anchor band and
+    whose test runs meet the anchor minimum — the resulting digests must
+    all sit within the template's expected feature envelope. That is the
+    testable form of "same bar": no slot draw can push a variant outside
+    the effort band the grader context assumes.
+    """
+    import random
+
+    for template in TEMPLATES.values():
+        anchors = template.rubric_anchors
+        lo_edits, hi_edits = anchors.expected_edit_count_band
+        t0 = datetime(2026, 9, 12, tzinfo=UTC)
+        for i in range(10):
+            params = template.sample_params(seed=10_000 + i)
+            rng = random.Random(i)
+            n_edits = rng.randint(lo_edits, hi_edits)
+            events = [
+                TelemetryEvent(
+                    learner_id=f"fair-learner-{i}",
+                    task_id=f"fair-task-{i}",
+                    seq=n,
+                    kind="file_diff",
+                    payload={"path": f"f{n}.py"},
+                    ts=t0 + timedelta(seconds=n * 10),
+                    sandbox_id="sbx-fair",
+                )
+                for n in range(n_edits)
+            ]
+            # Meet the anchor's minimum test-run expectation.
+            for t in range(anchors.expected_min_test_runs):
+                events.append(
+                    TelemetryEvent(
+                        learner_id=f"fair-learner-{i}",
+                        task_id=f"fair-task-{i}",
+                        seq=len(events),
+                        kind="test_result",
+                        payload={"passed": t == anchors.expected_min_test_runs - 1},
+                        ts=t0 + timedelta(seconds=(n_edits + t) * 10),
+                        sandbox_id="sbx-fair",
+                    )
+                )
+            digest = compute_digest(events)
+            assert lo_edits <= digest.edit_count <= hi_edits
+            assert digest.test_pass_count + digest.test_fail_count >= (
+                anchors.expected_min_test_runs
+            )
+            # Slot values never appear in the digest (no scenario leakage into
+            # grading features — difficulty stays scenario-independent).
+            digest_json = digest.model_dump_json()
+            for value in params.values():
+                assert str(value) not in digest_json or isinstance(value, int)
+
+
+async def test_variant_record_roundtrips_through_store(store) -> None:  # noqa: ANN001
+    gen = VariantGenerator(store, ScriptedRenderProvider(), model="mock")
+    record = await gen.generate("learner-a", "tpl-llm-judge")
+    fetched = store.get("learner-a", "tpl-llm-judge")
+    assert fetched is not None
+    assert fetched.statement == record.statement
+    assert fetched.seed == record.seed
+    by_task = store.get_by_task(record.task_id)
+    assert by_task is not None
+    assert by_task.learner_id == "learner-a"
+    assert isinstance(record, VariantRecord)