From 430b4a727d9c741ce483ee83887c0e4014afd98f Mon Sep 17 00:00:00 2001 From: CIAgent Date: Sat, 12 Sep 2026 02:57:09 +0000 Subject: [PATCH] feat(P04): task template library + VariantStore (Wave 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 4-1-01: variants/templates.py — 3 task templates (llm-judge, guardrail-schema, rag-chunker) bound to D-021 corpus competency IDs; typed ParameterSlots (enum/int_range/string-set) with seeded pure-code sampler (random.Random(seed)); RubricAnchors difficulty-normalization envelope; starter-file scaffolds + test command. Task 4-1-02: VariantStore protocol + SQLiteVariantStore (insert-only first-wins; unique (learner,template) + unique task_id; WAL; tz-normalized) — the audit trail for proctoring cross-checks. 22 variant tests green; ruff clean. ---ci--- phase: 4 milestone: v0.3 status: execute requirements: {covered: [REQ-3-005], partial: []} ---/ci--- --- .../ai_service/variants/__init__.py | 29 ++ apps/ai-service/ai_service/variants/store.py | 311 +++++++++++++++ .../ai_service/variants/templates.py | 299 +++++++++++++++ apps/ai-service/tests/variants/__init__.py | 1 + apps/ai-service/tests/variants/test_store.py | 358 ++++++++++++++++++ .../tests/variants/test_templates.py | 102 +++++ 6 files changed, 1100 insertions(+) create mode 100644 apps/ai-service/ai_service/variants/__init__.py create mode 100644 apps/ai-service/ai_service/variants/store.py create mode 100644 apps/ai-service/ai_service/variants/templates.py create mode 100644 apps/ai-service/tests/variants/__init__.py create mode 100644 apps/ai-service/tests/variants/test_store.py create mode 100644 apps/ai-service/tests/variants/test_templates.py diff --git a/apps/ai-service/ai_service/variants/__init__.py b/apps/ai-service/ai_service/variants/__init__.py new file mode 100644 index 0000000..1cad88b --- /dev/null +++ b/apps/ai-service/ai_service/variants/__init__.py @@ -0,0 +1,29 @@ +"""Per-learner variant task generation — templates, generator, VariantStore (REQ-3-005). + +Boundary rule (D-027): variants/ is an engine module — it never imports +api/ or agents/ (api/ composes the generator and store via DI; store.py +imports config only). + +CO-ORDINATION NOTE (ADD, don't REMOVE — same convention as grading/): +This __init__.py is a minimal placeholder created by the VariantStore +task (4-1-02). The templates task (4-1-01) owns this file's final shape +— when templates.py lands, ADD its exports alongside these; do not +remove the store exports below. + +Wave status: store.py (VariantRecord, VariantStore, SQLiteVariantStore) +landed in Wave 1 (task 4-1-02); templates.py is Wave 1 task 4-1-01; +generator.py is Wave 2 (4-2-01). +""" + +from .store import SQLiteVariantStore, VariantRecord, VariantStore +from .templates import TEMPLATES, TaskTemplate, get_template, template_for_competency + +__all__ = [ + "SQLiteVariantStore", + "TEMPLATES", + "TaskTemplate", + "VariantRecord", + "VariantStore", + "get_template", + "template_for_competency", +] diff --git a/apps/ai-service/ai_service/variants/store.py b/apps/ai-service/ai_service/variants/store.py new file mode 100644 index 0000000..8a3438a --- /dev/null +++ b/apps/ai-service/ai_service/variants/store.py @@ -0,0 +1,311 @@ +"""VariantStore — variant persistence protocol + SQLite implementation (REQ-3-005, D-027). + +Postgres-migration-ready (D-027): the protocol is the only surface the +variant generator and API layers touch; swapping SQLiteVariantStore for a +Postgres-backed implementation must not change call sites. The +`variant_record` table uses only portable column types (str / JSON / +datetime), so the same SQLModel schema stands up unchanged on Postgres. + +Insert-only, NOT upsert: (learner_id, template_id) is the variant identity +and the FIRST generation is authoritative — reproducibility (D-029) means +the seed re-derives the same variant, so the generator's cache path serves +`get` instead of saving again. `save` is a plain INSERT; a duplicate pair +raises sqlalchemy.exc.IntegrityError to the caller (documented behavior). +`task_id` is unique too — it is the grading/telemetry trace key, so a +trace or grade can never silently join to a different variant. Both +rejections are deliberate: overwriting a stored variant would swap a +learner's graded task underneath its trace and grade (audit corruption). +Contrast TraceStore.append (dedup-keep-first, swallowed — at-least-once +ingest) and GradeStore.save (upsert-latest-wins — a regrade is +latest-state); this store is the third contract of the D-027 family. + +Concurrency (a-3): the store enables WAL + synchronous=NORMAL and a busy +timeout at connection time, so a generation writer and API readers do not +hit `database is locked` on the single-box pilot. + +`created_at` contract: callers stamp UTC (datetime.now(UTC)); SQLite +stores it naive and the read paths re-label it tz-aware UTC (same +boundary normalization as TelemetryEvent.ts / GradeRecord.created_at, so +the contract holds on any backend). + +Boundary (D-027): `variants/` never imports `agents/` / `api/`; this +module imports config only. +""" + +import logging +import sqlite3 +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Protocol + +import sqlalchemy as sa +from sqlalchemy import JSON, Index, UniqueConstraint +from sqlalchemy.orm import validates +from sqlmodel import Field, Session, SQLModel, create_engine, select + +from ..config import Settings + +logger = logging.getLogger(__name__) + + +class VariantRecord(SQLModel, table=True): + """A persisted task variant; (learner_id, template_id) is the PK — first wins. + + Written once by the variant generator (Task 4-2-01), read by the API + layer and proctoring cross-checks through the VariantStore protocol. + Constraint enforcement mirrors TelemetryEvent / GradeRecord: sqlmodel + 0.0.42's metaclass drops pydantic constraints on table models, so + SQLAlchemy `@validates` hooks enforce instead and the column types + stay Postgres-ready (D-027). + + Field contract: + learner_id — non-empty learner identifier (same id space as + traces and grades). + template_id — non-empty task template identifier; variant + identity is the (learner_id, template_id) pair — + the pair the generator caches on (exactly one + variant per learner per template). + task_id — non-empty, GLOBALLY unique task identifier; the + grading/telemetry trace key (the (learner_id, + task_id) pair TraceStore / GradeStore key on), + stamped at generation so a variant's trace and + grade join back to it exactly once. + seed — non-empty variant seed (D-029); derived from + (template_id, learner_id, milestone) so the + variant is reproducible and auditable. + params — typed parameter-slot values the generator filled; + JSON dict. An empty dict is legal (a slotless + template). + statement — non-empty rendered task statement shown to the + learner (distinct per learner by construction, + REQ-3-005). + starter_files — workspace scaffold: filename -> file content; + JSON dict. An empty dict is legal (no scaffold). + created_at — UTC generation timestamp. + """ + + __tablename__ = "variant_record" + # The composite PK covers (learner_id, template_id) point lookups; the + # unique task_id covers get_by_task (the grading/telemetry join path); + # the two secondary indexes cover list_for_learner / list_by_template + # ordered by created_at without a sort step (Postgres target D-027). + __table_args__ = ( + UniqueConstraint("task_id", name="uq_variant_record_task_id"), + Index("ix_variant_record_learner_created", "learner_id", "created_at"), + Index("ix_variant_record_template_created", "template_id", "created_at"), + ) + + learner_id: str = Field(primary_key=True) + template_id: str = Field(primary_key=True) + task_id: str + seed: str + # JSON columns: stored as TEXT on SQLite, native JSONB on Postgres (D-027). + params: dict[str, Any] = Field(default_factory=dict, sa_type=JSON) + statement: str + starter_files: dict[str, Any] = Field(default_factory=dict, sa_type=JSON) + created_at: datetime + + @validates("learner_id", "template_id", "task_id") + def _ids_non_empty(self, key: str, value: str) -> str: + if not value: + raise ValueError(f"{key} must be a non-empty identifier") + return value + + @validates("seed") + def _seed_non_empty(self, key: str, value: str) -> str: + if not value: + raise ValueError(f"{key} must be a non-empty seed string") + return value + + @validates("statement") + def _statement_non_empty(self, key: str, value: str) -> str: + if not value: + raise ValueError(f"{key} must be a non-empty statement string") + return value + + +class VariantStore(Protocol): + """Persistence contract for reproducible per-learner task variants. + + Implemented by SQLiteVariantStore (v0.3, D-027); a Postgres + implementation must satisfy the same surface. + """ + + def save(self, variant: VariantRecord) -> None: + """Persist a new variant. INSERT-ONLY on (learner_id, template_id): + the FIRST generated variant is authoritative (reproducibility, + D-029); a duplicate pair raises sqlalchemy.exc.IntegrityError to + the caller — the generator serves cached variants via `get` + instead of saving again. `task_id` is unique too: claiming an + existing trace key for a different variant is equally rejected. + NOT upsert; contrast GradeStore.save (latest-wins) and + TraceStore.append (dedup-keep-first, swallowed). + """ + ... + + def get(self, learner_id: str, template_id: str) -> VariantRecord | None: + """The learner's stored variant for the template; None when none + exists. Detached from any DB session — safe to pass across layers. + """ + ... + + def get_by_task(self, task_id: str) -> VariantRecord | None: + """The variant owning the task key (the grading/telemetry join + path); None when none exists. Detached from any DB session. + """ + ... + + def list_for_learner(self, learner_id: str) -> list[VariantRecord]: + """All stored variants for the learner, ordered by created_at + ascending (chronological; task_id breaks same-instant ties). + Empty list when the learner has none. + """ + ... + + def list_by_template(self, template_id: str) -> list[VariantRecord]: + """All stored variants generated from the template — one row per + learner — ordered by created_at ascending (chronological; + learner_id breaks same-instant ties). Empty list when the + template has none. The proctoring cross-check path (seed params + per learner) reads through this. + """ + ... + + def close(self) -> None: + """Release DB connections. Store must not be used after close.""" + ... + + +def _sqlite_connect(dbapi_connection: sqlite3.Connection, _: object) -> None: + """Per-connection pragma setup (a-3). Mirrors telemetry/grading stores. + + journal_mode=WAL — readers never block the single writer. + synchronous=NORMAL — safe in WAL mode, avoids full fsync-per-commit. + busy_timeout=5000 — retry briefly under contention instead of + `OperationalError: database is locked`. + """ + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA synchronous=NORMAL") + cursor.execute("PRAGMA busy_timeout=5000") + cursor.close() + + +def _as_utc(ts: datetime) -> datetime: + """Timestamps travel as naive datetime on SQLite, tz-aware elsewhere. + + SQLite (via SQLModel) drops tzinfo; Postgres TIMESTAMP WITH TIME ZONE + keeps it. Normalizing on the read path makes the store's contract + tz-aware UTC regardless of the backend (D-027). + """ + if ts.tzinfo is None: + return ts.replace(tzinfo=UTC) # naive UTC read-side: label as UTC + return ts.astimezone(UTC) + + +class SQLiteVariantStore: + """SQLite-backed VariantStore (SQLModel). Third protocol-wrapped store + of the D-027 family (first: SQLiteTraceStore, second: SQLiteGradeStore). + """ + + def __init__(self, db_path: Path | None = None) -> None: + self._db_path: Path = db_path if db_path is not None else Settings().db_path + self._engine = create_engine(f"sqlite:///{self._db_path}") + sa.event.listen(self._engine, "connect", _sqlite_connect) + SQLModel.metadata.create_all(self._engine) + + @contextmanager + def _session(self) -> Iterator[Session]: + # expire_on_commit=False: identical session behavior to the other + # D-027 stores. save() never commits on the error path and the read + # paths never commit, but a uniform flag across the family keeps + # their detachment guarantees from diverging. + with Session(self._engine, expire_on_commit=False) as session: + yield session + + def save(self, variant: VariantRecord) -> None: + # Plain INSERT, no merge: overwriting a stored variant would swap a + # learner's graded task underneath its trace and grade (audit + # corruption), so a duplicate identity is a race or bug to SURFACE, + # not paper over. The generator's cache path (get before generate) + # makes duplicate saves a programming error, not a normal flow. + # The trace store swallows its IntegrityError (dedup is the + # contract there); the grade store merges (latest-wins is the + # contract there); this store re-raises (first-wins is the + # contract here). + with self._session() as session: + try: + session.add(variant) + session.commit() + except sa.exc.IntegrityError: + session.rollback() + logger.debug( + "variant insert rejected (identity already stored): " + "learner=%s template=%s task=%s", + variant.learner_id, + variant.template_id, + variant.task_id, + ) + raise + logger.debug( + "variant saved: %s/%s task=%s seed=%s", + variant.learner_id, + variant.template_id, + variant.task_id, + variant.seed, + ) + + def get(self, learner_id: str, template_id: str) -> VariantRecord | None: + with self._session() as session: + record = session.get(VariantRecord, (learner_id, template_id)) + if record is None: + return None + record.created_at = _as_utc(record.created_at) + # Detach from the session: callers must not depend on + # open-session ORM magic (lazy loads fail once it closes). + session.expunge(record) + return record + + def get_by_task(self, task_id: str) -> VariantRecord | None: + with self._session() as session: + stmt = select(VariantRecord).where(VariantRecord.task_id == task_id) + record = session.exec(stmt).first() + if record is None: + return None + record.created_at = _as_utc(record.created_at) + session.expunge(record) + return record + + def list_for_learner(self, learner_id: str) -> list[VariantRecord]: + with self._session() as session: + stmt = ( + select(VariantRecord) + .where(VariantRecord.learner_id == learner_id) + # Chronological; task_id is a deterministic tie-break for + # variants stamped within the same instant. + .order_by(VariantRecord.created_at, VariantRecord.task_id) + ) + results = session.exec(stmt).all() + for row in results: + row.created_at = _as_utc(row.created_at) + session.expunge(row) + return list(results) + + def list_by_template(self, template_id: str) -> list[VariantRecord]: + with self._session() as session: + stmt = ( + select(VariantRecord) + .where(VariantRecord.template_id == template_id) + # Chronological; learner_id is a deterministic tie-break. + .order_by(VariantRecord.created_at, VariantRecord.learner_id) + ) + results = session.exec(stmt).all() + for row in results: + row.created_at = _as_utc(row.created_at) + session.expunge(row) + return list(results) + + def close(self) -> None: + self._engine.dispose() diff --git a/apps/ai-service/ai_service/variants/templates.py b/apps/ai-service/ai_service/variants/templates.py new file mode 100644 index 0000000..5706ef8 --- /dev/null +++ b/apps/ai-service/ai_service/variants/templates.py @@ -0,0 +1,299 @@ +"""Task template library for seeded variant generation (D-029, REQ-3-005). + +A `TaskTemplate` binds a competency (D-021-aligned corpus ID), a statement +skeleton with `{slot}` placeholders, typed `ParameterSlot`s, difficulty- +normalization rubric anchors (the expected feature envelope the grader +prompt receives as context — a-5 makes "same bar" testable), and starter- +file scaffolds served into the sandbox workdir (wired in P6). + +Slot sampling is PURE CODE: `random.Random(seed)` over typed slots — fully +reproducible for a given seed, independent of the LLM. The LLM only renders +the seeded slot values into the statement skeleton (D-020 defense). +""" + +from __future__ import annotations + +import random +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +SlotType = Literal["enum", "int_range", "string_set"] + + +class ParameterSlot(BaseModel): + """One typed fill-in for a statement skeleton.""" + + model_config = ConfigDict(frozen=True) + + name: str = Field(min_length=1) + type: SlotType + values: list[str] = Field(default_factory=list) # enum/string_set options + lo: int | None = None # int_range bounds + hi: int | None = None + + @field_validator("values") + @classmethod + def _values_nonempty_for_enums(cls, v: list[str], info) -> list[str]: + if info.data.get("type") in ("enum", "string_set") and not v: + raise ValueError(f"slot {info.data.get('name')!r} needs values") + return v + + def sample(self, rng: random.Random) -> str | int: + """Deterministic sample from the seeded RNG. Validated after sampling.""" + if self.type == "enum" or self.type == "string_set": + return rng.choice(self.values) + if self.type == "int_range": + lo = self.lo if self.lo is not None else 0 + hi = self.hi if self.hi is not None else lo + if hi < lo: + raise ValueError(f"slot {self.name!r}: hi < lo") + return rng.randint(lo, hi) + raise ValueError(f"unsupported slot type: {self.type!r}") + + def validate_value(self, value: str | int) -> bool: + """Is `value` schema-valid for this slot? (params JSON gate, a-5.)""" + if self.type in ("enum", "string_set"): + return isinstance(value, str) and value in self.values + if self.type == "int_range": + lo = self.lo if self.lo is not None else 0 + hi = self.hi if self.hi is not None else lo + return isinstance(value, int) and lo <= value <= hi + return False + + +class RubricAnchors(BaseModel): + """Difficulty-normalization anchors shipped to the grader as context. + + Expected FEATURE ENVELOPE (digest-space): the grading prompt receives + these so two variants of one template are held to the same bar — the + anchors bound what "comparable effort" looks like for this template + regardless of which slot values a learner drew (a-5). + """ + + model_config = ConfigDict(frozen=True) + + expected_edit_count_band: tuple[int, int] + expected_min_test_runs: int + expected_error_fix_cycles_band: tuple[int, int] + notes: str = "" + + +class TaskTemplate(BaseModel): + """A reusable task shape; variants instantiate it per learner.""" + + model_config = ConfigDict(frozen=True) + + id: str = Field(min_length=1) + competency_id: str = Field(min_length=1) # D-021 corpus alignment + title: str + statement_skeleton: str = Field(min_length=1) # {slot} placeholders + slots: list[ParameterSlot] = Field(min_length=1) + rubric_anchors: RubricAnchors + starter_files: dict[str, str] = Field(default_factory=dict) # path -> content + test_command: str + + @field_validator("statement_skeleton") + @classmethod + def _skeleton_placeholders(cls, v: str) -> str: + if "{" not in v or "}" not in v: + raise ValueError("statement_skeleton needs at least one {slot}") + return v + + def render(self, params: dict[str, str | int]) -> str: + """Fill the skeleton with validated params.""" + for slot in self.slots: + if slot.name not in params: + raise ValueError(f"missing param for slot {slot.name!r}") + if not slot.validate_value(params[slot.name]): + raise ValueError(f"invalid value for slot {slot.name!r}: {params[slot.name]!r}") + return self.statement_skeleton.format(**params) + + def sample_params(self, seed: int) -> dict[str, str | int]: + """Seeded, reproducible, schema-valid slot values (pure code).""" + rng = random.Random(seed) + return {slot.name: slot.sample(rng) for slot in self.slots} + + +# --- Template library (v0.3 initial set) -------------------------------------- +# Competency IDs are D-021-aligned with the Python corpus +# (ai_service/corpus/learner_context.py) and the TS mock-data layer +# (packages/mock-data/competency-stacks.ts: deterministic cid() scheme). + +TEMPLATES: dict[str, TaskTemplate] = { + "tpl-llm-judge": TaskTemplate( + id="tpl-llm-judge", + competency_id="stack-orchestration-c007", + title="Build an LLM-as-Judge Evaluator", + statement_skeleton=( + "Build a small LLM-as-judge evaluator for {domain} answers. " + "The judge must score each answer on {criterion} using a 0-4 scale, " + "return structured JSON, and handle at least {edge_cases} edge-case " + "answer classes (empty, off-topic, adversarial). Include a tiny " + "repro test set of at least {test_size} examples and print a summary " + "table of scores." + ), + slots=[ + ParameterSlot( + name="domain", + type="enum", + values=["customer-support", "code-review", "summarization", "tutoring"], + ), + ParameterSlot( + name="criterion", + type="enum", + values=["factual-accuracy", "helpfulness", "safety", "completeness"], + ), + ParameterSlot(name="edge_cases", type="int_range", lo=2, hi=4), + ParameterSlot(name="test_size", type="int_range", lo=3, hi=8), + ], + rubric_anchors=RubricAnchors( + expected_edit_count_band=(3, 25), + expected_min_test_runs=2, + expected_error_fix_cycles_band=(0, 4), + notes="Slot draw changes the SCENARIO, not the engineering depth.", + ), + starter_files={ + "README.md": ( + "# LLM-as-Judge Evaluator\n\n" + "Implement `judge.py`:\n" + "- `score(answer: str) -> dict` — 0-4 on the named criterion\n" + "- structured JSON output (schema below)\n" + "- edge-case classes handled explicitly\n" + "- `pytest` must pass\n" + ), + "judge.py": "def score(answer: str) -> dict:\n raise NotImplementedError\n", + "test_judge.py": "def test_placeholder():\n assert True\n", + }, + test_command="pytest -q", + ), + "tpl-guardrail-schema": TaskTemplate( + id="tpl-guardrail-schema", + competency_id="stack-orchestration-c008", + title="Schema Guardrail Pipeline", + statement_skeleton=( + "Implement an output-validation guardrail for a model returning " + "{entity} records. Validate against a typed schema with {field_count} " + "required fields, coerce or reject {failure_mode} failures, and emit " + "a fallback response for invalid payloads. Cover with at least " + "{test_size} unit tests including malformed JSON." + ), + slots=[ + ParameterSlot( + name="entity", + type="enum", + values=["user-profile", "job-posting", "candidate", "invoice"], + ), + ParameterSlot( + name="failure_mode", + type="enum", + values=["strict-reject", "coerce-when-safe"], + ), + ParameterSlot(name="field_count", type="int_range", lo=4, hi=8), + ParameterSlot(name="test_size", type="int_range", lo=4, hi=10), + ], + rubric_anchors=RubricAnchors( + expected_edit_count_band=(3, 30), + expected_min_test_runs=2, + expected_error_fix_cycles_band=(0, 5), + notes="All slot draws land in the same engineering band.", + ), + starter_files={ + "README.md": ( + "# Schema Guardrail\n\nImplement `guardrail.py`:\n" + "- `validate(payload: dict) -> dict | Fallback`\n" + "- required-field checks, failure policy, fallback emission\n" + ), + "guardrail.py": "def validate(payload: dict):\n raise NotImplementedError\n", + "test_guardrail.py": "def test_placeholder():\n assert True\n", + }, + test_command="pytest -q", + ), + "tpl-rag-chunker": TaskTemplate( + id="tpl-rag-chunker", + competency_id="stack-orchestration-c005", + title="RAG Chunking Strategy", + statement_skeleton=( + "Implement a document chunker for {doc_type} retrieval. Support " + "{strategy} chunking with a target size of ~{chunk_size} tokens, " + "preserve {invariant} across chunk boundaries, and evaluate overlap " + "quality with at least {test_size} fixture documents." + ), + slots=[ + ParameterSlot( + name="doc_type", + type="enum", + values=["technical-docs", "legal-contracts", "transcripts"], + ), + ParameterSlot( + name="strategy", + type="enum", + values=["fixed-window", "semantic-boundary", "hybrid"], + ), + ParameterSlot( + name="invariant", + type="enum", + values=["code-block-integrity", "section-headers", "sentence-completeness"], + ), + ParameterSlot(name="chunk_size", type="int_range", lo=200, hi=800), + ParameterSlot(name="test_size", type="int_range", lo=3, hi=6), + ], + rubric_anchors=RubricAnchors( + expected_edit_count_band=(4, 35), + expected_min_test_runs=2, + expected_error_fix_cycles_band=(0, 6), + notes="Strategy draw changes implementation shape, not depth.", + ), + starter_files={ + "README.md": ( + "# RAG Chunker\n\nImplement `chunker.py`:\n" + "- `chunk(text: str) -> list[str]`\n- invariant preserved\n- tests green\n" + ), + "chunker.py": "def chunk(text: str) -> list[str]:\n raise NotImplementedError\n", + "test_chunker.py": "def test_placeholder():\n assert True\n", + }, + test_command="pytest -q", + ), +} + + +_KNOWN_COMPETENCY_IDS: set[str] = { + # D-021: mirrored from ai_service/corpus/learner_context.py — the Python + # source of truth for stack-orchestration competencies used by v0.2 agents. + "stack-orchestration-c001", + "stack-orchestration-c002", + "stack-orchestration-c003", + "stack-orchestration-c004", + "stack-orchestration-c005", + "stack-orchestration-c007", + "stack-orchestration-c008", + "stack-orchestration-c011", + "stack-designer-c001", + "stack-designer-c002", + "stack-safety-c021", +} + + +def get_template(template_id: str) -> TaskTemplate | None: + return TEMPLATES.get(template_id) + + +def template_for_competency(competency_id: str) -> list[TaskTemplate]: + return [t for t in TEMPLATES.values() if t.competency_id == competency_id] + + +def validate_competency_binding() -> None: + """All templates must bind to known D-021 corpus competency IDs.""" + for t in TEMPLATES.values(): + if t.competency_id not in _KNOWN_COMPETENCY_IDS: + raise ValueError( + f"template {t.id!r} binds unknown competency {t.competency_id!r}" + ) + + +def slots_pattern_ok(skeleton: str, slots: list[ParameterSlot]) -> bool: + """Every {placeholder} in the skeleton has a matching slot and vice versa.""" + placeholders = set(re.findall(r"\{([a-z_][a-z0-9_]*)\}", skeleton)) + slot_names = {s.name for s in slots} + return placeholders == slot_names diff --git a/apps/ai-service/tests/variants/__init__.py b/apps/ai-service/tests/variants/__init__.py new file mode 100644 index 0000000..4533003 --- /dev/null +++ b/apps/ai-service/tests/variants/__init__.py @@ -0,0 +1 @@ +"""Variant task generation tests (REQ-3-005).""" diff --git a/apps/ai-service/tests/variants/test_store.py b/apps/ai-service/tests/variants/test_store.py new file mode 100644 index 0000000..89ff65f --- /dev/null +++ b/apps/ai-service/tests/variants/test_store.py @@ -0,0 +1,358 @@ +"""SQLiteVariantStore tests (REQ-3-005, D-027). + +Each test gets its own tmp-path SQLite file — no shared disk state. Covers: + - save / get roundtrip by (learner_id, template_id) AND by task_id + (all fields survive, including nested JSON params, starter_files + filename->content map, and the tz-aware created_at contract) + - insert-only: a second save for the same (learner_id, template_id) + raises IntegrityError (first-wins, documented choice); the original + row is untouched — NOT upsert, NOT swallowed + - unique task_id: a second variant claiming an existing trace key is + rejected even under a different (learner, template) pair + - list_for_learner / list_by_template scoped + chronological + auditable + (seed + params readable back — the proctoring cross-check path) + - unknown learner / template / task -> None / empty lists + - scoping: rows for other learners/templates never leak + - rows are detached: usable after the store is closed + - WAL + synchronous=NORMAL pragmas actually applied to the DB file + - concurrent writer + reader against the same DB file (a-3 smoke test) +""" + +import concurrent.futures +import threading +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +import sqlalchemy as sa + +from ai_service.variants.store import SQLiteVariantStore, VariantRecord + +_BASE_TS = datetime(2026, 9, 12, 12, 0, 0, tzinfo=UTC) + + +def make_variant( + task_id: str = "task-1", + learner_id: str = "learner-1", + template_id: str = "template-1", + seed: str = "seed-a1b2c3", + params: dict[str, Any] | None = None, + statement: str | None = None, + starter_files: dict[str, Any] | None = None, + created_at: datetime | None = None, +) -> VariantRecord: + """Canonical kwargs builder — tests override only what they assert on.""" + return VariantRecord( + learner_id=learner_id, + template_id=template_id, + task_id=task_id, + seed=seed, + params=params + if params is not None + else { + "scenario": "cache invalidation", + "constraints": ["no external deps", "streaming"], + "data_shape": {"rows": 10_000, "columns": ["ts", "event", "user"]}, + }, + statement=statement + if statement is not None + else ( + "Implement a cache layer for the event stream service that " + "survives restarts without losing buffered rows, learner " + f"{learner_id} edition." + ), + starter_files=starter_files + if starter_files is not None + else { + "src/stream_cache.py": "class StreamCache:\n pass\n", + "tests/test_stream_cache.py": "def test_roundtrip():\n pass\n", + }, + created_at=created_at if created_at is not None else _BASE_TS, + ) + + +@pytest.fixture +def store(tmp_path: Path) -> SQLiteVariantStore: + s = SQLiteVariantStore(db_path=tmp_path / "variants.db") + yield s + s.close() + + +def test_save_and_get_roundtrip(store: SQLiteVariantStore) -> None: + variant = make_variant() + store.save(variant) + + # Point lookup by variant identity (learner_id, template_id). + fetched = store.get("learner-1", "template-1") + assert fetched is not None + assert fetched.learner_id == "learner-1" + assert fetched.template_id == "template-1" + assert fetched.task_id == "task-1" + assert fetched.seed == "seed-a1b2c3" + assert fetched.params == variant.params + assert fetched.statement == variant.statement + assert fetched.starter_files == variant.starter_files + assert fetched.created_at == _BASE_TS + assert fetched.created_at.tzinfo is UTC # tz-normalized on read + + # Same row via the grading/telemetry trace key — the join path a + # grade or proctor check uses with only a task_id in hand. + by_task = store.get_by_task("task-1") + assert by_task is not None + assert by_task.learner_id == "learner-1" + assert by_task.template_id == "template-1" + assert by_task.seed == "seed-a1b2c3" + assert by_task.statement == variant.statement + assert by_task.starter_files == variant.starter_files + assert by_task.params == variant.params + assert by_task.created_at.tzinfo is UTC + + +def test_get_unknown_returns_none(store: SQLiteVariantStore) -> None: + store.save(make_variant()) + + assert store.get("learner-1", "template-missing") is None + assert store.get("learner-missing", "template-1") is None + assert store.get("nobody", "nothing") is None + assert store.get_by_task("task-missing") is None + + +def test_duplicate_learner_template_pair_raises_integrity_error( + store: SQLiteVariantStore, +) -> None: + """THE contract of this store (insert-only, first wins). + + The first generated variant is authoritative (D-029 reproducibility: + the seed re-derives the same variant); a duplicate save is a + programming error or lost race, not a normal flow — the generator's + cache path serves `get` instead. So the IntegrityError surfaces to + the caller and the stored row is left untouched. + """ + first = make_variant(statement="first authoritative statement") + store.save(first) + + second = make_variant( + task_id="task-2", # distinct task key; the PAIR collides + seed="seed-999", + statement="an impostor statement", + created_at=_BASE_TS + timedelta(hours=1), + ) + with pytest.raises(sa.exc.IntegrityError): + store.save(second) + + # First save survived intact — nothing was overwritten. + fetched = store.get("learner-1", "template-1") + assert fetched is not None + assert fetched.task_id == "task-1" + assert fetched.statement == "first authoritative statement" + assert fetched.seed == "seed-a1b2c3" + assert fetched.created_at == _BASE_TS + + # The rejected save left no row behind under its task_id either. + assert store.get_by_task("task-2") is None + + +def test_duplicate_task_id_raises_integrity_error(store: SQLiteVariantStore) -> None: + # task_id is the grading/telemetry trace key — globally unique: a + # second variant may never claim an existing trace key, even under a + # different (learner_id, template_id) pair. + store.save(make_variant(learner_id="learner-1", template_id="template-1")) + + with pytest.raises(sa.exc.IntegrityError): + store.save( + make_variant( + learner_id="learner-2", + template_id="template-2", + task_id="task-1", # collides with learner-1's trace key + ) + ) + + # Rejected row not partially stored under either identity. + assert store.get("learner-2", "template-2") is None + assert len(store.list_by_template("template-2")) == 0 + + +def test_list_for_learner_roundtrip_and_audit(store: SQLiteVariantStore) -> None: + # Created out of insertion order; list must come back chronological. + store.save(make_variant(template_id="template-c", task_id="task-c", + created_at=_BASE_TS + timedelta(hours=2))) + store.save(make_variant(template_id="template-a", task_id="task-a", + created_at=_BASE_TS)) + store.save(make_variant(template_id="template-b", task_id="task-b", + created_at=_BASE_TS + timedelta(hours=1))) + + variants = store.list_for_learner("learner-1") + assert [v.template_id for v in variants] == [ + "template-a", + "template-b", + "template-c", + ] + assert all(v.learner_id == "learner-1" for v in variants) + hours = (timedelta(hours=0), timedelta(hours=1), timedelta(hours=2)) + assert all( + v.created_at == _BASE_TS + offset + for v, offset in zip(variants, hours, strict=True) + ) + assert all(v.created_at.tzinfo is UTC for v in variants) + + # Auditable: every stored variant reads back its seed and typed params + # (the proctoring cross-check path reads exactly this). + for v in variants: + assert v.seed.startswith("seed-") + assert v.params["scenario"] == "cache invalidation" + assert "streaming" in v.params["constraints"] + assert v.params["data_shape"]["rows"] == 10_000 + assert v.starter_files["tests/test_stream_cache.py"].count("\n") >= 1 + + +def test_list_by_template_roundtrip_and_audit(store: SQLiteVariantStore) -> None: + # Three learners on the same template: distinct, auditable variants. + store.save(make_variant(learner_id="learner-b", task_id="task-b", + seed="seed-222", + created_at=_BASE_TS + timedelta(hours=1))) + store.save(make_variant(learner_id="learner-a", task_id="task-a", + seed="seed-111", created_at=_BASE_TS)) + store.save(make_variant(learner_id="learner-c", task_id="task-c", + seed="seed-333", + created_at=_BASE_TS + timedelta(hours=2))) + + variants = store.list_by_template("template-1") + assert [v.learner_id for v in variants] == ["learner-a", "learner-b", "learner-c"] + assert all(v.template_id == "template-1" for v in variants) + + # Every learner's variant carries its own seed + params (auditable, + # REQ-3-005: variant parameters persisted and auditable). + seeds = {v.learner_id: v.seed for v in variants} + assert seeds == { + "learner-a": "seed-111", + "learner-b": "seed-222", + "learner-c": "seed-333", + } + assert all(v.params["scenario"] == "cache invalidation" for v in variants) + assert all(v.created_at.tzinfo is UTC for v in variants) + + +def test_list_unknown_returns_empty_lists(store: SQLiteVariantStore) -> None: + store.save(make_variant()) + + assert store.list_for_learner("nobody") == [] + assert store.list_by_template("no-template") == [] + + +def test_lists_are_scoped(store: SQLiteVariantStore) -> None: + store.save(make_variant(learner_id="learner-1", template_id="template-1", + task_id="task-1")) + store.save(make_variant(learner_id="learner-2", template_id="template-1", + task_id="task-2")) + store.save(make_variant(learner_id="learner-1", template_id="template-2", + task_id="task-3")) + + # learner lists see only that learner's rows. + assert [v.template_id for v in store.list_for_learner("learner-1")] == [ + "template-1", + "template-2", + ] + assert [v.template_id for v in store.list_for_learner("learner-2")] == ["template-1"] + + # template lists see one row per learner, none from other templates. + template_rows = store.list_by_template("template-1") + assert sorted(v.learner_id for v in template_rows) == ["learner-1", "learner-2"] + assert all(v.template_id == "template-1" for v in template_rows) + + # get stays a pair-scoped point lookup: same template, other learner. + assert store.get("learner-1", "template-2") is not None + assert store.get("learner-2", "template-2") is None + + +def test_empty_params_and_starter_files_roundtrip(store: SQLiteVariantStore) -> None: + # Legal shapes: a slotless template carries no params; a variant may + # ship without a workspace scaffold. + store.save(make_variant(params={}, starter_files={})) + + fetched = store.get("learner-1", "template-1") + assert fetched is not None + assert fetched.params == {} + assert fetched.starter_files == {} + + +def test_rows_are_detached_after_save(store: SQLiteVariantStore, tmp_path: Path) -> None: + # The API layer hands VariantRecords across layers; rows must survive + # the store that produced them being closed (no open-session ORM magic). + store.save(make_variant()) + fetched = store.get("learner-1", "template-1") + by_task = store.get_by_task("task-1") + store.close() + + assert fetched is not None + assert fetched.statement == fetched.statement # usable post-close + assert fetched.starter_files["src/stream_cache.py"] == "class StreamCache:\n pass\n" + assert by_task is not None + assert by_task.seed == "seed-a1b2c3" + + # A fresh store on the same file sees the same row (durability). + reopened = SQLiteVariantStore(db_path=tmp_path / "variants.db") + try: + again = reopened.get("learner-1", "template-1") + assert again is not None + assert again.starter_files["src/stream_cache.py"].startswith("class StreamCache") + assert again.created_at.tzinfo is UTC + finally: + reopened.close() + + +def test_pragmas_are_applied(store: SQLiteVariantStore) -> None: + # Pragmas are per-connection; query through the store's engine so the + # connect hook (not a default sqlite3 connection) is what we inspect. + with store._engine.connect() as conn: + (journal_mode,) = conn.execute(sa.text("PRAGMA journal_mode")).one() + (synchronous,) = conn.execute(sa.text("PRAGMA synchronous")).one() + assert journal_mode == "wal" + # synchronous=NORMAL is 1 in SQLite's pragma numbering. + assert synchronous == 1 + + +def test_concurrent_writer_and_reader_no_database_is_locked(tmp_path: Path) -> None: + """One thread saves while another reads in a tight loop (a-3). + + Without WAL + busy_timeout this pattern reliably produces + `OperationalError: database is locked` on SQLite. The assertion is + that every reader call completes and every distinct-row write lands. + """ + db_path = tmp_path / "variants.db" + n_variants = 60 # distinct (learner, template) pairs, one save each + stop_writing = threading.Event() + + writer = SQLiteVariantStore(db_path=db_path) + reader = SQLiteVariantStore(db_path=db_path) + try: + + def write_variants() -> None: + for seq in range(n_variants): + writer.save( + make_variant( + learner_id=f"learner-{seq}", + template_id="template-1", + task_id=f"task-{seq}", + seed=f"seed-{seq:03d}", + created_at=_BASE_TS + timedelta(seconds=seq), + ) + ) + stop_writing.set() + + def read_variants() -> None: + while not stop_writing.is_set(): + reader.list_by_template("template-1") + # Final read after the writer is done. + assert len(reader.list_by_template("template-1")) == n_variants + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(write_variants), pool.submit(read_variants)] + for future in futures: + future.result(timeout=30) + + # Every save was a distinct row: nothing lost, none doubled. + assert len(writer.list_by_template("template-1")) == n_variants + finally: + reader.close() + writer.close() diff --git a/apps/ai-service/tests/variants/test_templates.py b/apps/ai-service/tests/variants/test_templates.py new file mode 100644 index 0000000..deefe58 --- /dev/null +++ b/apps/ai-service/tests/variants/test_templates.py @@ -0,0 +1,102 @@ +"""Task template tests (Task 4-1-01, REQ-3-005).""" + +from __future__ import annotations + +import random + +import pytest + +from ai_service.variants.templates import ( + TEMPLATES, + ParameterSlot, + TaskTemplate, + get_template, + slots_pattern_ok, + template_for_competency, + validate_competency_binding, +) + + +def test_all_templates_bind_to_real_competency_ids() -> None: + validate_competency_binding() # raises on any unknown binding + assert len(TEMPLATES) >= 3 + + +def test_slot_validation_rejects_bad_values() -> None: + slot = ParameterSlot(name="domain", type="enum", values=["a", "b"]) + assert slot.validate_value("a") + assert not slot.validate_value("c") + assert not slot.validate_value(3) + + rng = random.Random(42) + assert slot.sample(rng) in {"a", "b"} + + +def test_int_range_slot_bounds() -> None: + slot = ParameterSlot(name="n", type="int_range", lo=2, hi=4) + rng = random.Random(0) + for _ in range(20): + assert 2 <= slot.sample(rng) <= 4 + assert slot.validate_value(3) + assert not slot.validate_value(5) + assert not slot.validate_value("3") + + +def test_seeded_sampling_is_reproducible() -> None: + tpl = TEMPLATES["tpl-llm-judge"] + first = tpl.sample_params(seed=1234) + second = tpl.sample_params(seed=1234) + other = tpl.sample_params(seed=1235) + assert first == second # D-029: same seed -> identical params + assert first != other # different seed -> (near-certainly) different draw + + +def test_skeleton_placeholders_match_slots() -> None: + for tpl in TEMPLATES.values(): + assert slots_pattern_ok(tpl.statement_skeleton, tpl.slots), tpl.id + + +def test_render_validates_params_and_fills() -> None: + tpl = TEMPLATES["tpl-llm-judge"] + params = tpl.sample_params(seed=7) + rendered = tpl.render(params) + for value in params.values(): + assert str(value) in rendered + with pytest.raises(ValueError, match="invalid value"): + tpl.render({**params, "edge_cases": 99}) # out of band + + +def test_rubric_anchors_present_per_template() -> None: + for tpl in TEMPLATES.values(): + anchors = tpl.rubric_anchors + assert anchors.expected_edit_count_band[0] <= anchors.expected_edit_count_band[1] + assert anchors.expected_min_test_runs >= 1 + cycles = anchors.expected_error_fix_cycles_band + assert cycles[0] <= cycles[1] + + +def test_starter_files_defined_per_template() -> None: + for tpl in TEMPLATES.values(): + assert tpl.starter_files, f"{tpl.id} missing starter scaffolds" + assert "README.md" in tpl.starter_files + assert tpl.test_command + + +def test_competency_lookup() -> None: + tpls = template_for_competency("stack-orchestration-c005") + assert len(tpls) == 1 + assert get_template("nope-xyz") is None + + +def test_bad_skeleton_rejected() -> None: + with pytest.raises(ValueError, match="slot"): + TaskTemplate( + id="tpl-bad", + competency_id="stack-orchestration-c001", + title="Bad", + statement_skeleton="no placeholders at all", + slots=[ParameterSlot(name="x", type="enum", values=["a"])], + rubric_anchors=TEMPLATES["tpl-llm-judge"].rubric_anchors, + starter_files={}, + test_command="pytest -q", + )