feat(P04): task template library + VariantStore (Wave 1)
Task 4-1-01: variants/templates.py — 3 task templates (llm-judge, guardrail-schema,
rag-chunker) bound to D-021 corpus competency IDs; typed ParameterSlots
(enum/int_range/string-set) with seeded pure-code sampler (random.Random(seed));
RubricAnchors difficulty-normalization envelope; starter-file scaffolds + test command.
Task 4-1-02: VariantStore protocol + SQLiteVariantStore (insert-only first-wins;
unique (learner,template) + unique task_id; WAL; tz-normalized) — the audit trail
for proctoring cross-checks.
22 variant tests green; ruff clean.
---ci---
phase: 4
milestone: v0.3
status: execute
requirements: {covered: [REQ-3-005], partial: []}
---/ci---
This commit is contained in:
CIAgent
@@ -0,0 +1,29 @@
|
||||
"""Per-learner variant task generation — templates, generator, VariantStore (REQ-3-005).
|
||||
|
||||
Boundary rule (D-027): variants/ is an engine module — it never imports
|
||||
api/ or agents/ (api/ composes the generator and store via DI; store.py
|
||||
imports config only).
|
||||
|
||||
CO-ORDINATION NOTE (ADD, don't REMOVE — same convention as grading/):
|
||||
This __init__.py is a minimal placeholder created by the VariantStore
|
||||
task (4-1-02). The templates task (4-1-01) owns this file's final shape
|
||||
— when templates.py lands, ADD its exports alongside these; do not
|
||||
remove the store exports below.
|
||||
|
||||
Wave status: store.py (VariantRecord, VariantStore, SQLiteVariantStore)
|
||||
landed in Wave 1 (task 4-1-02); templates.py is Wave 1 task 4-1-01;
|
||||
generator.py is Wave 2 (4-2-01).
|
||||
"""
|
||||
|
||||
from .store import SQLiteVariantStore, VariantRecord, VariantStore
|
||||
from .templates import TEMPLATES, TaskTemplate, get_template, template_for_competency
|
||||
|
||||
__all__ = [
|
||||
"SQLiteVariantStore",
|
||||
"TEMPLATES",
|
||||
"TaskTemplate",
|
||||
"VariantRecord",
|
||||
"VariantStore",
|
||||
"get_template",
|
||||
"template_for_competency",
|
||||
]
|
||||
@@ -0,0 +1,311 @@
|
||||
"""VariantStore — variant persistence protocol + SQLite implementation (REQ-3-005, D-027).
|
||||
|
||||
Postgres-migration-ready (D-027): the protocol is the only surface the
|
||||
variant generator and API layers touch; swapping SQLiteVariantStore for a
|
||||
Postgres-backed implementation must not change call sites. The
|
||||
`variant_record` table uses only portable column types (str / JSON /
|
||||
datetime), so the same SQLModel schema stands up unchanged on Postgres.
|
||||
|
||||
Insert-only, NOT upsert: (learner_id, template_id) is the variant identity
|
||||
and the FIRST generation is authoritative — reproducibility (D-029) means
|
||||
the seed re-derives the same variant, so the generator's cache path serves
|
||||
`get` instead of saving again. `save` is a plain INSERT; a duplicate pair
|
||||
raises sqlalchemy.exc.IntegrityError to the caller (documented behavior).
|
||||
`task_id` is unique too — it is the grading/telemetry trace key, so a
|
||||
trace or grade can never silently join to a different variant. Both
|
||||
rejections are deliberate: overwriting a stored variant would swap a
|
||||
learner's graded task underneath its trace and grade (audit corruption).
|
||||
Contrast TraceStore.append (dedup-keep-first, swallowed — at-least-once
|
||||
ingest) and GradeStore.save (upsert-latest-wins — a regrade is
|
||||
latest-state); this store is the third contract of the D-027 family.
|
||||
|
||||
Concurrency (a-3): the store enables WAL + synchronous=NORMAL and a busy
|
||||
timeout at connection time, so a generation writer and API readers do not
|
||||
hit `database is locked` on the single-box pilot.
|
||||
|
||||
`created_at` contract: callers stamp UTC (datetime.now(UTC)); SQLite
|
||||
stores it naive and the read paths re-label it tz-aware UTC (same
|
||||
boundary normalization as TelemetryEvent.ts / GradeRecord.created_at, so
|
||||
the contract holds on any backend).
|
||||
|
||||
Boundary (D-027): `variants/` never imports `agents/` / `api/`; this
|
||||
module imports config only.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import JSON, Index, UniqueConstraint
|
||||
from sqlalchemy.orm import validates
|
||||
from sqlmodel import Field, Session, SQLModel, create_engine, select
|
||||
|
||||
from ..config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VariantRecord(SQLModel, table=True):
|
||||
"""A persisted task variant; (learner_id, template_id) is the PK — first wins.
|
||||
|
||||
Written once by the variant generator (Task 4-2-01), read by the API
|
||||
layer and proctoring cross-checks through the VariantStore protocol.
|
||||
Constraint enforcement mirrors TelemetryEvent / GradeRecord: sqlmodel
|
||||
0.0.42's metaclass drops pydantic constraints on table models, so
|
||||
SQLAlchemy `@validates` hooks enforce instead and the column types
|
||||
stay Postgres-ready (D-027).
|
||||
|
||||
Field contract:
|
||||
learner_id — non-empty learner identifier (same id space as
|
||||
traces and grades).
|
||||
template_id — non-empty task template identifier; variant
|
||||
identity is the (learner_id, template_id) pair —
|
||||
the pair the generator caches on (exactly one
|
||||
variant per learner per template).
|
||||
task_id — non-empty, GLOBALLY unique task identifier; the
|
||||
grading/telemetry trace key (the (learner_id,
|
||||
task_id) pair TraceStore / GradeStore key on),
|
||||
stamped at generation so a variant's trace and
|
||||
grade join back to it exactly once.
|
||||
seed — non-empty variant seed (D-029); derived from
|
||||
(template_id, learner_id, milestone) so the
|
||||
variant is reproducible and auditable.
|
||||
params — typed parameter-slot values the generator filled;
|
||||
JSON dict. An empty dict is legal (a slotless
|
||||
template).
|
||||
statement — non-empty rendered task statement shown to the
|
||||
learner (distinct per learner by construction,
|
||||
REQ-3-005).
|
||||
starter_files — workspace scaffold: filename -> file content;
|
||||
JSON dict. An empty dict is legal (no scaffold).
|
||||
created_at — UTC generation timestamp.
|
||||
"""
|
||||
|
||||
__tablename__ = "variant_record"
|
||||
# The composite PK covers (learner_id, template_id) point lookups; the
|
||||
# unique task_id covers get_by_task (the grading/telemetry join path);
|
||||
# the two secondary indexes cover list_for_learner / list_by_template
|
||||
# ordered by created_at without a sort step (Postgres target D-027).
|
||||
__table_args__ = (
|
||||
UniqueConstraint("task_id", name="uq_variant_record_task_id"),
|
||||
Index("ix_variant_record_learner_created", "learner_id", "created_at"),
|
||||
Index("ix_variant_record_template_created", "template_id", "created_at"),
|
||||
)
|
||||
|
||||
learner_id: str = Field(primary_key=True)
|
||||
template_id: str = Field(primary_key=True)
|
||||
task_id: str
|
||||
seed: str
|
||||
# JSON columns: stored as TEXT on SQLite, native JSONB on Postgres (D-027).
|
||||
params: dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
|
||||
statement: str
|
||||
starter_files: dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
|
||||
created_at: datetime
|
||||
|
||||
@validates("learner_id", "template_id", "task_id")
|
||||
def _ids_non_empty(self, key: str, value: str) -> str:
|
||||
if not value:
|
||||
raise ValueError(f"{key} must be a non-empty identifier")
|
||||
return value
|
||||
|
||||
@validates("seed")
|
||||
def _seed_non_empty(self, key: str, value: str) -> str:
|
||||
if not value:
|
||||
raise ValueError(f"{key} must be a non-empty seed string")
|
||||
return value
|
||||
|
||||
@validates("statement")
|
||||
def _statement_non_empty(self, key: str, value: str) -> str:
|
||||
if not value:
|
||||
raise ValueError(f"{key} must be a non-empty statement string")
|
||||
return value
|
||||
|
||||
|
||||
class VariantStore(Protocol):
|
||||
"""Persistence contract for reproducible per-learner task variants.
|
||||
|
||||
Implemented by SQLiteVariantStore (v0.3, D-027); a Postgres
|
||||
implementation must satisfy the same surface.
|
||||
"""
|
||||
|
||||
def save(self, variant: VariantRecord) -> None:
|
||||
"""Persist a new variant. INSERT-ONLY on (learner_id, template_id):
|
||||
the FIRST generated variant is authoritative (reproducibility,
|
||||
D-029); a duplicate pair raises sqlalchemy.exc.IntegrityError to
|
||||
the caller — the generator serves cached variants via `get`
|
||||
instead of saving again. `task_id` is unique too: claiming an
|
||||
existing trace key for a different variant is equally rejected.
|
||||
NOT upsert; contrast GradeStore.save (latest-wins) and
|
||||
TraceStore.append (dedup-keep-first, swallowed).
|
||||
"""
|
||||
...
|
||||
|
||||
def get(self, learner_id: str, template_id: str) -> VariantRecord | None:
|
||||
"""The learner's stored variant for the template; None when none
|
||||
exists. Detached from any DB session — safe to pass across layers.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_by_task(self, task_id: str) -> VariantRecord | None:
|
||||
"""The variant owning the task key (the grading/telemetry join
|
||||
path); None when none exists. Detached from any DB session.
|
||||
"""
|
||||
...
|
||||
|
||||
def list_for_learner(self, learner_id: str) -> list[VariantRecord]:
|
||||
"""All stored variants for the learner, ordered by created_at
|
||||
ascending (chronological; task_id breaks same-instant ties).
|
||||
Empty list when the learner has none.
|
||||
"""
|
||||
...
|
||||
|
||||
def list_by_template(self, template_id: str) -> list[VariantRecord]:
|
||||
"""All stored variants generated from the template — one row per
|
||||
learner — ordered by created_at ascending (chronological;
|
||||
learner_id breaks same-instant ties). Empty list when the
|
||||
template has none. The proctoring cross-check path (seed params
|
||||
per learner) reads through this.
|
||||
"""
|
||||
...
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release DB connections. Store must not be used after close."""
|
||||
...
|
||||
|
||||
|
||||
def _sqlite_connect(dbapi_connection: sqlite3.Connection, _: object) -> None:
|
||||
"""Per-connection pragma setup (a-3). Mirrors telemetry/grading stores.
|
||||
|
||||
journal_mode=WAL — readers never block the single writer.
|
||||
synchronous=NORMAL — safe in WAL mode, avoids full fsync-per-commit.
|
||||
busy_timeout=5000 — retry briefly under contention instead of
|
||||
`OperationalError: database is locked`.
|
||||
"""
|
||||
cursor = dbapi_connection.cursor()
|
||||
cursor.execute("PRAGMA journal_mode=WAL")
|
||||
cursor.execute("PRAGMA synchronous=NORMAL")
|
||||
cursor.execute("PRAGMA busy_timeout=5000")
|
||||
cursor.close()
|
||||
|
||||
|
||||
def _as_utc(ts: datetime) -> datetime:
|
||||
"""Timestamps travel as naive datetime on SQLite, tz-aware elsewhere.
|
||||
|
||||
SQLite (via SQLModel) drops tzinfo; Postgres TIMESTAMP WITH TIME ZONE
|
||||
keeps it. Normalizing on the read path makes the store's contract
|
||||
tz-aware UTC regardless of the backend (D-027).
|
||||
"""
|
||||
if ts.tzinfo is None:
|
||||
return ts.replace(tzinfo=UTC) # naive UTC read-side: label as UTC
|
||||
return ts.astimezone(UTC)
|
||||
|
||||
|
||||
class SQLiteVariantStore:
|
||||
"""SQLite-backed VariantStore (SQLModel). Third protocol-wrapped store
|
||||
of the D-027 family (first: SQLiteTraceStore, second: SQLiteGradeStore).
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Path | None = None) -> None:
|
||||
self._db_path: Path = db_path if db_path is not None else Settings().db_path
|
||||
self._engine = create_engine(f"sqlite:///{self._db_path}")
|
||||
sa.event.listen(self._engine, "connect", _sqlite_connect)
|
||||
SQLModel.metadata.create_all(self._engine)
|
||||
|
||||
@contextmanager
|
||||
def _session(self) -> Iterator[Session]:
|
||||
# expire_on_commit=False: identical session behavior to the other
|
||||
# D-027 stores. save() never commits on the error path and the read
|
||||
# paths never commit, but a uniform flag across the family keeps
|
||||
# their detachment guarantees from diverging.
|
||||
with Session(self._engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
|
||||
def save(self, variant: VariantRecord) -> None:
|
||||
# Plain INSERT, no merge: overwriting a stored variant would swap a
|
||||
# learner's graded task underneath its trace and grade (audit
|
||||
# corruption), so a duplicate identity is a race or bug to SURFACE,
|
||||
# not paper over. The generator's cache path (get before generate)
|
||||
# makes duplicate saves a programming error, not a normal flow.
|
||||
# The trace store swallows its IntegrityError (dedup is the
|
||||
# contract there); the grade store merges (latest-wins is the
|
||||
# contract there); this store re-raises (first-wins is the
|
||||
# contract here).
|
||||
with self._session() as session:
|
||||
try:
|
||||
session.add(variant)
|
||||
session.commit()
|
||||
except sa.exc.IntegrityError:
|
||||
session.rollback()
|
||||
logger.debug(
|
||||
"variant insert rejected (identity already stored): "
|
||||
"learner=%s template=%s task=%s",
|
||||
variant.learner_id,
|
||||
variant.template_id,
|
||||
variant.task_id,
|
||||
)
|
||||
raise
|
||||
logger.debug(
|
||||
"variant saved: %s/%s task=%s seed=%s",
|
||||
variant.learner_id,
|
||||
variant.template_id,
|
||||
variant.task_id,
|
||||
variant.seed,
|
||||
)
|
||||
|
||||
def get(self, learner_id: str, template_id: str) -> VariantRecord | None:
|
||||
with self._session() as session:
|
||||
record = session.get(VariantRecord, (learner_id, template_id))
|
||||
if record is None:
|
||||
return None
|
||||
record.created_at = _as_utc(record.created_at)
|
||||
# Detach from the session: callers must not depend on
|
||||
# open-session ORM magic (lazy loads fail once it closes).
|
||||
session.expunge(record)
|
||||
return record
|
||||
|
||||
def get_by_task(self, task_id: str) -> VariantRecord | None:
|
||||
with self._session() as session:
|
||||
stmt = select(VariantRecord).where(VariantRecord.task_id == task_id)
|
||||
record = session.exec(stmt).first()
|
||||
if record is None:
|
||||
return None
|
||||
record.created_at = _as_utc(record.created_at)
|
||||
session.expunge(record)
|
||||
return record
|
||||
|
||||
def list_for_learner(self, learner_id: str) -> list[VariantRecord]:
|
||||
with self._session() as session:
|
||||
stmt = (
|
||||
select(VariantRecord)
|
||||
.where(VariantRecord.learner_id == learner_id)
|
||||
# Chronological; task_id is a deterministic tie-break for
|
||||
# variants stamped within the same instant.
|
||||
.order_by(VariantRecord.created_at, VariantRecord.task_id)
|
||||
)
|
||||
results = session.exec(stmt).all()
|
||||
for row in results:
|
||||
row.created_at = _as_utc(row.created_at)
|
||||
session.expunge(row)
|
||||
return list(results)
|
||||
|
||||
def list_by_template(self, template_id: str) -> list[VariantRecord]:
|
||||
with self._session() as session:
|
||||
stmt = (
|
||||
select(VariantRecord)
|
||||
.where(VariantRecord.template_id == template_id)
|
||||
# Chronological; learner_id is a deterministic tie-break.
|
||||
.order_by(VariantRecord.created_at, VariantRecord.learner_id)
|
||||
)
|
||||
results = session.exec(stmt).all()
|
||||
for row in results:
|
||||
row.created_at = _as_utc(row.created_at)
|
||||
session.expunge(row)
|
||||
return list(results)
|
||||
|
||||
def close(self) -> None:
|
||||
self._engine.dispose()
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Task template library for seeded variant generation (D-029, REQ-3-005).
|
||||
|
||||
A `TaskTemplate` binds a competency (D-021-aligned corpus ID), a statement
|
||||
skeleton with `{slot}` placeholders, typed `ParameterSlot`s, difficulty-
|
||||
normalization rubric anchors (the expected feature envelope the grader
|
||||
prompt receives as context — a-5 makes "same bar" testable), and starter-
|
||||
file scaffolds served into the sandbox workdir (wired in P6).
|
||||
|
||||
Slot sampling is PURE CODE: `random.Random(seed)` over typed slots — fully
|
||||
reproducible for a given seed, independent of the LLM. The LLM only renders
|
||||
the seeded slot values into the statement skeleton (D-020 defense).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
SlotType = Literal["enum", "int_range", "string_set"]
|
||||
|
||||
|
||||
class ParameterSlot(BaseModel):
|
||||
"""One typed fill-in for a statement skeleton."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
name: str = Field(min_length=1)
|
||||
type: SlotType
|
||||
values: list[str] = Field(default_factory=list) # enum/string_set options
|
||||
lo: int | None = None # int_range bounds
|
||||
hi: int | None = None
|
||||
|
||||
@field_validator("values")
|
||||
@classmethod
|
||||
def _values_nonempty_for_enums(cls, v: list[str], info) -> list[str]:
|
||||
if info.data.get("type") in ("enum", "string_set") and not v:
|
||||
raise ValueError(f"slot {info.data.get('name')!r} needs values")
|
||||
return v
|
||||
|
||||
def sample(self, rng: random.Random) -> str | int:
|
||||
"""Deterministic sample from the seeded RNG. Validated after sampling."""
|
||||
if self.type == "enum" or self.type == "string_set":
|
||||
return rng.choice(self.values)
|
||||
if self.type == "int_range":
|
||||
lo = self.lo if self.lo is not None else 0
|
||||
hi = self.hi if self.hi is not None else lo
|
||||
if hi < lo:
|
||||
raise ValueError(f"slot {self.name!r}: hi < lo")
|
||||
return rng.randint(lo, hi)
|
||||
raise ValueError(f"unsupported slot type: {self.type!r}")
|
||||
|
||||
def validate_value(self, value: str | int) -> bool:
|
||||
"""Is `value` schema-valid for this slot? (params JSON gate, a-5.)"""
|
||||
if self.type in ("enum", "string_set"):
|
||||
return isinstance(value, str) and value in self.values
|
||||
if self.type == "int_range":
|
||||
lo = self.lo if self.lo is not None else 0
|
||||
hi = self.hi if self.hi is not None else lo
|
||||
return isinstance(value, int) and lo <= value <= hi
|
||||
return False
|
||||
|
||||
|
||||
class RubricAnchors(BaseModel):
|
||||
"""Difficulty-normalization anchors shipped to the grader as context.
|
||||
|
||||
Expected FEATURE ENVELOPE (digest-space): the grading prompt receives
|
||||
these so two variants of one template are held to the same bar — the
|
||||
anchors bound what "comparable effort" looks like for this template
|
||||
regardless of which slot values a learner drew (a-5).
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
expected_edit_count_band: tuple[int, int]
|
||||
expected_min_test_runs: int
|
||||
expected_error_fix_cycles_band: tuple[int, int]
|
||||
notes: str = ""
|
||||
|
||||
|
||||
class TaskTemplate(BaseModel):
|
||||
"""A reusable task shape; variants instantiate it per learner."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str = Field(min_length=1)
|
||||
competency_id: str = Field(min_length=1) # D-021 corpus alignment
|
||||
title: str
|
||||
statement_skeleton: str = Field(min_length=1) # {slot} placeholders
|
||||
slots: list[ParameterSlot] = Field(min_length=1)
|
||||
rubric_anchors: RubricAnchors
|
||||
starter_files: dict[str, str] = Field(default_factory=dict) # path -> content
|
||||
test_command: str
|
||||
|
||||
@field_validator("statement_skeleton")
|
||||
@classmethod
|
||||
def _skeleton_placeholders(cls, v: str) -> str:
|
||||
if "{" not in v or "}" not in v:
|
||||
raise ValueError("statement_skeleton needs at least one {slot}")
|
||||
return v
|
||||
|
||||
def render(self, params: dict[str, str | int]) -> str:
|
||||
"""Fill the skeleton with validated params."""
|
||||
for slot in self.slots:
|
||||
if slot.name not in params:
|
||||
raise ValueError(f"missing param for slot {slot.name!r}")
|
||||
if not slot.validate_value(params[slot.name]):
|
||||
raise ValueError(f"invalid value for slot {slot.name!r}: {params[slot.name]!r}")
|
||||
return self.statement_skeleton.format(**params)
|
||||
|
||||
def sample_params(self, seed: int) -> dict[str, str | int]:
|
||||
"""Seeded, reproducible, schema-valid slot values (pure code)."""
|
||||
rng = random.Random(seed)
|
||||
return {slot.name: slot.sample(rng) for slot in self.slots}
|
||||
|
||||
|
||||
# --- Template library (v0.3 initial set) --------------------------------------
|
||||
# Competency IDs are D-021-aligned with the Python corpus
|
||||
# (ai_service/corpus/learner_context.py) and the TS mock-data layer
|
||||
# (packages/mock-data/competency-stacks.ts: deterministic cid() scheme).
|
||||
|
||||
TEMPLATES: dict[str, TaskTemplate] = {
|
||||
"tpl-llm-judge": TaskTemplate(
|
||||
id="tpl-llm-judge",
|
||||
competency_id="stack-orchestration-c007",
|
||||
title="Build an LLM-as-Judge Evaluator",
|
||||
statement_skeleton=(
|
||||
"Build a small LLM-as-judge evaluator for {domain} answers. "
|
||||
"The judge must score each answer on {criterion} using a 0-4 scale, "
|
||||
"return structured JSON, and handle at least {edge_cases} edge-case "
|
||||
"answer classes (empty, off-topic, adversarial). Include a tiny "
|
||||
"repro test set of at least {test_size} examples and print a summary "
|
||||
"table of scores."
|
||||
),
|
||||
slots=[
|
||||
ParameterSlot(
|
||||
name="domain",
|
||||
type="enum",
|
||||
values=["customer-support", "code-review", "summarization", "tutoring"],
|
||||
),
|
||||
ParameterSlot(
|
||||
name="criterion",
|
||||
type="enum",
|
||||
values=["factual-accuracy", "helpfulness", "safety", "completeness"],
|
||||
),
|
||||
ParameterSlot(name="edge_cases", type="int_range", lo=2, hi=4),
|
||||
ParameterSlot(name="test_size", type="int_range", lo=3, hi=8),
|
||||
],
|
||||
rubric_anchors=RubricAnchors(
|
||||
expected_edit_count_band=(3, 25),
|
||||
expected_min_test_runs=2,
|
||||
expected_error_fix_cycles_band=(0, 4),
|
||||
notes="Slot draw changes the SCENARIO, not the engineering depth.",
|
||||
),
|
||||
starter_files={
|
||||
"README.md": (
|
||||
"# LLM-as-Judge Evaluator\n\n"
|
||||
"Implement `judge.py`:\n"
|
||||
"- `score(answer: str) -> dict` — 0-4 on the named criterion\n"
|
||||
"- structured JSON output (schema below)\n"
|
||||
"- edge-case classes handled explicitly\n"
|
||||
"- `pytest` must pass\n"
|
||||
),
|
||||
"judge.py": "def score(answer: str) -> dict:\n raise NotImplementedError\n",
|
||||
"test_judge.py": "def test_placeholder():\n assert True\n",
|
||||
},
|
||||
test_command="pytest -q",
|
||||
),
|
||||
"tpl-guardrail-schema": TaskTemplate(
|
||||
id="tpl-guardrail-schema",
|
||||
competency_id="stack-orchestration-c008",
|
||||
title="Schema Guardrail Pipeline",
|
||||
statement_skeleton=(
|
||||
"Implement an output-validation guardrail for a model returning "
|
||||
"{entity} records. Validate against a typed schema with {field_count} "
|
||||
"required fields, coerce or reject {failure_mode} failures, and emit "
|
||||
"a fallback response for invalid payloads. Cover with at least "
|
||||
"{test_size} unit tests including malformed JSON."
|
||||
),
|
||||
slots=[
|
||||
ParameterSlot(
|
||||
name="entity",
|
||||
type="enum",
|
||||
values=["user-profile", "job-posting", "candidate", "invoice"],
|
||||
),
|
||||
ParameterSlot(
|
||||
name="failure_mode",
|
||||
type="enum",
|
||||
values=["strict-reject", "coerce-when-safe"],
|
||||
),
|
||||
ParameterSlot(name="field_count", type="int_range", lo=4, hi=8),
|
||||
ParameterSlot(name="test_size", type="int_range", lo=4, hi=10),
|
||||
],
|
||||
rubric_anchors=RubricAnchors(
|
||||
expected_edit_count_band=(3, 30),
|
||||
expected_min_test_runs=2,
|
||||
expected_error_fix_cycles_band=(0, 5),
|
||||
notes="All slot draws land in the same engineering band.",
|
||||
),
|
||||
starter_files={
|
||||
"README.md": (
|
||||
"# Schema Guardrail\n\nImplement `guardrail.py`:\n"
|
||||
"- `validate(payload: dict) -> dict | Fallback`\n"
|
||||
"- required-field checks, failure policy, fallback emission\n"
|
||||
),
|
||||
"guardrail.py": "def validate(payload: dict):\n raise NotImplementedError\n",
|
||||
"test_guardrail.py": "def test_placeholder():\n assert True\n",
|
||||
},
|
||||
test_command="pytest -q",
|
||||
),
|
||||
"tpl-rag-chunker": TaskTemplate(
|
||||
id="tpl-rag-chunker",
|
||||
competency_id="stack-orchestration-c005",
|
||||
title="RAG Chunking Strategy",
|
||||
statement_skeleton=(
|
||||
"Implement a document chunker for {doc_type} retrieval. Support "
|
||||
"{strategy} chunking with a target size of ~{chunk_size} tokens, "
|
||||
"preserve {invariant} across chunk boundaries, and evaluate overlap "
|
||||
"quality with at least {test_size} fixture documents."
|
||||
),
|
||||
slots=[
|
||||
ParameterSlot(
|
||||
name="doc_type",
|
||||
type="enum",
|
||||
values=["technical-docs", "legal-contracts", "transcripts"],
|
||||
),
|
||||
ParameterSlot(
|
||||
name="strategy",
|
||||
type="enum",
|
||||
values=["fixed-window", "semantic-boundary", "hybrid"],
|
||||
),
|
||||
ParameterSlot(
|
||||
name="invariant",
|
||||
type="enum",
|
||||
values=["code-block-integrity", "section-headers", "sentence-completeness"],
|
||||
),
|
||||
ParameterSlot(name="chunk_size", type="int_range", lo=200, hi=800),
|
||||
ParameterSlot(name="test_size", type="int_range", lo=3, hi=6),
|
||||
],
|
||||
rubric_anchors=RubricAnchors(
|
||||
expected_edit_count_band=(4, 35),
|
||||
expected_min_test_runs=2,
|
||||
expected_error_fix_cycles_band=(0, 6),
|
||||
notes="Strategy draw changes implementation shape, not depth.",
|
||||
),
|
||||
starter_files={
|
||||
"README.md": (
|
||||
"# RAG Chunker\n\nImplement `chunker.py`:\n"
|
||||
"- `chunk(text: str) -> list[str]`\n- invariant preserved\n- tests green\n"
|
||||
),
|
||||
"chunker.py": "def chunk(text: str) -> list[str]:\n raise NotImplementedError\n",
|
||||
"test_chunker.py": "def test_placeholder():\n assert True\n",
|
||||
},
|
||||
test_command="pytest -q",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_KNOWN_COMPETENCY_IDS: set[str] = {
|
||||
# D-021: mirrored from ai_service/corpus/learner_context.py — the Python
|
||||
# source of truth for stack-orchestration competencies used by v0.2 agents.
|
||||
"stack-orchestration-c001",
|
||||
"stack-orchestration-c002",
|
||||
"stack-orchestration-c003",
|
||||
"stack-orchestration-c004",
|
||||
"stack-orchestration-c005",
|
||||
"stack-orchestration-c007",
|
||||
"stack-orchestration-c008",
|
||||
"stack-orchestration-c011",
|
||||
"stack-designer-c001",
|
||||
"stack-designer-c002",
|
||||
"stack-safety-c021",
|
||||
}
|
||||
|
||||
|
||||
def get_template(template_id: str) -> TaskTemplate | None:
|
||||
return TEMPLATES.get(template_id)
|
||||
|
||||
|
||||
def template_for_competency(competency_id: str) -> list[TaskTemplate]:
|
||||
return [t for t in TEMPLATES.values() if t.competency_id == competency_id]
|
||||
|
||||
|
||||
def validate_competency_binding() -> None:
|
||||
"""All templates must bind to known D-021 corpus competency IDs."""
|
||||
for t in TEMPLATES.values():
|
||||
if t.competency_id not in _KNOWN_COMPETENCY_IDS:
|
||||
raise ValueError(
|
||||
f"template {t.id!r} binds unknown competency {t.competency_id!r}"
|
||||
)
|
||||
|
||||
|
||||
def slots_pattern_ok(skeleton: str, slots: list[ParameterSlot]) -> bool:
|
||||
"""Every {placeholder} in the skeleton has a matching slot and vice versa."""
|
||||
placeholders = set(re.findall(r"\{([a-z_][a-z0-9_]*)\}", skeleton))
|
||||
slot_names = {s.name for s in slots}
|
||||
return placeholders == slot_names
|
||||
@@ -0,0 +1 @@
|
||||
"""Variant task generation tests (REQ-3-005)."""
|
||||
@@ -0,0 +1,358 @@
|
||||
"""SQLiteVariantStore tests (REQ-3-005, D-027).
|
||||
|
||||
Each test gets its own tmp-path SQLite file — no shared disk state. Covers:
|
||||
- save / get roundtrip by (learner_id, template_id) AND by task_id
|
||||
(all fields survive, including nested JSON params, starter_files
|
||||
filename->content map, and the tz-aware created_at contract)
|
||||
- insert-only: a second save for the same (learner_id, template_id)
|
||||
raises IntegrityError (first-wins, documented choice); the original
|
||||
row is untouched — NOT upsert, NOT swallowed
|
||||
- unique task_id: a second variant claiming an existing trace key is
|
||||
rejected even under a different (learner, template) pair
|
||||
- list_for_learner / list_by_template scoped + chronological + auditable
|
||||
(seed + params readable back — the proctoring cross-check path)
|
||||
- unknown learner / template / task -> None / empty lists
|
||||
- scoping: rows for other learners/templates never leak
|
||||
- rows are detached: usable after the store is closed
|
||||
- WAL + synchronous=NORMAL pragmas actually applied to the DB file
|
||||
- concurrent writer + reader against the same DB file (a-3 smoke test)
|
||||
"""
|
||||
|
||||
import concurrent.futures
|
||||
import threading
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import sqlalchemy as sa
|
||||
|
||||
from ai_service.variants.store import SQLiteVariantStore, VariantRecord
|
||||
|
||||
_BASE_TS = datetime(2026, 9, 12, 12, 0, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def make_variant(
|
||||
task_id: str = "task-1",
|
||||
learner_id: str = "learner-1",
|
||||
template_id: str = "template-1",
|
||||
seed: str = "seed-a1b2c3",
|
||||
params: dict[str, Any] | None = None,
|
||||
statement: str | None = None,
|
||||
starter_files: dict[str, Any] | None = None,
|
||||
created_at: datetime | None = None,
|
||||
) -> VariantRecord:
|
||||
"""Canonical kwargs builder — tests override only what they assert on."""
|
||||
return VariantRecord(
|
||||
learner_id=learner_id,
|
||||
template_id=template_id,
|
||||
task_id=task_id,
|
||||
seed=seed,
|
||||
params=params
|
||||
if params is not None
|
||||
else {
|
||||
"scenario": "cache invalidation",
|
||||
"constraints": ["no external deps", "streaming"],
|
||||
"data_shape": {"rows": 10_000, "columns": ["ts", "event", "user"]},
|
||||
},
|
||||
statement=statement
|
||||
if statement is not None
|
||||
else (
|
||||
"Implement a cache layer for the event stream service that "
|
||||
"survives restarts without losing buffered rows, learner "
|
||||
f"{learner_id} edition."
|
||||
),
|
||||
starter_files=starter_files
|
||||
if starter_files is not None
|
||||
else {
|
||||
"src/stream_cache.py": "class StreamCache:\n pass\n",
|
||||
"tests/test_stream_cache.py": "def test_roundtrip():\n pass\n",
|
||||
},
|
||||
created_at=created_at if created_at is not None else _BASE_TS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> SQLiteVariantStore:
|
||||
s = SQLiteVariantStore(db_path=tmp_path / "variants.db")
|
||||
yield s
|
||||
s.close()
|
||||
|
||||
|
||||
def test_save_and_get_roundtrip(store: SQLiteVariantStore) -> None:
|
||||
variant = make_variant()
|
||||
store.save(variant)
|
||||
|
||||
# Point lookup by variant identity (learner_id, template_id).
|
||||
fetched = store.get("learner-1", "template-1")
|
||||
assert fetched is not None
|
||||
assert fetched.learner_id == "learner-1"
|
||||
assert fetched.template_id == "template-1"
|
||||
assert fetched.task_id == "task-1"
|
||||
assert fetched.seed == "seed-a1b2c3"
|
||||
assert fetched.params == variant.params
|
||||
assert fetched.statement == variant.statement
|
||||
assert fetched.starter_files == variant.starter_files
|
||||
assert fetched.created_at == _BASE_TS
|
||||
assert fetched.created_at.tzinfo is UTC # tz-normalized on read
|
||||
|
||||
# Same row via the grading/telemetry trace key — the join path a
|
||||
# grade or proctor check uses with only a task_id in hand.
|
||||
by_task = store.get_by_task("task-1")
|
||||
assert by_task is not None
|
||||
assert by_task.learner_id == "learner-1"
|
||||
assert by_task.template_id == "template-1"
|
||||
assert by_task.seed == "seed-a1b2c3"
|
||||
assert by_task.statement == variant.statement
|
||||
assert by_task.starter_files == variant.starter_files
|
||||
assert by_task.params == variant.params
|
||||
assert by_task.created_at.tzinfo is UTC
|
||||
|
||||
|
||||
def test_get_unknown_returns_none(store: SQLiteVariantStore) -> None:
|
||||
store.save(make_variant())
|
||||
|
||||
assert store.get("learner-1", "template-missing") is None
|
||||
assert store.get("learner-missing", "template-1") is None
|
||||
assert store.get("nobody", "nothing") is None
|
||||
assert store.get_by_task("task-missing") is None
|
||||
|
||||
|
||||
def test_duplicate_learner_template_pair_raises_integrity_error(
|
||||
store: SQLiteVariantStore,
|
||||
) -> None:
|
||||
"""THE contract of this store (insert-only, first wins).
|
||||
|
||||
The first generated variant is authoritative (D-029 reproducibility:
|
||||
the seed re-derives the same variant); a duplicate save is a
|
||||
programming error or lost race, not a normal flow — the generator's
|
||||
cache path serves `get` instead. So the IntegrityError surfaces to
|
||||
the caller and the stored row is left untouched.
|
||||
"""
|
||||
first = make_variant(statement="first authoritative statement")
|
||||
store.save(first)
|
||||
|
||||
second = make_variant(
|
||||
task_id="task-2", # distinct task key; the PAIR collides
|
||||
seed="seed-999",
|
||||
statement="an impostor statement",
|
||||
created_at=_BASE_TS + timedelta(hours=1),
|
||||
)
|
||||
with pytest.raises(sa.exc.IntegrityError):
|
||||
store.save(second)
|
||||
|
||||
# First save survived intact — nothing was overwritten.
|
||||
fetched = store.get("learner-1", "template-1")
|
||||
assert fetched is not None
|
||||
assert fetched.task_id == "task-1"
|
||||
assert fetched.statement == "first authoritative statement"
|
||||
assert fetched.seed == "seed-a1b2c3"
|
||||
assert fetched.created_at == _BASE_TS
|
||||
|
||||
# The rejected save left no row behind under its task_id either.
|
||||
assert store.get_by_task("task-2") is None
|
||||
|
||||
|
||||
def test_duplicate_task_id_raises_integrity_error(store: SQLiteVariantStore) -> None:
|
||||
# task_id is the grading/telemetry trace key — globally unique: a
|
||||
# second variant may never claim an existing trace key, even under a
|
||||
# different (learner_id, template_id) pair.
|
||||
store.save(make_variant(learner_id="learner-1", template_id="template-1"))
|
||||
|
||||
with pytest.raises(sa.exc.IntegrityError):
|
||||
store.save(
|
||||
make_variant(
|
||||
learner_id="learner-2",
|
||||
template_id="template-2",
|
||||
task_id="task-1", # collides with learner-1's trace key
|
||||
)
|
||||
)
|
||||
|
||||
# Rejected row not partially stored under either identity.
|
||||
assert store.get("learner-2", "template-2") is None
|
||||
assert len(store.list_by_template("template-2")) == 0
|
||||
|
||||
|
||||
def test_list_for_learner_roundtrip_and_audit(store: SQLiteVariantStore) -> None:
|
||||
# Created out of insertion order; list must come back chronological.
|
||||
store.save(make_variant(template_id="template-c", task_id="task-c",
|
||||
created_at=_BASE_TS + timedelta(hours=2)))
|
||||
store.save(make_variant(template_id="template-a", task_id="task-a",
|
||||
created_at=_BASE_TS))
|
||||
store.save(make_variant(template_id="template-b", task_id="task-b",
|
||||
created_at=_BASE_TS + timedelta(hours=1)))
|
||||
|
||||
variants = store.list_for_learner("learner-1")
|
||||
assert [v.template_id for v in variants] == [
|
||||
"template-a",
|
||||
"template-b",
|
||||
"template-c",
|
||||
]
|
||||
assert all(v.learner_id == "learner-1" for v in variants)
|
||||
hours = (timedelta(hours=0), timedelta(hours=1), timedelta(hours=2))
|
||||
assert all(
|
||||
v.created_at == _BASE_TS + offset
|
||||
for v, offset in zip(variants, hours, strict=True)
|
||||
)
|
||||
assert all(v.created_at.tzinfo is UTC for v in variants)
|
||||
|
||||
# Auditable: every stored variant reads back its seed and typed params
|
||||
# (the proctoring cross-check path reads exactly this).
|
||||
for v in variants:
|
||||
assert v.seed.startswith("seed-")
|
||||
assert v.params["scenario"] == "cache invalidation"
|
||||
assert "streaming" in v.params["constraints"]
|
||||
assert v.params["data_shape"]["rows"] == 10_000
|
||||
assert v.starter_files["tests/test_stream_cache.py"].count("\n") >= 1
|
||||
|
||||
|
||||
def test_list_by_template_roundtrip_and_audit(store: SQLiteVariantStore) -> None:
|
||||
# Three learners on the same template: distinct, auditable variants.
|
||||
store.save(make_variant(learner_id="learner-b", task_id="task-b",
|
||||
seed="seed-222",
|
||||
created_at=_BASE_TS + timedelta(hours=1)))
|
||||
store.save(make_variant(learner_id="learner-a", task_id="task-a",
|
||||
seed="seed-111", created_at=_BASE_TS))
|
||||
store.save(make_variant(learner_id="learner-c", task_id="task-c",
|
||||
seed="seed-333",
|
||||
created_at=_BASE_TS + timedelta(hours=2)))
|
||||
|
||||
variants = store.list_by_template("template-1")
|
||||
assert [v.learner_id for v in variants] == ["learner-a", "learner-b", "learner-c"]
|
||||
assert all(v.template_id == "template-1" for v in variants)
|
||||
|
||||
# Every learner's variant carries its own seed + params (auditable,
|
||||
# REQ-3-005: variant parameters persisted and auditable).
|
||||
seeds = {v.learner_id: v.seed for v in variants}
|
||||
assert seeds == {
|
||||
"learner-a": "seed-111",
|
||||
"learner-b": "seed-222",
|
||||
"learner-c": "seed-333",
|
||||
}
|
||||
assert all(v.params["scenario"] == "cache invalidation" for v in variants)
|
||||
assert all(v.created_at.tzinfo is UTC for v in variants)
|
||||
|
||||
|
||||
def test_list_unknown_returns_empty_lists(store: SQLiteVariantStore) -> None:
|
||||
store.save(make_variant())
|
||||
|
||||
assert store.list_for_learner("nobody") == []
|
||||
assert store.list_by_template("no-template") == []
|
||||
|
||||
|
||||
def test_lists_are_scoped(store: SQLiteVariantStore) -> None:
|
||||
store.save(make_variant(learner_id="learner-1", template_id="template-1",
|
||||
task_id="task-1"))
|
||||
store.save(make_variant(learner_id="learner-2", template_id="template-1",
|
||||
task_id="task-2"))
|
||||
store.save(make_variant(learner_id="learner-1", template_id="template-2",
|
||||
task_id="task-3"))
|
||||
|
||||
# learner lists see only that learner's rows.
|
||||
assert [v.template_id for v in store.list_for_learner("learner-1")] == [
|
||||
"template-1",
|
||||
"template-2",
|
||||
]
|
||||
assert [v.template_id for v in store.list_for_learner("learner-2")] == ["template-1"]
|
||||
|
||||
# template lists see one row per learner, none from other templates.
|
||||
template_rows = store.list_by_template("template-1")
|
||||
assert sorted(v.learner_id for v in template_rows) == ["learner-1", "learner-2"]
|
||||
assert all(v.template_id == "template-1" for v in template_rows)
|
||||
|
||||
# get stays a pair-scoped point lookup: same template, other learner.
|
||||
assert store.get("learner-1", "template-2") is not None
|
||||
assert store.get("learner-2", "template-2") is None
|
||||
|
||||
|
||||
def test_empty_params_and_starter_files_roundtrip(store: SQLiteVariantStore) -> None:
|
||||
# Legal shapes: a slotless template carries no params; a variant may
|
||||
# ship without a workspace scaffold.
|
||||
store.save(make_variant(params={}, starter_files={}))
|
||||
|
||||
fetched = store.get("learner-1", "template-1")
|
||||
assert fetched is not None
|
||||
assert fetched.params == {}
|
||||
assert fetched.starter_files == {}
|
||||
|
||||
|
||||
def test_rows_are_detached_after_save(store: SQLiteVariantStore, tmp_path: Path) -> None:
|
||||
# The API layer hands VariantRecords across layers; rows must survive
|
||||
# the store that produced them being closed (no open-session ORM magic).
|
||||
store.save(make_variant())
|
||||
fetched = store.get("learner-1", "template-1")
|
||||
by_task = store.get_by_task("task-1")
|
||||
store.close()
|
||||
|
||||
assert fetched is not None
|
||||
assert fetched.statement == fetched.statement # usable post-close
|
||||
assert fetched.starter_files["src/stream_cache.py"] == "class StreamCache:\n pass\n"
|
||||
assert by_task is not None
|
||||
assert by_task.seed == "seed-a1b2c3"
|
||||
|
||||
# A fresh store on the same file sees the same row (durability).
|
||||
reopened = SQLiteVariantStore(db_path=tmp_path / "variants.db")
|
||||
try:
|
||||
again = reopened.get("learner-1", "template-1")
|
||||
assert again is not None
|
||||
assert again.starter_files["src/stream_cache.py"].startswith("class StreamCache")
|
||||
assert again.created_at.tzinfo is UTC
|
||||
finally:
|
||||
reopened.close()
|
||||
|
||||
|
||||
def test_pragmas_are_applied(store: SQLiteVariantStore) -> None:
|
||||
# Pragmas are per-connection; query through the store's engine so the
|
||||
# connect hook (not a default sqlite3 connection) is what we inspect.
|
||||
with store._engine.connect() as conn:
|
||||
(journal_mode,) = conn.execute(sa.text("PRAGMA journal_mode")).one()
|
||||
(synchronous,) = conn.execute(sa.text("PRAGMA synchronous")).one()
|
||||
assert journal_mode == "wal"
|
||||
# synchronous=NORMAL is 1 in SQLite's pragma numbering.
|
||||
assert synchronous == 1
|
||||
|
||||
|
||||
def test_concurrent_writer_and_reader_no_database_is_locked(tmp_path: Path) -> None:
|
||||
"""One thread saves while another reads in a tight loop (a-3).
|
||||
|
||||
Without WAL + busy_timeout this pattern reliably produces
|
||||
`OperationalError: database is locked` on SQLite. The assertion is
|
||||
that every reader call completes and every distinct-row write lands.
|
||||
"""
|
||||
db_path = tmp_path / "variants.db"
|
||||
n_variants = 60 # distinct (learner, template) pairs, one save each
|
||||
stop_writing = threading.Event()
|
||||
|
||||
writer = SQLiteVariantStore(db_path=db_path)
|
||||
reader = SQLiteVariantStore(db_path=db_path)
|
||||
try:
|
||||
|
||||
def write_variants() -> None:
|
||||
for seq in range(n_variants):
|
||||
writer.save(
|
||||
make_variant(
|
||||
learner_id=f"learner-{seq}",
|
||||
template_id="template-1",
|
||||
task_id=f"task-{seq}",
|
||||
seed=f"seed-{seq:03d}",
|
||||
created_at=_BASE_TS + timedelta(seconds=seq),
|
||||
)
|
||||
)
|
||||
stop_writing.set()
|
||||
|
||||
def read_variants() -> None:
|
||||
while not stop_writing.is_set():
|
||||
reader.list_by_template("template-1")
|
||||
# Final read after the writer is done.
|
||||
assert len(reader.list_by_template("template-1")) == n_variants
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(write_variants), pool.submit(read_variants)]
|
||||
for future in futures:
|
||||
future.result(timeout=30)
|
||||
|
||||
# Every save was a distinct row: nothing lost, none doubled.
|
||||
assert len(writer.list_by_template("template-1")) == n_variants
|
||||
finally:
|
||||
reader.close()
|
||||
writer.close()
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Task template tests (Task 4-1-01, REQ-3-005)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from ai_service.variants.templates import (
|
||||
TEMPLATES,
|
||||
ParameterSlot,
|
||||
TaskTemplate,
|
||||
get_template,
|
||||
slots_pattern_ok,
|
||||
template_for_competency,
|
||||
validate_competency_binding,
|
||||
)
|
||||
|
||||
|
||||
def test_all_templates_bind_to_real_competency_ids() -> None:
|
||||
validate_competency_binding() # raises on any unknown binding
|
||||
assert len(TEMPLATES) >= 3
|
||||
|
||||
|
||||
def test_slot_validation_rejects_bad_values() -> None:
|
||||
slot = ParameterSlot(name="domain", type="enum", values=["a", "b"])
|
||||
assert slot.validate_value("a")
|
||||
assert not slot.validate_value("c")
|
||||
assert not slot.validate_value(3)
|
||||
|
||||
rng = random.Random(42)
|
||||
assert slot.sample(rng) in {"a", "b"}
|
||||
|
||||
|
||||
def test_int_range_slot_bounds() -> None:
|
||||
slot = ParameterSlot(name="n", type="int_range", lo=2, hi=4)
|
||||
rng = random.Random(0)
|
||||
for _ in range(20):
|
||||
assert 2 <= slot.sample(rng) <= 4
|
||||
assert slot.validate_value(3)
|
||||
assert not slot.validate_value(5)
|
||||
assert not slot.validate_value("3")
|
||||
|
||||
|
||||
def test_seeded_sampling_is_reproducible() -> None:
|
||||
tpl = TEMPLATES["tpl-llm-judge"]
|
||||
first = tpl.sample_params(seed=1234)
|
||||
second = tpl.sample_params(seed=1234)
|
||||
other = tpl.sample_params(seed=1235)
|
||||
assert first == second # D-029: same seed -> identical params
|
||||
assert first != other # different seed -> (near-certainly) different draw
|
||||
|
||||
|
||||
def test_skeleton_placeholders_match_slots() -> None:
|
||||
for tpl in TEMPLATES.values():
|
||||
assert slots_pattern_ok(tpl.statement_skeleton, tpl.slots), tpl.id
|
||||
|
||||
|
||||
def test_render_validates_params_and_fills() -> None:
|
||||
tpl = TEMPLATES["tpl-llm-judge"]
|
||||
params = tpl.sample_params(seed=7)
|
||||
rendered = tpl.render(params)
|
||||
for value in params.values():
|
||||
assert str(value) in rendered
|
||||
with pytest.raises(ValueError, match="invalid value"):
|
||||
tpl.render({**params, "edge_cases": 99}) # out of band
|
||||
|
||||
|
||||
def test_rubric_anchors_present_per_template() -> None:
|
||||
for tpl in TEMPLATES.values():
|
||||
anchors = tpl.rubric_anchors
|
||||
assert anchors.expected_edit_count_band[0] <= anchors.expected_edit_count_band[1]
|
||||
assert anchors.expected_min_test_runs >= 1
|
||||
cycles = anchors.expected_error_fix_cycles_band
|
||||
assert cycles[0] <= cycles[1]
|
||||
|
||||
|
||||
def test_starter_files_defined_per_template() -> None:
|
||||
for tpl in TEMPLATES.values():
|
||||
assert tpl.starter_files, f"{tpl.id} missing starter scaffolds"
|
||||
assert "README.md" in tpl.starter_files
|
||||
assert tpl.test_command
|
||||
|
||||
|
||||
def test_competency_lookup() -> None:
|
||||
tpls = template_for_competency("stack-orchestration-c005")
|
||||
assert len(tpls) == 1
|
||||
assert get_template("nope-xyz") is None
|
||||
|
||||
|
||||
def test_bad_skeleton_rejected() -> None:
|
||||
with pytest.raises(ValueError, match="slot"):
|
||||
TaskTemplate(
|
||||
id="tpl-bad",
|
||||
competency_id="stack-orchestration-c001",
|
||||
title="Bad",
|
||||
statement_skeleton="no placeholders at all",
|
||||
slots=[ParameterSlot(name="x", type="enum", values=["a"])],
|
||||
rubric_anchors=TEMPLATES["tpl-llm-judge"].rubric_anchors,
|
||||
starter_files={},
|
||||
test_command="pytest -q",
|
||||
)
|
||||
Reference in New Issue
Block a user