feat(P04): variants API + TS types (Wave 3)
Task 4-3-01: POST /v1/variants (template_id or competency_id resolution; cache-first
D-029; 404 unknown template/competency; 422 neither), GET /v1/variants/{task_id},
GET /v1/variants?learner_id=. SQLiteVariantStore + VariantGenerator wired into lifespan.
Task 4-3-02: packages/types/variants.ts (TaskVariant, VariantParams) + grading.ts
(RubricScore + GradeRecord, criteria-typed) — field-for-field Python parity,
scores typed as the documented rubric-or-gate union.
11 endpoint tests green (distinct learners -> distinct statements at API level; cache
hit -> zero LLM calls); suite 324 green; typecheck 7/7; ruff clean.
---ci---
phase: 4
milestone: v0.3
status: execute
requirements: {covered: [REQ-3-005], partial: []}
---/ci---
This commit is contained in:
@@ -10,6 +10,7 @@ from .mentor import router as mentor_router
|
||||
from .proctor import router as proctor_router
|
||||
from .sandboxes import router as sandboxes_router
|
||||
from .telemetry import router as telemetry_router
|
||||
from .variants import router as variants_router
|
||||
|
||||
__all__ = [
|
||||
"assessment_router",
|
||||
@@ -19,4 +20,5 @@ __all__ = [
|
||||
"proctor_router",
|
||||
"sandboxes_router",
|
||||
"telemetry_router",
|
||||
"variants_router",
|
||||
]
|
||||
|
||||
@@ -12,6 +12,8 @@ from ..sandbox.manager import SandboxManager
|
||||
from ..sandbox.workdir import SandboxDir
|
||||
from ..telemetry.ingest import TraceIntegrityMap
|
||||
from ..telemetry.store import TraceStore
|
||||
from ..variants.generator import VariantGenerator
|
||||
from ..variants.store import VariantStore
|
||||
|
||||
|
||||
def get_settings(request: Request) -> Settings:
|
||||
@@ -53,3 +55,11 @@ def get_grade_store(request: Request) -> GradeStore:
|
||||
|
||||
def get_grading_engine(request: Request) -> GradingEngine:
|
||||
return request.app.state.grading_engine
|
||||
|
||||
|
||||
def get_variant_generator(request: Request) -> VariantGenerator:
|
||||
return request.app.state.variant_generator
|
||||
|
||||
|
||||
def get_variant_store(request: Request) -> VariantStore:
|
||||
return request.app.state.variant_store
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
"""/v1/variants — seeded per-learner task variant endpoints (REQ-3-005, D-029).
|
||||
|
||||
Three faces over the variant engine (the generator + store stay FastAPI-free;
|
||||
this module owns all HTTP wiring — the D-027/D-032 house pattern):
|
||||
|
||||
POST /v1/variants {learner_id, template_id | competency_id}
|
||||
→ 200 the learner's variant — GENERATED on the first request,
|
||||
CACHED (store read, zero LLM calls) on every repeat: D-029
|
||||
reproducibility means one (learner_id, template_id) is ONE
|
||||
variant forever, so a regenerate is always a 200 of the SAME
|
||||
variant, never a second render.
|
||||
→ 404 unknown template_id, or competency_id with no bound template.
|
||||
→ 422 neither template_id nor competency_id given.
|
||||
GET /v1/variants/{task_id}
|
||||
→ 200 the stored variant owning the task key (the grading and
|
||||
telemetry join path); 404 when no variant was ever generated
|
||||
for the task.
|
||||
GET /v1/variants?learner_id=...
|
||||
→ 200 the learner's variants, chronological; [] when none.
|
||||
|
||||
Template resolution: an explicit `template_id` wins; without it the FIRST
|
||||
template bound to `competency_id` is used (`template_for_competency`,
|
||||
D-021 corpus alignment). The response carries `competency_id` resolved
|
||||
from the template library at read time — an enrichment, not a persisted
|
||||
column (the seed re-derives the whole variant, D-029) — so the learner
|
||||
surface can bind a variant to its competency without a library round-trip.
|
||||
|
||||
Distinctness (REQ-3-005): different learners on the same template draw
|
||||
different seeded params and receive distinct statements and task_ids;
|
||||
tests/api/test_variants.py asserts this end-to-end through the API.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Self
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from ..variants.generator import VariantGenerator
|
||||
from ..variants.store import VariantRecord, VariantStore
|
||||
from ..variants.templates import TaskTemplate, get_template, template_for_competency
|
||||
from .deps import get_variant_generator, get_variant_store
|
||||
|
||||
router = APIRouter(prefix="/v1/variants", tags=["variants"])
|
||||
|
||||
|
||||
# -- contracts ------------------------------------------------------------------
|
||||
|
||||
|
||||
class VariantGenerateRequest(BaseModel):
|
||||
"""One variant identity: an explicit `template_id`, or the first
|
||||
template bound to a `competency_id` (D-021). `template_id` wins when
|
||||
both are given (explicit identity beats derived); at least one is
|
||||
required — 422 otherwise.
|
||||
"""
|
||||
|
||||
learner_id: str = Field(min_length=1)
|
||||
template_id: str | None = None
|
||||
competency_id: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_template_or_competency(self) -> Self:
|
||||
if self.template_id is None and self.competency_id is None:
|
||||
raise ValueError("template_id or competency_id is required")
|
||||
return self
|
||||
|
||||
|
||||
class VariantResponse(BaseModel):
|
||||
"""VariantRecord over HTTP, plus the `competency_id` enrichment.
|
||||
|
||||
Every field except `competency_id` mirrors `VariantRecord` exactly
|
||||
(snake_case; `created_at` is an ISO 8601 UTC datetime) — the wire shape
|
||||
typed as `TaskVariant` in packages/types/variants.ts.
|
||||
"""
|
||||
|
||||
learner_id: str
|
||||
task_id: str
|
||||
template_id: str
|
||||
competency_id: str
|
||||
seed: str
|
||||
params: dict[str, str | int]
|
||||
statement: str
|
||||
starter_files: dict[str, str]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class VariantListResponse(BaseModel):
|
||||
variants: list[VariantResponse]
|
||||
|
||||
|
||||
# -- resolution + rendering -----------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_template(body: VariantGenerateRequest) -> TaskTemplate:
|
||||
"""Template for the request: the explicit id, else the first template
|
||||
bound to the competency; 404 when neither resolves."""
|
||||
if body.template_id is not None:
|
||||
template = get_template(body.template_id)
|
||||
if template is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"no task template with id {body.template_id!r}",
|
||||
)
|
||||
return template
|
||||
# The request validator guarantees the disjunction, so reaching here
|
||||
# means a competency_id was given (never None).
|
||||
assert body.competency_id is not None
|
||||
templates = template_for_competency(body.competency_id)
|
||||
if not templates:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"no task template for competency {body.competency_id!r}",
|
||||
)
|
||||
return templates[0]
|
||||
|
||||
|
||||
def _competency_for(template_id: str) -> str:
|
||||
"""competency_id enrichment for stored records (read paths)."""
|
||||
template = get_template(template_id)
|
||||
if template is None:
|
||||
# Integrity guard: a stored variant referencing a template that is
|
||||
# no longer in the library cannot be enriched; fail loudly rather
|
||||
# than fabricate a competency binding.
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"stored variant references unknown template {template_id!r}",
|
||||
)
|
||||
return template.competency_id
|
||||
|
||||
|
||||
def _to_response(record: VariantRecord, competency_id: str) -> VariantResponse:
|
||||
return VariantResponse(
|
||||
learner_id=record.learner_id,
|
||||
task_id=record.task_id,
|
||||
template_id=record.template_id,
|
||||
competency_id=competency_id,
|
||||
seed=record.seed,
|
||||
params=dict(record.params),
|
||||
statement=record.statement,
|
||||
starter_files=dict(record.starter_files),
|
||||
created_at=record.created_at,
|
||||
)
|
||||
|
||||
|
||||
# -- endpoints ------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("", response_model=VariantResponse)
|
||||
async def generate_variant(
|
||||
body: VariantGenerateRequest,
|
||||
generator: VariantGenerator = Depends(get_variant_generator),
|
||||
) -> VariantResponse:
|
||||
"""The learner's variant for the resolved template — generated on the
|
||||
first request, cached (no LLM call) on every repeat: D-029 makes a
|
||||
regenerate a 200 of the SAME stored variant.
|
||||
"""
|
||||
template = _resolve_template(body)
|
||||
record = await generator.generate(body.learner_id, template.id)
|
||||
return _to_response(record, competency_id=template.competency_id)
|
||||
|
||||
|
||||
@router.get("", response_model=VariantListResponse)
|
||||
async def list_variants(
|
||||
learner_id: str,
|
||||
store: VariantStore = Depends(get_variant_store),
|
||||
) -> VariantListResponse:
|
||||
"""All stored variants for the learner, chronological; [] when none."""
|
||||
variants = [
|
||||
_to_response(record, competency_id=_competency_for(record.template_id))
|
||||
for record in store.list_for_learner(learner_id)
|
||||
]
|
||||
return VariantListResponse(variants=variants)
|
||||
|
||||
|
||||
@router.get("/{task_id}", response_model=VariantResponse)
|
||||
async def get_variant(
|
||||
task_id: str,
|
||||
store: VariantStore = Depends(get_variant_store),
|
||||
) -> VariantResponse:
|
||||
"""The stored variant owning the task key — the grading and telemetry
|
||||
join path; 404 when no variant was ever generated for the task."""
|
||||
record = store.get_by_task(task_id)
|
||||
if record is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"no stored variant for task {task_id!r}"
|
||||
)
|
||||
return _to_response(record, competency_id=_competency_for(record.template_id))
|
||||
@@ -19,6 +19,7 @@ from .api import (
|
||||
proctor_router,
|
||||
sandboxes_router,
|
||||
telemetry_router,
|
||||
variants_router,
|
||||
)
|
||||
from .config import Settings
|
||||
from .grading.engine import GradingEngine
|
||||
@@ -27,6 +28,8 @@ from .llm import create_provider
|
||||
from .sandbox import SandboxManager, UnshareBackend
|
||||
from .telemetry.ingest import TraceIntegrityMap
|
||||
from .telemetry.store import SQLiteTraceStore
|
||||
from .variants.generator import VariantGenerator
|
||||
from .variants.store import SQLiteVariantStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -94,6 +97,27 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
model=settings.model,
|
||||
)
|
||||
|
||||
# Variant generation (REQ-3-005, Wave 3): VariantStore from the same
|
||||
# SQLite file as traces/grades (D-027), one VariantGenerator singleton
|
||||
# wired through app.state — the generator receives store + provider
|
||||
# via constructor DI and knows nothing of FastAPI (api/ composes it,
|
||||
# same pattern as GradingEngine). Tests may pre-set
|
||||
# app.state.variant_store / app.state.variant_generator (the same
|
||||
# state-injection override); the lifespan adopts a pre-set store but
|
||||
# NEVER rebuilds a pre-set generator (its provider binding is part
|
||||
# of the test fixture).
|
||||
variant_store = getattr(app.state, "variant_store", None)
|
||||
if variant_store is None:
|
||||
settings.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
variant_store = SQLiteVariantStore(db_path=settings.db_path)
|
||||
app.state.variant_store = variant_store
|
||||
if getattr(app.state, "variant_generator", None) is None:
|
||||
app.state.variant_generator = VariantGenerator(
|
||||
variant_store,
|
||||
app.state.provider,
|
||||
model=settings.model,
|
||||
)
|
||||
|
||||
async def _reaper_loop() -> None:
|
||||
# Wall-clock timeout + G-2 workdir-size sweep, one pass per tick.
|
||||
while True:
|
||||
@@ -115,6 +139,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
await manager.destroy_all()
|
||||
trace_store.close()
|
||||
grade_store.close()
|
||||
variant_store.close()
|
||||
await app.state.http_client.aclose()
|
||||
|
||||
app = FastAPI(title="Nextcraft AI Service", version="0.3.0", lifespan=lifespan)
|
||||
@@ -143,6 +168,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.include_router(proctor_router)
|
||||
app.include_router(sandboxes_router)
|
||||
app.include_router(telemetry_router)
|
||||
app.include_router(variants_router)
|
||||
return app
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Variant API tests — generation, cache, distinctness over HTTP (Task 4-3-01).
|
||||
|
||||
Contract under test (api/variants.py, REQ-3-005):
|
||||
|
||||
POST /v1/variants {learner_id, template_id}
|
||||
first request → 200 generated variant (task_id, seed, params,
|
||||
statement, starter_files, competency_id)
|
||||
repeat request → 200 the SAME cached variant with ZERO LLM
|
||||
calls (D-029 reproducibility through the
|
||||
whole HTTP stack)
|
||||
unknown template → 404
|
||||
POST /v1/variants {learner_id, competency_id}
|
||||
bound competency → 200 the first template for that competency
|
||||
unknown competency → 404
|
||||
neither id given → 422
|
||||
GET /v1/variants/{task_id}
|
||||
generated earlier → 200 the stored variant (generate→get roundtrip)
|
||||
unknown task → 404
|
||||
GET /v1/variants?learner_id=...
|
||||
→ 200 that learner's variants only (scoping); [] for a learner
|
||||
with none.
|
||||
|
||||
Distinctness (REQ-3-005, API level): two learners POSTing the same
|
||||
template receive distinct statements, seeds and task_ids, and GET by
|
||||
task_id hands each back their own variant.
|
||||
|
||||
Wiring: per-test tmp-path SQLiteVariantStore + a pre-set VariantGenerator
|
||||
(state-injection override — the lifespan adopts variant_store /
|
||||
variant_generator from app.state instead of constructing them; same
|
||||
pattern as test_grading.py / test_telemetry_ingest.py). The generator
|
||||
binds a ScriptedRenderProvider so each test controls the render LLM
|
||||
exactly, and counts calls so the cache test can assert the LLM was never
|
||||
reached on the second POST.
|
||||
|
||||
Zero network: providers are MockProvider family members only (conftest
|
||||
rule, enforced in _make_client).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from ai_service.config import Settings
|
||||
from ai_service.llm.mock import MockProvider
|
||||
from ai_service.main import create_app
|
||||
from ai_service.variants.generator import VariantGenerator
|
||||
from ai_service.variants.store import SQLiteVariantStore
|
||||
from ai_service.variants.templates import get_template
|
||||
|
||||
LEARNER_A = "variant-learner-a"
|
||||
LEARNER_B = "variant-learner-b"
|
||||
TEMPLATE = "tpl-llm-judge"
|
||||
COMPETENCY = "stack-orchestration-c007" # tpl-llm-judge's D-021 binding
|
||||
|
||||
|
||||
class ScriptedRenderProvider(MockProvider):
|
||||
"""Deterministic render whose statement embeds the params (distinct per
|
||||
draw); counts calls so the cache test asserts zero LLM calls on the
|
||||
second POST. Mirrors tests/variants/test_generator.py."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
async def chat(self, messages, *, model, temperature=0.7, response_format=None): # noqa: ANN001
|
||||
self.calls += 1
|
||||
import json
|
||||
|
||||
user = next(m.content for m in reversed(messages) if m.role == "user")
|
||||
# Distinct per distinct params: hash the seeded slot lines.
|
||||
fingerprint = abs(hash(user)) % 10_000
|
||||
return json.dumps({"statement": f"Scripted variant #{fingerprint} — build it."})
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def store(tmp_path: Path) -> Iterator[SQLiteVariantStore]:
|
||||
s = SQLiteVariantStore(db_path=tmp_path / "variants.db")
|
||||
yield s
|
||||
s.close()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def provider() -> ScriptedRenderProvider:
|
||||
return ScriptedRenderProvider()
|
||||
|
||||
|
||||
def _make_client(
|
||||
tmp_path: Path, store: SQLiteVariantStore, provider: MockProvider
|
||||
) -> TestClient:
|
||||
"""App + TestClient with a pre-set store + generator.
|
||||
|
||||
The lifespan adopts both (state-injection override); the generator
|
||||
binds OUR provider, so the fixture — not Settings — scripts the LLM.
|
||||
Cloud-free guard: the provider must be a MockProvider family member
|
||||
(conftest rule, enforced here because this module builds its own
|
||||
client rather than consuming the conftest one).
|
||||
"""
|
||||
assert isinstance(provider, MockProvider)
|
||||
settings = Settings(
|
||||
provider="mock",
|
||||
db_path=tmp_path / "variant-test.db",
|
||||
sandbox_dir=tmp_path / "sandboxes",
|
||||
)
|
||||
app = create_app(settings)
|
||||
app.state.variant_store = store
|
||||
if getattr(app.state, "variant_generator", None) is None:
|
||||
app.state.variant_generator = VariantGenerator(
|
||||
store, provider, model="gemma4:31b"
|
||||
)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _post(
|
||||
client: TestClient,
|
||||
learner_id: str,
|
||||
template_id: str | None = TEMPLATE,
|
||||
competency_id: str | None = None,
|
||||
):
|
||||
return client.post(
|
||||
"/v1/variants",
|
||||
json={
|
||||
"learner_id": learner_id,
|
||||
**({"template_id": template_id} if template_id is not None else {}),
|
||||
**({"competency_id": competency_id} if competency_id is not None else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# -- POST: generation + response shape --------------------------------------------
|
||||
|
||||
|
||||
class TestGenerate:
|
||||
def test_post_returns_full_variant_payload(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
response = _post(client, LEARNER_A)
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["learner_id"] == LEARNER_A
|
||||
assert body["template_id"] == TEMPLATE
|
||||
assert body["competency_id"] == COMPETENCY # enrichment from get_template
|
||||
assert body["task_id"].startswith("task-") and len(body["task_id"]) == len("task-") + 16
|
||||
assert body["seed"] # non-empty D-029 seed
|
||||
assert body["statement"] # rendered, non-empty
|
||||
assert body["starter_files"] == get_template(TEMPLATE).starter_files
|
||||
assert body["params"] # seeded slot values
|
||||
assert body["created_at"] # ISO timestamp travels
|
||||
assert provider.calls == 1 # first POST renders exactly once
|
||||
|
||||
def test_post_then_get_roundtrip(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
posted = _post(client, LEARNER_A)
|
||||
assert posted.status_code == 200
|
||||
|
||||
fetched = client.get(f"/v1/variants/{posted.json()['task_id']}")
|
||||
|
||||
assert fetched.status_code == 200
|
||||
assert fetched.json() == posted.json() # identical stored variant
|
||||
|
||||
def test_post_regenerate_is_cache_hit_no_llm_call(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
first = _post(client, LEARNER_A)
|
||||
assert first.status_code == 200
|
||||
assert provider.calls == 1
|
||||
|
||||
second = _post(client, LEARNER_A) # same (learner, template) → cache
|
||||
assert second.status_code == 200
|
||||
assert second.json() == first.json() # D-029: the SAME variant
|
||||
assert provider.calls == 1, "cached regenerate must not render again"
|
||||
|
||||
|
||||
# -- POST: distinctness (REQ-3-005, API level) ------------------------------------
|
||||
|
||||
|
||||
class TestDistinctLearners:
|
||||
def test_two_learners_distinct_statements_and_task_ids(
|
||||
self, tmp_path, store, provider
|
||||
):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
a = _post(client, LEARNER_A)
|
||||
b = _post(client, LEARNER_B)
|
||||
assert a.status_code == 200 and b.status_code == 200
|
||||
|
||||
a_body, b_body = a.json(), b.json()
|
||||
|
||||
# ...and each learner GETs back exactly their own variant
|
||||
a_fetched = client.get(f"/v1/variants/{a_body['task_id']}")
|
||||
b_fetched = client.get(f"/v1/variants/{b_body['task_id']}")
|
||||
|
||||
assert a_body["statement"] != b_body["statement"]
|
||||
assert a_body["seed"] != b_body["seed"]
|
||||
assert a_body["task_id"] != b_body["task_id"]
|
||||
assert a_body["competency_id"] == b_body["competency_id"] # same bar (a-5)
|
||||
assert a_fetched.status_code == 200 and a_fetched.json() == a_body
|
||||
assert b_fetched.status_code == 200 and b_fetched.json() == b_body
|
||||
|
||||
|
||||
# -- POST: template / competency resolution ---------------------------------------
|
||||
|
||||
|
||||
class TestTemplateResolution:
|
||||
def test_unknown_template_404(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
response = _post(client, LEARNER_A, template_id="tpl-does-not-exist")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert "no task template" in response.json()["detail"]
|
||||
|
||||
def test_competency_lookup_uses_first_bound_template(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
response = _post(
|
||||
client, LEARNER_A, template_id=None, competency_id=COMPETENCY
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["template_id"] == TEMPLATE # the competency's first template
|
||||
assert body["competency_id"] == COMPETENCY
|
||||
|
||||
def test_unknown_competency_404(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
response = _post(
|
||||
client, LEARNER_A, template_id=None, competency_id="stack-none-c999"
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert "no task template for competency" in response.json()["detail"]
|
||||
|
||||
def test_neither_id_given_422(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
response = _post(client, LEARNER_A, template_id=None, competency_id=None)
|
||||
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
# -- GET: stored reads ------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStoredReads:
|
||||
def test_get_unknown_task_404(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
response = client.get("/v1/variants/task-0000000000000000")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert "no stored variant" in response.json()["detail"]
|
||||
|
||||
def test_list_scoped_to_one_learner(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
assert _post(client, LEARNER_A).status_code == 200
|
||||
assert _post(
|
||||
client, LEARNER_A, template_id="tpl-guardrail-schema"
|
||||
).status_code == 200
|
||||
assert _post(client, LEARNER_B).status_code == 200
|
||||
|
||||
listed = client.get("/v1/variants", params={"learner_id": LEARNER_A})
|
||||
empty = client.get("/v1/variants", params={"learner_id": "nobody"})
|
||||
|
||||
assert listed.status_code == 200
|
||||
variants = listed.json()["variants"]
|
||||
assert len(variants) == 2 # LEARNER_A's two, LEARNER_B's excluded
|
||||
assert {v["learner_id"] for v in variants} == {LEARNER_A}
|
||||
assert {v["template_id"] for v in variants} == {TEMPLATE, "tpl-guardrail-schema"}
|
||||
# chronological ordering; both carry the competency enrichment
|
||||
assert all(v["competency_id"] for v in variants)
|
||||
|
||||
assert empty.status_code == 200
|
||||
assert empty.json()["variants"] == []
|
||||
|
||||
def test_list_missing_learner_param_422(self, tmp_path, store, provider):
|
||||
with _make_client(tmp_path, store, provider) as client:
|
||||
response = client.get("/v1/variants")
|
||||
|
||||
assert response.status_code == 422
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* Nextcraft — Grading Types
|
||||
*
|
||||
* Rubric-scored grades over real process traces: per-criterion 0-4
|
||||
* scores, strengths, gaps, and verdict, persisted latest-state per
|
||||
* (learner_id, task_id) with the trace digest that fed the rubric prompt.
|
||||
*
|
||||
* Source of truth: the Python models in
|
||||
* `apps/ai-service/ai_service/grading/engine.py` (`RubricScore`) and
|
||||
* `apps/ai-service/ai_service/grading/store.py` (`GradeRecord`),
|
||||
* served over HTTP by `apps/ai-service/ai_service/api/assessment.py`
|
||||
* (`GradeResponse`). This file mirrors those models field-for-field;
|
||||
* any schema change must be made in both places.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The graded rubric criteria set (RUBRIC_CRITERIA, grading/engine.py).
|
||||
* Scores are 0-4 per criterion; the exact four keys are required.
|
||||
*/
|
||||
export interface RubricCriteria {
|
||||
process_quality: number;
|
||||
correctness: number;
|
||||
debugging_discipline: number;
|
||||
test_usage: number;
|
||||
}
|
||||
|
||||
/** Rubric verdict carried in `RubricScore.verdict`. */
|
||||
export type RubricVerdict = 'mastered' | 'developing' | 'not_yet';
|
||||
|
||||
/**
|
||||
* Validated LLM rubric output for a GRADED trace. Mirrors `RubricScore`
|
||||
* (ai_service/grading/engine.py): criteria 0-4, strengths/gaps (1-2
|
||||
* sentences each), rubric verdict.
|
||||
*/
|
||||
export interface RubricScore {
|
||||
criteria: RubricCriteria;
|
||||
strengths: string[];
|
||||
gaps: string[];
|
||||
verdict: RubricVerdict;
|
||||
}
|
||||
|
||||
/** Machine-readable grade outcome (`GradeRecord.verdict`). */
|
||||
export type GradeOutcome =
|
||||
| 'GRADED'
|
||||
| 'UNGRADABLE_TRACE_INCOMPLETE'
|
||||
| 'UNGRADABLE_EMPTY_TRACE';
|
||||
|
||||
/**
|
||||
* Gate detail surfaced in `scores` for UNGRADABLE_* records (G-4):
|
||||
* the integrity flag reason and/or missing seq numbers. Never co-present
|
||||
* with rubric scores — a grade carries one or the other.
|
||||
*/
|
||||
export interface GradeGateDetail {
|
||||
integrity_flag: string | null;
|
||||
missing_seqs: number[];
|
||||
}
|
||||
|
||||
/** Union of the two `scores` shapes a grade can carry. */
|
||||
export type GradeScores = RubricScore | GradeGateDetail;
|
||||
|
||||
/**
|
||||
* A persisted grade for one (learner_id, task_id) trace — latest state
|
||||
* (a regrade replaces the row). Mirrors `GradeRecord`
|
||||
* (ai_service/grading/store.py) field-for-field, except `scores` is
|
||||
* typed as the discriminated `GradeScores` union instead of the Python
|
||||
* side's untyped JSON dict.
|
||||
*/
|
||||
export interface GradeRecord {
|
||||
learner_id: string;
|
||||
/** The graded task — joins to `TaskVariant.task_id`. */
|
||||
task_id: string;
|
||||
/** Task-variant seed (D-029); null while grading is variant-blind. */
|
||||
variant_seed: string | null;
|
||||
/** Compact trace digest (D-028) that fed the rubric prompt; {} for gate records. */
|
||||
digest: Record<string, unknown>;
|
||||
/** Rubric scores (GRADED) or gate detail (UNGRADABLE_*); never both. */
|
||||
scores: GradeScores;
|
||||
/** Machine-readable outcome; rubric verdict rides in `scores.verdict`. */
|
||||
verdict: GradeOutcome;
|
||||
/** Provider model that produced the scores; "none" for gate records. */
|
||||
model: string;
|
||||
/** ISO 8601 UTC grade timestamp; a regrade replaces it. */
|
||||
created_at: string;
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
export * from './domain';
|
||||
export * from './grading';
|
||||
export * from './marketplace';
|
||||
export * from './telemetry';
|
||||
export * from './user';
|
||||
export * from './ui';
|
||||
export * from './ui';
|
||||
export * from './variants';
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Nextcraft — Task Variant Types
|
||||
*
|
||||
* Seeded per-learner task variants: a task template instantiated for one
|
||||
* learner with a reproducible seed, sampled parameter slots, a rendered
|
||||
* statement, and starter-file scaffolds. One (learner_id, template_id)
|
||||
* pair is exactly ONE variant, forever — regeneration is a cache read.
|
||||
*
|
||||
* Source of truth: the Python model in
|
||||
* `apps/ai-service/ai_service/variants/store.py` (`VariantRecord`),
|
||||
* served over HTTP by `apps/ai-service/ai_service/api/variants.py`
|
||||
* (`VariantResponse`, which adds the `competency_id` enrichment). This
|
||||
* file mirrors those fields field-for-field; any schema change must be
|
||||
* made in both places.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Typed parameter-slot values the generator sampled for the variant's
|
||||
* template (seed-derivable, D-029). Enum/string-set slots draw strings;
|
||||
* int-range slots draw numbers. An empty object is legal (a slotless
|
||||
* template).
|
||||
*/
|
||||
export interface VariantParams {
|
||||
[slotName: string]: string | number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A generated task variant. Mirrors `VariantRecord`
|
||||
* (ai_service/variants/store.py) plus the API-layer `competency_id`
|
||||
* enrichment (ai_service/api/variants.py) — the D-021 competency the
|
||||
* variant's template binds to.
|
||||
*/
|
||||
export interface TaskVariant {
|
||||
learner_id: string;
|
||||
/** Globally unique task key — the grading and telemetry join key. */
|
||||
task_id: string;
|
||||
template_id: string;
|
||||
/** D-021 competency the template binds to (resolved at read time). */
|
||||
competency_id: string;
|
||||
/** Reproducible per-(template, learner, milestone) seed (D-029). */
|
||||
seed: string;
|
||||
/** Seeded slot values the statement was rendered from. */
|
||||
params: VariantParams;
|
||||
/** Rendered task statement shown to the learner. */
|
||||
statement: string;
|
||||
/** Workspace scaffold: filename → file content. */
|
||||
starter_files: Record<string, string>;
|
||||
/** ISO 8601 UTC generation timestamp. */
|
||||
created_at: string;
|
||||
}
|
||||
Reference in New Issue
Block a user