feat(P04): variants API + TS types (Wave 3)
Task 4-3-01: POST /v1/variants (template_id or competency_id resolution; cache-first
D-029; 404 unknown template/competency; 422 neither), GET /v1/variants/{task_id},
GET /v1/variants?learner_id=. SQLiteVariantStore + VariantGenerator wired into lifespan.
Task 4-3-02: packages/types/variants.ts (TaskVariant, VariantParams) + grading.ts
(RubricScore + GradeRecord, criteria-typed) — field-for-field Python parity,
scores typed as the documented rubric-or-gate union.
11 endpoint tests green (distinct learners -> distinct statements at API level; cache
hit -> zero LLM calls); suite 324 green; typecheck 7/7; ruff clean.
---ci---
phase: 4
milestone: v0.3
status: execute
requirements: {covered: [REQ-3-005], partial: []}
---/ci---
This commit is contained in:
@@ -10,6 +10,7 @@ from .mentor import router as mentor_router
|
|||||||
from .proctor import router as proctor_router
|
from .proctor import router as proctor_router
|
||||||
from .sandboxes import router as sandboxes_router
|
from .sandboxes import router as sandboxes_router
|
||||||
from .telemetry import router as telemetry_router
|
from .telemetry import router as telemetry_router
|
||||||
|
from .variants import router as variants_router
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"assessment_router",
|
"assessment_router",
|
||||||
@@ -19,4 +20,5 @@ __all__ = [
|
|||||||
"proctor_router",
|
"proctor_router",
|
||||||
"sandboxes_router",
|
"sandboxes_router",
|
||||||
"telemetry_router",
|
"telemetry_router",
|
||||||
|
"variants_router",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ from ..sandbox.manager import SandboxManager
|
|||||||
from ..sandbox.workdir import SandboxDir
|
from ..sandbox.workdir import SandboxDir
|
||||||
from ..telemetry.ingest import TraceIntegrityMap
|
from ..telemetry.ingest import TraceIntegrityMap
|
||||||
from ..telemetry.store import TraceStore
|
from ..telemetry.store import TraceStore
|
||||||
|
from ..variants.generator import VariantGenerator
|
||||||
|
from ..variants.store import VariantStore
|
||||||
|
|
||||||
|
|
||||||
def get_settings(request: Request) -> Settings:
|
def get_settings(request: Request) -> Settings:
|
||||||
@@ -53,3 +55,11 @@ def get_grade_store(request: Request) -> GradeStore:
|
|||||||
|
|
||||||
def get_grading_engine(request: Request) -> GradingEngine:
|
def get_grading_engine(request: Request) -> GradingEngine:
|
||||||
return request.app.state.grading_engine
|
return request.app.state.grading_engine
|
||||||
|
|
||||||
|
|
||||||
|
def get_variant_generator(request: Request) -> VariantGenerator:
|
||||||
|
return request.app.state.variant_generator
|
||||||
|
|
||||||
|
|
||||||
|
def get_variant_store(request: Request) -> VariantStore:
|
||||||
|
return request.app.state.variant_store
|
||||||
|
|||||||
@@ -0,0 +1,187 @@
|
|||||||
|
"""/v1/variants — seeded per-learner task variant endpoints (REQ-3-005, D-029).
|
||||||
|
|
||||||
|
Three faces over the variant engine (the generator + store stay FastAPI-free;
|
||||||
|
this module owns all HTTP wiring — the D-027/D-032 house pattern):
|
||||||
|
|
||||||
|
POST /v1/variants {learner_id, template_id | competency_id}
|
||||||
|
→ 200 the learner's variant — GENERATED on the first request,
|
||||||
|
CACHED (store read, zero LLM calls) on every repeat: D-029
|
||||||
|
reproducibility means one (learner_id, template_id) is ONE
|
||||||
|
variant forever, so a regenerate is always a 200 of the SAME
|
||||||
|
variant, never a second render.
|
||||||
|
→ 404 unknown template_id, or competency_id with no bound template.
|
||||||
|
→ 422 neither template_id nor competency_id given.
|
||||||
|
GET /v1/variants/{task_id}
|
||||||
|
→ 200 the stored variant owning the task key (the grading and
|
||||||
|
telemetry join path); 404 when no variant was ever generated
|
||||||
|
for the task.
|
||||||
|
GET /v1/variants?learner_id=...
|
||||||
|
→ 200 the learner's variants, chronological; [] when none.
|
||||||
|
|
||||||
|
Template resolution: an explicit `template_id` wins; without it the FIRST
|
||||||
|
template bound to `competency_id` is used (`template_for_competency`,
|
||||||
|
D-021 corpus alignment). The response carries `competency_id` resolved
|
||||||
|
from the template library at read time — an enrichment, not a persisted
|
||||||
|
column (the seed re-derives the whole variant, D-029) — so the learner
|
||||||
|
surface can bind a variant to its competency without a library round-trip.
|
||||||
|
|
||||||
|
Distinctness (REQ-3-005): different learners on the same template draw
|
||||||
|
different seeded params and receive distinct statements and task_ids;
|
||||||
|
tests/api/test_variants.py asserts this end-to-end through the API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Self
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
from pydantic import BaseModel, Field, model_validator
|
||||||
|
|
||||||
|
from ..variants.generator import VariantGenerator
|
||||||
|
from ..variants.store import VariantRecord, VariantStore
|
||||||
|
from ..variants.templates import TaskTemplate, get_template, template_for_competency
|
||||||
|
from .deps import get_variant_generator, get_variant_store
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/v1/variants", tags=["variants"])
|
||||||
|
|
||||||
|
|
||||||
|
# -- contracts ------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class VariantGenerateRequest(BaseModel):
|
||||||
|
"""One variant identity: an explicit `template_id`, or the first
|
||||||
|
template bound to a `competency_id` (D-021). `template_id` wins when
|
||||||
|
both are given (explicit identity beats derived); at least one is
|
||||||
|
required — 422 otherwise.
|
||||||
|
"""
|
||||||
|
|
||||||
|
learner_id: str = Field(min_length=1)
|
||||||
|
template_id: str | None = None
|
||||||
|
competency_id: str | None = None
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def _require_template_or_competency(self) -> Self:
|
||||||
|
if self.template_id is None and self.competency_id is None:
|
||||||
|
raise ValueError("template_id or competency_id is required")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class VariantResponse(BaseModel):
|
||||||
|
"""VariantRecord over HTTP, plus the `competency_id` enrichment.
|
||||||
|
|
||||||
|
Every field except `competency_id` mirrors `VariantRecord` exactly
|
||||||
|
(snake_case; `created_at` is an ISO 8601 UTC datetime) — the wire shape
|
||||||
|
typed as `TaskVariant` in packages/types/variants.ts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
learner_id: str
|
||||||
|
task_id: str
|
||||||
|
template_id: str
|
||||||
|
competency_id: str
|
||||||
|
seed: str
|
||||||
|
params: dict[str, str | int]
|
||||||
|
statement: str
|
||||||
|
starter_files: dict[str, str]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class VariantListResponse(BaseModel):
|
||||||
|
variants: list[VariantResponse]
|
||||||
|
|
||||||
|
|
||||||
|
# -- resolution + rendering -----------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_template(body: VariantGenerateRequest) -> TaskTemplate:
|
||||||
|
"""Template for the request: the explicit id, else the first template
|
||||||
|
bound to the competency; 404 when neither resolves."""
|
||||||
|
if body.template_id is not None:
|
||||||
|
template = get_template(body.template_id)
|
||||||
|
if template is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404,
|
||||||
|
detail=f"no task template with id {body.template_id!r}",
|
||||||
|
)
|
||||||
|
return template
|
||||||
|
# The request validator guarantees the disjunction, so reaching here
|
||||||
|
# means a competency_id was given (never None).
|
||||||
|
assert body.competency_id is not None
|
||||||
|
templates = template_for_competency(body.competency_id)
|
||||||
|
if not templates:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404,
|
||||||
|
detail=f"no task template for competency {body.competency_id!r}",
|
||||||
|
)
|
||||||
|
return templates[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _competency_for(template_id: str) -> str:
|
||||||
|
"""competency_id enrichment for stored records (read paths)."""
|
||||||
|
template = get_template(template_id)
|
||||||
|
if template is None:
|
||||||
|
# Integrity guard: a stored variant referencing a template that is
|
||||||
|
# no longer in the library cannot be enriched; fail loudly rather
|
||||||
|
# than fabricate a competency binding.
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=500,
|
||||||
|
detail=f"stored variant references unknown template {template_id!r}",
|
||||||
|
)
|
||||||
|
return template.competency_id
|
||||||
|
|
||||||
|
|
||||||
|
def _to_response(record: VariantRecord, competency_id: str) -> VariantResponse:
|
||||||
|
return VariantResponse(
|
||||||
|
learner_id=record.learner_id,
|
||||||
|
task_id=record.task_id,
|
||||||
|
template_id=record.template_id,
|
||||||
|
competency_id=competency_id,
|
||||||
|
seed=record.seed,
|
||||||
|
params=dict(record.params),
|
||||||
|
statement=record.statement,
|
||||||
|
starter_files=dict(record.starter_files),
|
||||||
|
created_at=record.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# -- endpoints ------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", response_model=VariantResponse)
|
||||||
|
async def generate_variant(
|
||||||
|
body: VariantGenerateRequest,
|
||||||
|
generator: VariantGenerator = Depends(get_variant_generator),
|
||||||
|
) -> VariantResponse:
|
||||||
|
"""The learner's variant for the resolved template — generated on the
|
||||||
|
first request, cached (no LLM call) on every repeat: D-029 makes a
|
||||||
|
regenerate a 200 of the SAME stored variant.
|
||||||
|
"""
|
||||||
|
template = _resolve_template(body)
|
||||||
|
record = await generator.generate(body.learner_id, template.id)
|
||||||
|
return _to_response(record, competency_id=template.competency_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=VariantListResponse)
|
||||||
|
async def list_variants(
|
||||||
|
learner_id: str,
|
||||||
|
store: VariantStore = Depends(get_variant_store),
|
||||||
|
) -> VariantListResponse:
|
||||||
|
"""All stored variants for the learner, chronological; [] when none."""
|
||||||
|
variants = [
|
||||||
|
_to_response(record, competency_id=_competency_for(record.template_id))
|
||||||
|
for record in store.list_for_learner(learner_id)
|
||||||
|
]
|
||||||
|
return VariantListResponse(variants=variants)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{task_id}", response_model=VariantResponse)
|
||||||
|
async def get_variant(
|
||||||
|
task_id: str,
|
||||||
|
store: VariantStore = Depends(get_variant_store),
|
||||||
|
) -> VariantResponse:
|
||||||
|
"""The stored variant owning the task key — the grading and telemetry
|
||||||
|
join path; 404 when no variant was ever generated for the task."""
|
||||||
|
record = store.get_by_task(task_id)
|
||||||
|
if record is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404, detail=f"no stored variant for task {task_id!r}"
|
||||||
|
)
|
||||||
|
return _to_response(record, competency_id=_competency_for(record.template_id))
|
||||||
@@ -19,6 +19,7 @@ from .api import (
|
|||||||
proctor_router,
|
proctor_router,
|
||||||
sandboxes_router,
|
sandboxes_router,
|
||||||
telemetry_router,
|
telemetry_router,
|
||||||
|
variants_router,
|
||||||
)
|
)
|
||||||
from .config import Settings
|
from .config import Settings
|
||||||
from .grading.engine import GradingEngine
|
from .grading.engine import GradingEngine
|
||||||
@@ -27,6 +28,8 @@ from .llm import create_provider
|
|||||||
from .sandbox import SandboxManager, UnshareBackend
|
from .sandbox import SandboxManager, UnshareBackend
|
||||||
from .telemetry.ingest import TraceIntegrityMap
|
from .telemetry.ingest import TraceIntegrityMap
|
||||||
from .telemetry.store import SQLiteTraceStore
|
from .telemetry.store import SQLiteTraceStore
|
||||||
|
from .variants.generator import VariantGenerator
|
||||||
|
from .variants.store import SQLiteVariantStore
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -94,6 +97,27 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
model=settings.model,
|
model=settings.model,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Variant generation (REQ-3-005, Wave 3): VariantStore from the same
|
||||||
|
# SQLite file as traces/grades (D-027), one VariantGenerator singleton
|
||||||
|
# wired through app.state — the generator receives store + provider
|
||||||
|
# via constructor DI and knows nothing of FastAPI (api/ composes it,
|
||||||
|
# same pattern as GradingEngine). Tests may pre-set
|
||||||
|
# app.state.variant_store / app.state.variant_generator (the same
|
||||||
|
# state-injection override); the lifespan adopts a pre-set store but
|
||||||
|
# NEVER rebuilds a pre-set generator (its provider binding is part
|
||||||
|
# of the test fixture).
|
||||||
|
variant_store = getattr(app.state, "variant_store", None)
|
||||||
|
if variant_store is None:
|
||||||
|
settings.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
variant_store = SQLiteVariantStore(db_path=settings.db_path)
|
||||||
|
app.state.variant_store = variant_store
|
||||||
|
if getattr(app.state, "variant_generator", None) is None:
|
||||||
|
app.state.variant_generator = VariantGenerator(
|
||||||
|
variant_store,
|
||||||
|
app.state.provider,
|
||||||
|
model=settings.model,
|
||||||
|
)
|
||||||
|
|
||||||
async def _reaper_loop() -> None:
|
async def _reaper_loop() -> None:
|
||||||
# Wall-clock timeout + G-2 workdir-size sweep, one pass per tick.
|
# Wall-clock timeout + G-2 workdir-size sweep, one pass per tick.
|
||||||
while True:
|
while True:
|
||||||
@@ -115,6 +139,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
await manager.destroy_all()
|
await manager.destroy_all()
|
||||||
trace_store.close()
|
trace_store.close()
|
||||||
grade_store.close()
|
grade_store.close()
|
||||||
|
variant_store.close()
|
||||||
await app.state.http_client.aclose()
|
await app.state.http_client.aclose()
|
||||||
|
|
||||||
app = FastAPI(title="Nextcraft AI Service", version="0.3.0", lifespan=lifespan)
|
app = FastAPI(title="Nextcraft AI Service", version="0.3.0", lifespan=lifespan)
|
||||||
@@ -143,6 +168,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
app.include_router(proctor_router)
|
app.include_router(proctor_router)
|
||||||
app.include_router(sandboxes_router)
|
app.include_router(sandboxes_router)
|
||||||
app.include_router(telemetry_router)
|
app.include_router(telemetry_router)
|
||||||
|
app.include_router(variants_router)
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,277 @@
|
|||||||
|
"""Variant API tests — generation, cache, distinctness over HTTP (Task 4-3-01).
|
||||||
|
|
||||||
|
Contract under test (api/variants.py, REQ-3-005):
|
||||||
|
|
||||||
|
POST /v1/variants {learner_id, template_id}
|
||||||
|
first request → 200 generated variant (task_id, seed, params,
|
||||||
|
statement, starter_files, competency_id)
|
||||||
|
repeat request → 200 the SAME cached variant with ZERO LLM
|
||||||
|
calls (D-029 reproducibility through the
|
||||||
|
whole HTTP stack)
|
||||||
|
unknown template → 404
|
||||||
|
POST /v1/variants {learner_id, competency_id}
|
||||||
|
bound competency → 200 the first template for that competency
|
||||||
|
unknown competency → 404
|
||||||
|
neither id given → 422
|
||||||
|
GET /v1/variants/{task_id}
|
||||||
|
generated earlier → 200 the stored variant (generate→get roundtrip)
|
||||||
|
unknown task → 404
|
||||||
|
GET /v1/variants?learner_id=...
|
||||||
|
→ 200 that learner's variants only (scoping); [] for a learner
|
||||||
|
with none.
|
||||||
|
|
||||||
|
Distinctness (REQ-3-005, API level): two learners POSTing the same
|
||||||
|
template receive distinct statements, seeds and task_ids, and GET by
|
||||||
|
task_id hands each back their own variant.
|
||||||
|
|
||||||
|
Wiring: per-test tmp-path SQLiteVariantStore + a pre-set VariantGenerator
|
||||||
|
(state-injection override — the lifespan adopts variant_store /
|
||||||
|
variant_generator from app.state instead of constructing them; same
|
||||||
|
pattern as test_grading.py / test_telemetry_ingest.py). The generator
|
||||||
|
binds a ScriptedRenderProvider so each test controls the render LLM
|
||||||
|
exactly, and counts calls so the cache test can assert the LLM was never
|
||||||
|
reached on the second POST.
|
||||||
|
|
||||||
|
Zero network: providers are MockProvider family members only (conftest
|
||||||
|
rule, enforced in _make_client).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from ai_service.config import Settings
|
||||||
|
from ai_service.llm.mock import MockProvider
|
||||||
|
from ai_service.main import create_app
|
||||||
|
from ai_service.variants.generator import VariantGenerator
|
||||||
|
from ai_service.variants.store import SQLiteVariantStore
|
||||||
|
from ai_service.variants.templates import get_template
|
||||||
|
|
||||||
|
LEARNER_A = "variant-learner-a"
|
||||||
|
LEARNER_B = "variant-learner-b"
|
||||||
|
TEMPLATE = "tpl-llm-judge"
|
||||||
|
COMPETENCY = "stack-orchestration-c007" # tpl-llm-judge's D-021 binding
|
||||||
|
|
||||||
|
|
||||||
|
class ScriptedRenderProvider(MockProvider):
|
||||||
|
"""Deterministic render whose statement embeds the params (distinct per
|
||||||
|
draw); counts calls so the cache test asserts zero LLM calls on the
|
||||||
|
second POST. Mirrors tests/variants/test_generator.py."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
async def chat(self, messages, *, model, temperature=0.7, response_format=None): # noqa: ANN001
|
||||||
|
self.calls += 1
|
||||||
|
import json
|
||||||
|
|
||||||
|
user = next(m.content for m in reversed(messages) if m.role == "user")
|
||||||
|
# Distinct per distinct params: hash the seeded slot lines.
|
||||||
|
fingerprint = abs(hash(user)) % 10_000
|
||||||
|
return json.dumps({"statement": f"Scripted variant #{fingerprint} — build it."})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def store(tmp_path: Path) -> Iterator[SQLiteVariantStore]:
|
||||||
|
s = SQLiteVariantStore(db_path=tmp_path / "variants.db")
|
||||||
|
yield s
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def provider() -> ScriptedRenderProvider:
|
||||||
|
return ScriptedRenderProvider()
|
||||||
|
|
||||||
|
|
||||||
|
def _make_client(
|
||||||
|
tmp_path: Path, store: SQLiteVariantStore, provider: MockProvider
|
||||||
|
) -> TestClient:
|
||||||
|
"""App + TestClient with a pre-set store + generator.
|
||||||
|
|
||||||
|
The lifespan adopts both (state-injection override); the generator
|
||||||
|
binds OUR provider, so the fixture — not Settings — scripts the LLM.
|
||||||
|
Cloud-free guard: the provider must be a MockProvider family member
|
||||||
|
(conftest rule, enforced here because this module builds its own
|
||||||
|
client rather than consuming the conftest one).
|
||||||
|
"""
|
||||||
|
assert isinstance(provider, MockProvider)
|
||||||
|
settings = Settings(
|
||||||
|
provider="mock",
|
||||||
|
db_path=tmp_path / "variant-test.db",
|
||||||
|
sandbox_dir=tmp_path / "sandboxes",
|
||||||
|
)
|
||||||
|
app = create_app(settings)
|
||||||
|
app.state.variant_store = store
|
||||||
|
if getattr(app.state, "variant_generator", None) is None:
|
||||||
|
app.state.variant_generator = VariantGenerator(
|
||||||
|
store, provider, model="gemma4:31b"
|
||||||
|
)
|
||||||
|
return TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def _post(
|
||||||
|
client: TestClient,
|
||||||
|
learner_id: str,
|
||||||
|
template_id: str | None = TEMPLATE,
|
||||||
|
competency_id: str | None = None,
|
||||||
|
):
|
||||||
|
return client.post(
|
||||||
|
"/v1/variants",
|
||||||
|
json={
|
||||||
|
"learner_id": learner_id,
|
||||||
|
**({"template_id": template_id} if template_id is not None else {}),
|
||||||
|
**({"competency_id": competency_id} if competency_id is not None else {}),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# -- POST: generation + response shape --------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestGenerate:
|
||||||
|
def test_post_returns_full_variant_payload(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
response = _post(client, LEARNER_A)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
body = response.json()
|
||||||
|
assert body["learner_id"] == LEARNER_A
|
||||||
|
assert body["template_id"] == TEMPLATE
|
||||||
|
assert body["competency_id"] == COMPETENCY # enrichment from get_template
|
||||||
|
assert body["task_id"].startswith("task-") and len(body["task_id"]) == len("task-") + 16
|
||||||
|
assert body["seed"] # non-empty D-029 seed
|
||||||
|
assert body["statement"] # rendered, non-empty
|
||||||
|
assert body["starter_files"] == get_template(TEMPLATE).starter_files
|
||||||
|
assert body["params"] # seeded slot values
|
||||||
|
assert body["created_at"] # ISO timestamp travels
|
||||||
|
assert provider.calls == 1 # first POST renders exactly once
|
||||||
|
|
||||||
|
def test_post_then_get_roundtrip(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
posted = _post(client, LEARNER_A)
|
||||||
|
assert posted.status_code == 200
|
||||||
|
|
||||||
|
fetched = client.get(f"/v1/variants/{posted.json()['task_id']}")
|
||||||
|
|
||||||
|
assert fetched.status_code == 200
|
||||||
|
assert fetched.json() == posted.json() # identical stored variant
|
||||||
|
|
||||||
|
def test_post_regenerate_is_cache_hit_no_llm_call(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
first = _post(client, LEARNER_A)
|
||||||
|
assert first.status_code == 200
|
||||||
|
assert provider.calls == 1
|
||||||
|
|
||||||
|
second = _post(client, LEARNER_A) # same (learner, template) → cache
|
||||||
|
assert second.status_code == 200
|
||||||
|
assert second.json() == first.json() # D-029: the SAME variant
|
||||||
|
assert provider.calls == 1, "cached regenerate must not render again"
|
||||||
|
|
||||||
|
|
||||||
|
# -- POST: distinctness (REQ-3-005, API level) ------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestDistinctLearners:
|
||||||
|
def test_two_learners_distinct_statements_and_task_ids(
|
||||||
|
self, tmp_path, store, provider
|
||||||
|
):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
a = _post(client, LEARNER_A)
|
||||||
|
b = _post(client, LEARNER_B)
|
||||||
|
assert a.status_code == 200 and b.status_code == 200
|
||||||
|
|
||||||
|
a_body, b_body = a.json(), b.json()
|
||||||
|
|
||||||
|
# ...and each learner GETs back exactly their own variant
|
||||||
|
a_fetched = client.get(f"/v1/variants/{a_body['task_id']}")
|
||||||
|
b_fetched = client.get(f"/v1/variants/{b_body['task_id']}")
|
||||||
|
|
||||||
|
assert a_body["statement"] != b_body["statement"]
|
||||||
|
assert a_body["seed"] != b_body["seed"]
|
||||||
|
assert a_body["task_id"] != b_body["task_id"]
|
||||||
|
assert a_body["competency_id"] == b_body["competency_id"] # same bar (a-5)
|
||||||
|
assert a_fetched.status_code == 200 and a_fetched.json() == a_body
|
||||||
|
assert b_fetched.status_code == 200 and b_fetched.json() == b_body
|
||||||
|
|
||||||
|
|
||||||
|
# -- POST: template / competency resolution ---------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestTemplateResolution:
|
||||||
|
def test_unknown_template_404(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
response = _post(client, LEARNER_A, template_id="tpl-does-not-exist")
|
||||||
|
|
||||||
|
assert response.status_code == 404
|
||||||
|
assert "no task template" in response.json()["detail"]
|
||||||
|
|
||||||
|
def test_competency_lookup_uses_first_bound_template(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
response = _post(
|
||||||
|
client, LEARNER_A, template_id=None, competency_id=COMPETENCY
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
body = response.json()
|
||||||
|
assert body["template_id"] == TEMPLATE # the competency's first template
|
||||||
|
assert body["competency_id"] == COMPETENCY
|
||||||
|
|
||||||
|
def test_unknown_competency_404(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
response = _post(
|
||||||
|
client, LEARNER_A, template_id=None, competency_id="stack-none-c999"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 404
|
||||||
|
assert "no task template for competency" in response.json()["detail"]
|
||||||
|
|
||||||
|
def test_neither_id_given_422(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
response = _post(client, LEARNER_A, template_id=None, competency_id=None)
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
# -- GET: stored reads ------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestStoredReads:
|
||||||
|
def test_get_unknown_task_404(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
response = client.get("/v1/variants/task-0000000000000000")
|
||||||
|
|
||||||
|
assert response.status_code == 404
|
||||||
|
assert "no stored variant" in response.json()["detail"]
|
||||||
|
|
||||||
|
def test_list_scoped_to_one_learner(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
assert _post(client, LEARNER_A).status_code == 200
|
||||||
|
assert _post(
|
||||||
|
client, LEARNER_A, template_id="tpl-guardrail-schema"
|
||||||
|
).status_code == 200
|
||||||
|
assert _post(client, LEARNER_B).status_code == 200
|
||||||
|
|
||||||
|
listed = client.get("/v1/variants", params={"learner_id": LEARNER_A})
|
||||||
|
empty = client.get("/v1/variants", params={"learner_id": "nobody"})
|
||||||
|
|
||||||
|
assert listed.status_code == 200
|
||||||
|
variants = listed.json()["variants"]
|
||||||
|
assert len(variants) == 2 # LEARNER_A's two, LEARNER_B's excluded
|
||||||
|
assert {v["learner_id"] for v in variants} == {LEARNER_A}
|
||||||
|
assert {v["template_id"] for v in variants} == {TEMPLATE, "tpl-guardrail-schema"}
|
||||||
|
# chronological ordering; both carry the competency enrichment
|
||||||
|
assert all(v["competency_id"] for v in variants)
|
||||||
|
|
||||||
|
assert empty.status_code == 200
|
||||||
|
assert empty.json()["variants"] == []
|
||||||
|
|
||||||
|
def test_list_missing_learner_param_422(self, tmp_path, store, provider):
|
||||||
|
with _make_client(tmp_path, store, provider) as client:
|
||||||
|
response = client.get("/v1/variants")
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
/**
|
||||||
|
* Nextcraft — Grading Types
|
||||||
|
*
|
||||||
|
* Rubric-scored grades over real process traces: per-criterion 0-4
|
||||||
|
* scores, strengths, gaps, and verdict, persisted latest-state per
|
||||||
|
* (learner_id, task_id) with the trace digest that fed the rubric prompt.
|
||||||
|
*
|
||||||
|
* Source of truth: the Python models in
|
||||||
|
* `apps/ai-service/ai_service/grading/engine.py` (`RubricScore`) and
|
||||||
|
* `apps/ai-service/ai_service/grading/store.py` (`GradeRecord`),
|
||||||
|
* served over HTTP by `apps/ai-service/ai_service/api/assessment.py`
|
||||||
|
* (`GradeResponse`). This file mirrors those models field-for-field;
|
||||||
|
* any schema change must be made in both places.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The graded rubric criteria set (RUBRIC_CRITERIA, grading/engine.py).
|
||||||
|
* Scores are 0-4 per criterion; the exact four keys are required.
|
||||||
|
*/
|
||||||
|
export interface RubricCriteria {
|
||||||
|
process_quality: number;
|
||||||
|
correctness: number;
|
||||||
|
debugging_discipline: number;
|
||||||
|
test_usage: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Rubric verdict carried in `RubricScore.verdict`. */
|
||||||
|
export type RubricVerdict = 'mastered' | 'developing' | 'not_yet';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validated LLM rubric output for a GRADED trace. Mirrors `RubricScore`
|
||||||
|
* (ai_service/grading/engine.py): criteria 0-4, strengths/gaps (1-2
|
||||||
|
* sentences each), rubric verdict.
|
||||||
|
*/
|
||||||
|
export interface RubricScore {
|
||||||
|
criteria: RubricCriteria;
|
||||||
|
strengths: string[];
|
||||||
|
gaps: string[];
|
||||||
|
verdict: RubricVerdict;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Machine-readable grade outcome (`GradeRecord.verdict`). */
|
||||||
|
export type GradeOutcome =
|
||||||
|
| 'GRADED'
|
||||||
|
| 'UNGRADABLE_TRACE_INCOMPLETE'
|
||||||
|
| 'UNGRADABLE_EMPTY_TRACE';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gate detail surfaced in `scores` for UNGRADABLE_* records (G-4):
|
||||||
|
* the integrity flag reason and/or missing seq numbers. Never co-present
|
||||||
|
* with rubric scores — a grade carries one or the other.
|
||||||
|
*/
|
||||||
|
export interface GradeGateDetail {
|
||||||
|
integrity_flag: string | null;
|
||||||
|
missing_seqs: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Union of the two `scores` shapes a grade can carry. */
|
||||||
|
export type GradeScores = RubricScore | GradeGateDetail;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A persisted grade for one (learner_id, task_id) trace — latest state
|
||||||
|
* (a regrade replaces the row). Mirrors `GradeRecord`
|
||||||
|
* (ai_service/grading/store.py) field-for-field, except `scores` is
|
||||||
|
* typed as the discriminated `GradeScores` union instead of the Python
|
||||||
|
* side's untyped JSON dict.
|
||||||
|
*/
|
||||||
|
export interface GradeRecord {
|
||||||
|
learner_id: string;
|
||||||
|
/** The graded task — joins to `TaskVariant.task_id`. */
|
||||||
|
task_id: string;
|
||||||
|
/** Task-variant seed (D-029); null while grading is variant-blind. */
|
||||||
|
variant_seed: string | null;
|
||||||
|
/** Compact trace digest (D-028) that fed the rubric prompt; {} for gate records. */
|
||||||
|
digest: Record<string, unknown>;
|
||||||
|
/** Rubric scores (GRADED) or gate detail (UNGRADABLE_*); never both. */
|
||||||
|
scores: GradeScores;
|
||||||
|
/** Machine-readable outcome; rubric verdict rides in `scores.verdict`. */
|
||||||
|
verdict: GradeOutcome;
|
||||||
|
/** Provider model that produced the scores; "none" for gate records. */
|
||||||
|
model: string;
|
||||||
|
/** ISO 8601 UTC grade timestamp; a regrade replaces it. */
|
||||||
|
created_at: string;
|
||||||
|
}
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
export * from './domain';
|
export * from './domain';
|
||||||
|
export * from './grading';
|
||||||
export * from './marketplace';
|
export * from './marketplace';
|
||||||
export * from './telemetry';
|
export * from './telemetry';
|
||||||
export * from './user';
|
export * from './user';
|
||||||
export * from './ui';
|
export * from './ui';
|
||||||
|
export * from './variants';
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
/**
|
||||||
|
* Nextcraft — Task Variant Types
|
||||||
|
*
|
||||||
|
* Seeded per-learner task variants: a task template instantiated for one
|
||||||
|
* learner with a reproducible seed, sampled parameter slots, a rendered
|
||||||
|
* statement, and starter-file scaffolds. One (learner_id, template_id)
|
||||||
|
* pair is exactly ONE variant, forever — regeneration is a cache read.
|
||||||
|
*
|
||||||
|
* Source of truth: the Python model in
|
||||||
|
* `apps/ai-service/ai_service/variants/store.py` (`VariantRecord`),
|
||||||
|
* served over HTTP by `apps/ai-service/ai_service/api/variants.py`
|
||||||
|
* (`VariantResponse`, which adds the `competency_id` enrichment). This
|
||||||
|
* file mirrors those fields field-for-field; any schema change must be
|
||||||
|
* made in both places.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Typed parameter-slot values the generator sampled for the variant's
|
||||||
|
* template (seed-derivable, D-029). Enum/string-set slots draw strings;
|
||||||
|
* int-range slots draw numbers. An empty object is legal (a slotless
|
||||||
|
* template).
|
||||||
|
*/
|
||||||
|
export interface VariantParams {
|
||||||
|
[slotName: string]: string | number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A generated task variant. Mirrors `VariantRecord`
|
||||||
|
* (ai_service/variants/store.py) plus the API-layer `competency_id`
|
||||||
|
* enrichment (ai_service/api/variants.py) — the D-021 competency the
|
||||||
|
* variant's template binds to.
|
||||||
|
*/
|
||||||
|
export interface TaskVariant {
|
||||||
|
learner_id: string;
|
||||||
|
/** Globally unique task key — the grading and telemetry join key. */
|
||||||
|
task_id: string;
|
||||||
|
template_id: string;
|
||||||
|
/** D-021 competency the template binds to (resolved at read time). */
|
||||||
|
competency_id: string;
|
||||||
|
/** Reproducible per-(template, learner, milestone) seed (D-029). */
|
||||||
|
seed: string;
|
||||||
|
/** Seeded slot values the statement was rendered from. */
|
||||||
|
params: VariantParams;
|
||||||
|
/** Rendered task statement shown to the learner. */
|
||||||
|
statement: string;
|
||||||
|
/** Workspace scaffold: filename → file content. */
|
||||||
|
starter_files: Record<string, string>;
|
||||||
|
/** ISO 8601 UTC generation timestamp. */
|
||||||
|
created_at: string;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user