nextcraft/apps/ai-service/tests/api/test_variants.py

"""Variant API tests — generation, cache, distinctness over HTTP (Task 4-3-01).

Contract under test (api/variants.py, REQ-3-005):

    POST /v1/variants {learner_id, template_id}
        first request       → 200 generated variant (task_id, seed, params,
                              statement, starter_files, competency_id)
        repeat request      → 200 the SAME cached variant with ZERO LLM
                              calls (D-029 reproducibility through the
                              whole HTTP stack)
        unknown template    → 404
    POST /v1/variants {learner_id, competency_id}
        bound competency    → 200 the first template for that competency
        unknown competency  → 404
        neither id given    → 422
    GET  /v1/variants/{task_id}
        generated earlier  → 200 the stored variant (generate→get roundtrip)
        unknown task       → 404
    GET  /v1/variants?learner_id=...
        → 200 that learner's variants only (scoping); [] for a learner
          with none.

Distinctness (REQ-3-005, API level): two learners POSTing the same
template receive distinct statements, seeds and task_ids, and GET by
task_id hands each back their own variant.

Wiring: per-test tmp-path SQLiteVariantStore + a pre-set VariantGenerator
(state-injection override — the lifespan adopts variant_store /
variant_generator from app.state instead of constructing them; same
pattern as test_grading.py / test_telemetry_ingest.py). The generator
binds a ScriptedRenderProvider so each test controls the render LLM
exactly, and counts calls so the cache test can assert the LLM was never
reached on the second POST.

Zero network: providers are MockProvider family members only (conftest
rule, enforced in _make_client).
"""

from __future__ import annotations

from collections.abc import Iterator
from pathlib import Path

import pytest
from fastapi.testclient import TestClient

from ai_service.config import Settings
from ai_service.identity.store import SQLiteIdentityStore
from ai_service.llm.mock import MockProvider
from ai_service.main import create_app
from ai_service.variants.generator import VariantGenerator
from ai_service.variants.store import SQLiteVariantStore
from ai_service.variants.templates import get_template

from ..conftest import SUITE_LEARNERS, seed_verified_identity

LEARNER_A = "variant-learner-a"
LEARNER_B = "variant-learner-b"
TEMPLATE = "tpl-llm-judge"
COMPETENCY = "stack-orchestration-c007"  # tpl-llm-judge's D-021 binding


class ScriptedRenderProvider(MockProvider):
    """Deterministic render whose statement embeds the params (distinct per
    draw); counts calls so the cache test asserts zero LLM calls on the
    second POST. Mirrors tests/variants/test_generator.py."""

    def __init__(self) -> None:
        super().__init__()
        self.calls = 0

    async def chat(self, messages, *, model, temperature=0.7, response_format=None):
        self.calls += 1
        import json

        user = next(m.content for m in reversed(messages) if m.role == "user")
        # Distinct per distinct params: hash the seeded slot lines.
        fingerprint = abs(hash(user)) % 10_000
        return json.dumps({"statement": f"Scripted variant #{fingerprint} — build it."})


@pytest.fixture()
def store(tmp_path: Path) -> Iterator[SQLiteVariantStore]:
    s = SQLiteVariantStore(db_path=tmp_path / "variants.db")
    yield s
    s.close()


@pytest.fixture()
def provider() -> ScriptedRenderProvider:
    return ScriptedRenderProvider()


def _make_client(
    tmp_path: Path, store: SQLiteVariantStore, provider: MockProvider
) -> TestClient:
    """App + TestClient with a pre-set store + generator.

    The lifespan adopts both (state-injection override); the generator
    binds OUR provider, so the fixture — not Settings — scripts the LLM.
    Cloud-free guard: the provider must be a MockProvider family member
    (conftest rule, enforced here because this module builds its own
    client rather than consuming the conftest one).
    """
    assert isinstance(provider, MockProvider)
    settings = Settings(
        provider="mock",
        db_path=tmp_path / "variant-test.db",
        sandbox_dir=tmp_path / "sandboxes",
        learner_allowlist=SUITE_LEARNERS,
    )
    app = create_app(settings)
    identity_store = SQLiteIdentityStore(db_path=tmp_path / "identity.db")
    seed_verified_identity(identity_store)
    app.state.identity_store = identity_store  # G-9
    app.state.variant_store = store
    if getattr(app.state, "variant_generator", None) is None:
        app.state.variant_generator = VariantGenerator(
            store, provider, model="gemma4:31b"
        )
    return TestClient(app)


def _post(
    client: TestClient,
    learner_id: str,
    template_id: str | None = TEMPLATE,
    competency_id: str | None = None,
):
    return client.post(
        "/v1/variants",
        json={
            "learner_id": learner_id,
            **({"template_id": template_id} if template_id is not None else {}),
            **({"competency_id": competency_id} if competency_id is not None else {}),
        },
    )


# -- POST: generation + response shape --------------------------------------------


class TestGenerate:
    def test_post_returns_full_variant_payload(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            response = _post(client, LEARNER_A)

        assert response.status_code == 200
        body = response.json()
        assert body["learner_id"] == LEARNER_A
        assert body["template_id"] == TEMPLATE
        assert body["competency_id"] == COMPETENCY  # enrichment from get_template
        assert body["task_id"].startswith("task-") and len(body["task_id"]) == len("task-") + 16
        assert body["seed"]  # non-empty D-029 seed
        assert body["statement"]  # rendered, non-empty
        template = get_template(TEMPLATE)
        assert template is not None, "seed template must resolve"
        assert body["starter_files"] == template.starter_files
        assert body["params"]  # seeded slot values
        assert body["created_at"]  # ISO timestamp travels
        assert provider.calls == 1  # first POST renders exactly once

    def test_post_then_get_roundtrip(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            posted = _post(client, LEARNER_A)
            assert posted.status_code == 200

            fetched = client.get(f"/v1/variants/{posted.json()['task_id']}")

        assert fetched.status_code == 200
        assert fetched.json() == posted.json()  # identical stored variant

    def test_post_regenerate_is_cache_hit_no_llm_call(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            first = _post(client, LEARNER_A)
            assert first.status_code == 200
            assert provider.calls == 1

            second = _post(client, LEARNER_A)  # same (learner, template) → cache
            assert second.status_code == 200
            assert second.json() == first.json()  # D-029: the SAME variant
            assert provider.calls == 1, "cached regenerate must not render again"


# -- POST: distinctness (REQ-3-005, API level) ------------------------------------


class TestDistinctLearners:
    def test_two_learners_distinct_statements_and_task_ids(
        self, tmp_path, store, provider
    ):
        with _make_client(tmp_path, store, provider) as client:
            a = _post(client, LEARNER_A)
            b = _post(client, LEARNER_B)
            assert a.status_code == 200 and b.status_code == 200

            a_body, b_body = a.json(), b.json()

            # ...and each learner GETs back exactly their own variant
            a_fetched = client.get(f"/v1/variants/{a_body['task_id']}")
            b_fetched = client.get(f"/v1/variants/{b_body['task_id']}")

        assert a_body["statement"] != b_body["statement"]
        assert a_body["seed"] != b_body["seed"]
        assert a_body["task_id"] != b_body["task_id"]
        assert a_body["competency_id"] == b_body["competency_id"]  # same bar (a-5)
        assert a_fetched.status_code == 200 and a_fetched.json() == a_body
        assert b_fetched.status_code == 200 and b_fetched.json() == b_body


# -- POST: template / competency resolution ---------------------------------------


class TestTemplateResolution:
    def test_unknown_template_404(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            response = _post(client, LEARNER_A, template_id="tpl-does-not-exist")

        assert response.status_code == 404
        assert "no task template" in response.json()["detail"]

    def test_competency_lookup_uses_first_bound_template(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            response = _post(
                client, LEARNER_A, template_id=None, competency_id=COMPETENCY
            )

        assert response.status_code == 200
        body = response.json()
        assert body["template_id"] == TEMPLATE  # the competency's first template
        assert body["competency_id"] == COMPETENCY

    def test_unknown_competency_404(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            response = _post(
                client, LEARNER_A, template_id=None, competency_id="stack-none-c999"
            )

        assert response.status_code == 404
        assert "no task template for competency" in response.json()["detail"]

    def test_neither_id_given_422(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            response = _post(client, LEARNER_A, template_id=None, competency_id=None)

        assert response.status_code == 422


# -- GET: stored reads ------------------------------------------------------------


class TestStoredReads:
    def test_get_unknown_task_404(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            response = client.get("/v1/variants/task-0000000000000000")

        assert response.status_code == 404
        assert "no stored variant" in response.json()["detail"]

    def test_list_scoped_to_one_learner(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            assert _post(client, LEARNER_A).status_code == 200
            assert _post(
                client, LEARNER_A, template_id="tpl-guardrail-schema"
            ).status_code == 200
            assert _post(client, LEARNER_B).status_code == 200

            listed = client.get("/v1/variants", params={"learner_id": LEARNER_A})
            empty = client.get("/v1/variants", params={"learner_id": "nobody"})

        assert listed.status_code == 200
        variants = listed.json()["variants"]
        assert len(variants) == 2  # LEARNER_A's two, LEARNER_B's excluded
        assert {v["learner_id"] for v in variants} == {LEARNER_A}
        assert {v["template_id"] for v in variants} == {TEMPLATE, "tpl-guardrail-schema"}
        # chronological ordering; both carry the competency enrichment
        assert all(v["competency_id"] for v in variants)

        assert empty.status_code == 200
        assert empty.json()["variants"] == []

    def test_list_missing_learner_param_422(self, tmp_path, store, provider):
        with _make_client(tmp_path, store, provider) as client:
            response = client.get("/v1/variants")

        assert response.status_code == 422