diff --git a/.ciagent/PERSONAS.md b/.ciagent/PERSONAS.md index a2f0513..cfbf0e3 100644 --- a/.ciagent/PERSONAS.md +++ b/.ciagent/PERSONAS.md @@ -40,7 +40,7 @@ reason: "v0.2 is the MVP: FastAPI backend + engine + adapters + trust loop (back ### security-engineer (phase-specific: P2) - **Domain**: security -- **Active**: true (P2 only) +- **Active**: false (P2 shipped — removed per phase-specific policy) - **Phase-specific**: true - **Frameworks**: httpx, pytest, STRIDE analysis - **Constraints**: secrets resolve from env-file scopes only (OLLAMA_API_KEY, TAVILY_API_KEY, GITEA_TOKEN via .ciagent/.env.secrets) — never shell env, never logged, never in git; liveness verification and trust-loop code must not leak keys in errors or traces; SSRF guard for fetched URLs (private-address refusal); authority tiering must not phone home diff --git a/.ciagent/PLAN.md b/.ciagent/PLAN.md index a120e7a..059a03d 100644 --- a/.ciagent/PLAN.md +++ b/.ciagent/PLAN.md @@ -1,61 +1,44 @@ -# PLAN — Milestone v0.2 Phase 4: Hardening + Adversarial Suite + E2E +# PLAN — Milestone v0.2 Phase 5 (FINAL): Review + Audit + Ship --- -phase: 4 +phase: 5 milestone: v0.2 -status: planned +status: executed must_haves: - - "Adversarial suite (REQ-32): hallucinated-URL writer caught by the verifier (report cites a URL never visited/anchored → flagged in ledger, export gate fails it); quote-anchor mismatch detection (near-miss and fabricated quotes flagged, never silently accepted); pause/resume mid-depth-loop over the HTTP API" - - "E2E happy path (REQ-17): new user → research running in ≤3 clicks using defaults only, validated against the REAL stack (FastAPI + engine via ASGI, live ApiClient binding at the boundary) — not the mock" - - "Export round-trip (REQ-34): report + ledger + citations survive as real links — markdown export carries citation URLs, JSON export round-trips into pydantic models" - - "Live-adapter smoke test tier (opt-in, G-004): keyed tests for Ollama + Tavily marked skippable when keys absent — zero-key CI unaffected" - - "Monorepo tooling polish (REQ-33 completion): one-command install/dev documented + root convenience scripts" + - "Multi-persona full-milestone review: 4 P0 + 15 P1 + 15 P2 found; all 4 P0s fixed in-phase with attack-proofs (A1 tautological anchoring → real writer-quote extraction; C2 gate denominator → cited-URL scoped; C1 zombie runs → shape-validated plans + catch-all FAILED persist; S1 fetch SSRF → per-hop re-guarding); 6 cheap P1s fixed (approve-with-plan, FAILED status, length caps, probe wiring, README env-var + adapter honesty)" + - "Audit: PASS-WITH-NOTES — reconstruction PASS (all REQ-01..34 covered across v0.2.0–v0.2.4), secrets clean (zero key values in history), branch hygiene exact; critical notes fixed in-phase (REQUIREMENTS traceability un-staled, P5 PLAN written, ROADMAP P5 status, security-engineer persona deactivated per its removal note)" + - "Milestone ship: v0.2.5 tagged as the MVP release candidate — the pipeline then HALTS at the merge_to_main owner approval gate (D-018)" + - "G-009 escalation presented at the gate: the owner ratifies that live mode covers the core research loop only (G-002); remaining prototype surfaces stay mock-backed to v0.3+" personas: - - backend-engineer - - frontend-engineer - ci-lead-developer -requirements: [REQ-17, REQ-32, REQ-34] +requirements: [REQ-01, REQ-02, REQ-03, REQ-04, REQ-05, REQ-06, REQ-07, REQ-08, REQ-09, REQ-10, REQ-11, REQ-12, REQ-13, REQ-14, REQ-15, REQ-16, REQ-17, REQ-18, REQ-19, REQ-20, REQ-21, REQ-22, REQ-23, REQ-24, REQ-25, REQ-26, REQ-31, REQ-32, REQ-33, REQ-34] --- ## User-Facing Surface -The adversarial suite is the deliverable: every trust-loop claim is now attacked by tests that try to fool it. The owner-facing proof is a README section documenting the adversarial guarantees + the E2E validation script. No UI changes. +The MVP release candidate: the approved v0.1 prototype frontend (Ask → Choose → Plan & Run → Reading Room, all 8 cards, shell, library, settings) now backed by a real FastAPI research engine in live mode — methodology registry (Rapid Answer + Deep Report), 12-dial framework, asyncio state machine with durable pause/cancel/resume, visited-only evidence store, attack-proven trust loop (quote anchoring, SSRF-guarded liveness, citation ledger, ≥99% export gate with zero-unanchored requirement), SSE live trace, adversarial suite (14 attacks), E2E happy path on the real stack. Releases v0.2.0–v0.2.5. -## Happy Path (P4 — written before execute) +## Happy Path (owner review script) -1. `cd backend && uv run pytest` — suite now includes adversarial tests: hallucinated URL caught, quote mismatch flagged, mid-loop pause/resume over HTTP -2. `uv run pytest -m live` (keys present) or default skip — live smoke tier -3. Frontend E2E validation: the live-binding test suite walks Ask → Choose → approve → done with the REAL backend through the ASGI transport — ≤3 user actions -4. Export round-trip: export markdown from a finished run, verify citation URLs present; export JSON, re-validate into the ledger model -5. All gates green: backend ruff/mypy/pytest, frontend typecheck/lint/test/build +1. `cd backend && uv sync && uv run pytest` — 129 tests green, zero API keys (live tier: `uv run pytest -m live`, skips cleanly) +2. `cd frontend && corepack pnpm install && corepack pnpm test` — 97 tests green (90 component + 5 live-binding + 2 real-stack E2E) +3. `uv run uvicorn researchx.api.app:app --port 8000` + `NEXT_PUBLIC_RESEARCHX_API_MODE=live corepack pnpm dev` → http://localhost:3000 → Ask a question → Research this → pick Deep Report → Start research → approve the plan (3 clicks) → live SSE trace → report with anchored citations → ledger → export Markdown (citation links) / JSON (backend-schema round-trip) +4. `uv run python -m researchx.engine.demo` — full pipeline on fakes, artifacts in a temp runs dir +5. Owner decision: approve → milestone merge to main + v0.2.5 published as the MVP release; or direct changes → binding fix cycle -## UX Acceptance Criteria (P4) +## UX Acceptance Criteria -- Trust claims are adversarially proven, not asserted: the suite demonstrates attacks being caught -- The 3-click criterion is measured on the real stack, defaults only -- Zero-key test discipline holds; live tier is opt-in and never blocks CI +- ≤3 clicks from cold start to initiated research, validated on the real stack (REQ-17) +- Every trust claim attack-proven: hallucinated citations fail the export gate; fabricated quotes on real passage IDs flag unanchored; SSRF redirects die; pause/resume/cancel survive the loop +- Honest scope: live mode = core research loop (G-002); library/settings/suggestions/refine stay mock-backed until v0.3+ — ratified at this gate (G-009) +- No secret value in any artifact, trace, or log; zero-key test discipline throughout -## Tasks (waves) +## What this phase did -### Wave 1 — Adversarial suite (backend-engineer) -- **T-4.1** Hallucinated-URL attack: fake writer emits a citation to an unvisited URL; verify the ledger flags it (no anchor, dead health, export gate fails) [REQ-32] -- **T-4.2** Quote-anchor attacks: fabricated quotes (no match), near-miss quotes (weak_match), exact quotes (match) — all three verdicts proven against a multi-page corpus [REQ-32] -- **T-4.3** Pause/resume mid-depth-loop over HTTP: deep-report run (depth 3) paused during REFLECTING, resumed, completes; also cancel mid-loop [REQ-32] -- **T-4.4** Liveness attack surface: private-IP redirect (must die), redirect chain to dead URL, oversized response cap +1. **Review** (multi-persona, probe-verified): 4 P0 + 15 P1 + 15 P2. The 4 P0s broke the milestone's flagship guarantees and were all fixed with attack-proofs: quote anchoring extracted the stored passage's own quote (always 1.0) instead of the writer's text; the export-gate denominator counted unprobed visited pages as dead; malformed planner JSON left zombie runs; fetch adapters followed redirects without re-guarding. Six cheap P1s also fixed. +2. **Audit**: PASS-WITH-NOTES — reconstruction, secrets sweep (clean), branch/tag hygiene exact; both critical notes (stale traceability rows, missing P5 plan) fixed in-phase. +3. **Ship (this commit sequence)**: PR #14 (phase/05 → milestone), squash-merge, tag v0.2.5 — the MVP release candidate, release with the full milestone summary. Per D-018 and the merge_to_main escalation hook, the milestone merge to main does NOT proceed autonomously — the pipeline halts for the recorded owner decision, with G-009 (live-mode scope) presented at the same gate. -### Wave 2 — E2E + round-trip (frontend-engineer + backend-engineer) -- **T-4.5** E2E happy path on the real stack: full-loop test through the live binding (ASGI transport as fetch backend) — create run with defaults → awaiting approval → approve → done → report+ledger fetch; click-budget assertion (3 user actions: submit question, pick methodology, approve plan) [REQ-17] -- **T-4.6** Export round-trip: markdown export contains citation URLs as links; JSON export validates back into the ledger model [REQ-34] +## Scope Guard -### Wave 3 — Live tier + polish (ci-lead-developer) -- **T-4.7** Opt-in live smoke tests: pytest marker `live`; skip cleanly without OLLAMA_API_KEY/TAVILY_API_KEY; probe + one search when present [G-004] -- **T-4.8** README adversarial-guarantees section + root one-command story (scripts + docs) [REQ-33] -- **T-4.9** REQ mapping table for verify: REQ-17/32/34 → tests - -## Out of scope for P4 -- Any UI change; new API routes; methodology additions (G-010 lever untouched — Deep Report ships in this milestone) - -## Verification hooks -- Gates: backend ruff/mypy/pytest (adversarial suite included, zero-key); frontend typecheck/lint/test/build + live-binding E2E -- REQ coverage: REQ-17 (E2E 3-click), REQ-32 (adversarial), REQ-34 (round-trip) -- Security: attacks stay in tests; no live endpoints hit without the marker \ No newline at end of file +Milestone merge to main, v0.2.5 publication, and any deploy wait on the owner approval gate (D-018/G-009). The pipeline does not proceed autonomously past this commit. \ No newline at end of file diff --git a/.ciagent/REQUIREMENTS.md b/.ciagent/REQUIREMENTS.md index 6046b57..7ae5641 100644 --- a/.ciagent/REQUIREMENTS.md +++ b/.ciagent/REQUIREMENTS.md @@ -130,8 +130,8 @@ | REQ-63, REQ-64, REQ-65, REQ-66 (app shell, library, settings, avatar) | v0.1 | P5 | complete | | REQ-67, REQ-68 (ask suggestions, refine dialogue) | v0.1 | P7 | complete | | REQ-01..06, REQ-08..11, REQ-31 (framework, adapters) | v0.2 | P1 | complete (v0.2.1; REQ-07/12 contracts proven, live probing in P2) | -| REQ-07, REQ-12, REQ-13..24 (probing, engine, trust) | v0.2 | P2 | blocked | -| REQ-25, REQ-26, REQ-33 (API, client, tooling) | v0.2 | P3 | blocked | -| REQ-17, REQ-32, REQ-34 (adversarial, E2E, round-trip) | v0.2 | P4 | blocked | -| All REQ-01..34 (milestone gate) | v0.2 | P5 | blocked | +| REQ-07, REQ-12, REQ-13..24 (probing, engine, trust) | v0.2 | P2 | complete (v0.2.2) | +| REQ-25, REQ-26, REQ-33 (API, client, tooling) | v0.2 | P3 | complete (v0.2.3) | +| REQ-17, REQ-32, REQ-34 (adversarial, E2E, round-trip) | v0.2 | P4 | complete (v0.2.4) | +| All REQ-01..34 (milestone gate) | v0.2 | P5 | complete (v0.2.5 — all requirements covered; owner gate pending per D-018) | | REQ-35..51 | v2 backlog (post-MVP milestones) | — | pending | diff --git a/.ciagent/ROADMAP.md b/.ciagent/ROADMAP.md index 3663b35..b583cd5 100644 --- a/.ciagent/ROADMAP.md +++ b/.ciagent/ROADMAP.md @@ -110,7 +110,7 @@ Phases renumbered from the original v1.0 plan (1-based within the milestone). Ta ### Phase 5 — Final Review + Ship (FINAL) - **Description**: Review + audit + milestone ship; final patch tag IS the MVP release. MAJOR stays 0 — the MVP ships as a 0.x release. v1.0.0 comes only after an owner-approved release decision. -- **Status**: blocked → next (sequential phases; unblock as each prior phase ships) +- **Status**: in progress (final phase — review complete, audit complete, ship v0.2.5 next; milestone merge HALTS at the owner approval gate per D-018) ## Milestone Releases - **v0.1.4 = extended-prototype candidate** (superseded by the P5/P6 extension cycle) diff --git a/README.md b/README.md index aa43c5e..34a4c24 100644 --- a/README.md +++ b/README.md @@ -84,11 +84,13 @@ The test suite needs **no** keys: fake model/search adapters (REQ-31) drive ever The frontend runs mock-backed by default (works standalone, zero keys). To run the core research loop against the real backend: ```bash -# terminal 1 — backend (fake adapters by default; live adapters via env-file keys) +# terminal 1 — backend (fake adapters by default; real Ollama/Tavily adapters +# are used by the engine in v0.2.5+ wiring — the served API currently ships +# fake-adapter runs for demos and tests; see Adapter configuration below) cd backend && uv run uvicorn researchx.api.app:app --reload --port 8000 # terminal 2 — frontend in live mode -cd frontend && RESEARCHX_API_MODE=live corepack pnpm dev +cd frontend && NEXT_PUBLIC_RESEARCHX_API_MODE=live corepack pnpm dev ``` Live mode covers the core loop (Ask → Choose → approve plan → live trace → report + ledger); the library, settings persistence, suggestions, refine, and graph surfaces stay mock-backed until v0.3. The API surface: `POST /api/runs`, `GET /api/methodologies`, run lifecycle (`approve/pause/cancel/resume`), `GET /api/runs/{id}/events` (SSE), `GET /api/config`, report + ledger artifacts. diff --git a/backend/src/researchx/adapters/capability_probe.py b/backend/src/researchx/adapters/capability_probe.py index 8fa7c60..cbc9edd 100644 --- a/backend/src/researchx/adapters/capability_probe.py +++ b/backend/src/researchx/adapters/capability_probe.py @@ -3,7 +3,6 @@ import json import httpx from researchx.adapters.model import ModelAdapter -from researchx.adapters.ollama import OllamaModelAdapter from researchx.adapters.probe import ( DEFAULT_EMBEDDING_BATCH, CapabilityProbe, @@ -13,19 +12,25 @@ from researchx.adapters.probe import ( from researchx.adapters.types import ModelMessage -async def probe_model_adapter(adapter: OllamaModelAdapter) -> CapabilityProbe: +def supports_probe(adapter: ModelAdapter) -> bool: + return getattr(adapter, "probing_supported", False) + + +async def probe_model_adapter(adapter: ModelAdapter) -> CapabilityProbe: models = await adapter.list_models() smoke_ok = await _structured_output_smoke_test(adapter) writer = adapter.roles().writer warning = structured_output_warning(smoke_ok, writer) + endpoint = getattr(adapter, "active_base_url", "unknown") + fallback_reason = getattr(adapter, "last_fallback_reason", None) return CapabilityProbe( - endpoint=adapter.active_base_url, + endpoint=str(endpoint), models=models, context_window=cap_context_window(None), embedding_batch=DEFAULT_EMBEDDING_BATCH, structured_outputs_supported=smoke_ok, structured_output_warning=warning, - fallback_reason=adapter.last_fallback_reason, + fallback_reason=fallback_reason, ) diff --git a/backend/src/researchx/adapters/duckduckgo.py b/backend/src/researchx/adapters/duckduckgo.py index f998f22..a8655e0 100644 --- a/backend/src/researchx/adapters/duckduckgo.py +++ b/backend/src/researchx/adapters/duckduckgo.py @@ -13,6 +13,7 @@ from researchx.adapters.url_guard import url_allowed _DDG_URL = "https://html.duckduckgo.com/html" _SEARCH_TIMEOUT = 10.0 _FETCH_TIMEOUT = 10.0 +_MAX_FETCH_REDIRECTS = 5 _BACKOFF_SECONDS = 1.0 _USER_AGENT = "researchx/0.2 (local research studio)" _RESULT_RE = re.compile( @@ -93,21 +94,34 @@ class DuckDuckGoSearchAdapter: return re.sub(r"\s+", " ", text).strip() async def fetch(self, url: str) -> PageContent: - if not url_allowed(url): - raise AdapterError("fetch refused: url not allowed (private or non-http scheme)") + from urllib.parse import urljoin + + current = url async with httpx.AsyncClient( timeout=_FETCH_TIMEOUT, - follow_redirects=True, + follow_redirects=False, headers={"User-Agent": _USER_AGENT}, ) as client: - response = await client.get(url) - response.raise_for_status() - return PageContent( - url=url, - title=extract_title(response.text), - text=extract_text(response.text), - fetched_at=datetime.now(UTC), - ) + for _ in range(_MAX_FETCH_REDIRECTS + 1): + if not url_allowed(current): + raise AdapterError( + "fetch refused: url not allowed (private or non-http scheme)" + ) + response = await client.get(current) + if 300 <= response.status_code < 400: + location = response.headers.get("location") + if not location: + raise AdapterError("fetch failed: redirect without location") + current = urljoin(current, location) + continue + response.raise_for_status() + return PageContent( + url=url, + title=extract_title(response.text), + text=extract_text(response.text), + fetched_at=datetime.now(UTC), + ) + raise AdapterError("fetch failed: too many redirects") @property def last_degradation(self) -> str | None: diff --git a/backend/src/researchx/adapters/ollama.py b/backend/src/researchx/adapters/ollama.py index 797c078..5558af1 100644 --- a/backend/src/researchx/adapters/ollama.py +++ b/backend/src/researchx/adapters/ollama.py @@ -12,6 +12,8 @@ _REQUEST_TIMEOUT = 60.0 class OllamaModelAdapter: + probing_supported = True + def __init__( self, secrets: EnvFileSecretsProvider, diff --git a/backend/src/researchx/adapters/tavily.py b/backend/src/researchx/adapters/tavily.py index 5486321..316e2db 100644 --- a/backend/src/researchx/adapters/tavily.py +++ b/backend/src/researchx/adapters/tavily.py @@ -10,6 +10,7 @@ from researchx.adapters.url_guard import url_allowed _TAVILY_URL = "https://api.tavily.com/search" _SEARCH_TIMEOUT = 10.0 _FETCH_TIMEOUT = 10.0 +_MAX_FETCH_REDIRECTS = 5 _USER_AGENT = "researchx/0.2 (local research studio)" _TITLE_RE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) _BLOCK_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) @@ -56,18 +57,31 @@ class TavilySearchAdapter: ][:max_results] async def fetch(self, url: str) -> PageContent: - if not url_allowed(url): - raise AdapterError("fetch refused: url not allowed (private or non-http scheme)") + from urllib.parse import urljoin + + current = url async with httpx.AsyncClient( timeout=_FETCH_TIMEOUT, - follow_redirects=True, + follow_redirects=False, headers={"User-Agent": _USER_AGENT}, ) as client: - response = await client.get(url) - response.raise_for_status() - return PageContent( - url=url, - title=extract_title(response.text), - text=extract_text(response.text), - fetched_at=datetime.now(UTC), - ) \ No newline at end of file + for _ in range(_MAX_FETCH_REDIRECTS + 1): + if not url_allowed(current): + raise AdapterError( + "fetch refused: url not allowed (private or non-http scheme)" + ) + response = await client.get(current) + if 300 <= response.status_code < 400: + location = response.headers.get("location") + if not location: + raise AdapterError("fetch failed: redirect without location") + current = urljoin(current, location) + continue + response.raise_for_status() + return PageContent( + url=url, + title=extract_title(response.text), + text=extract_text(response.text), + fetched_at=datetime.now(UTC), + ) + raise AdapterError("fetch failed: too many redirects") \ No newline at end of file diff --git a/backend/src/researchx/api/app.py b/backend/src/researchx/api/app.py index ae38173..ffc4271 100644 --- a/backend/src/researchx/api/app.py +++ b/backend/src/researchx/api/app.py @@ -8,12 +8,13 @@ from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel, Field, ValidationError from sse_starlette.sse import EventSourceResponse +from researchx.adapters.capability_probe import probe_model_adapter, supports_probe from researchx.adapters.fakes import CORPUS, FakeModelAdapter, FakeSearchAdapter from researchx.adapters.local_storage import LocalStorageAdapter from researchx.adapters.types import AdapterError, ChatResult from researchx.engine.actor import RunActor, make_run_config from researchx.engine.events import EventBus, RunEvent -from researchx.engine.run_state import RunState +from researchx.engine.run_state import Plan, RunState from researchx.engine.states import RunStateName from researchx.methodologies import registry @@ -28,8 +29,8 @@ _CANNED_WRITER = ( class CreateRunBody(BaseModel): - question: str = Field(min_length=1) - context: str = "" + question: str = Field(min_length=1, max_length=2000) + context: str = Field(default="", max_length=8000) methodology: str dial_overrides: dict[str, Any] = Field(default_factory=dict) @@ -76,7 +77,7 @@ _STATUS_MAP = { RunStateName.VERIFYING: "verifying", RunStateName.DONE: "done", RunStateName.CANCELLED: "cancelled", - RunStateName.FAILED: "cancelled", + RunStateName.FAILED: "failed", } _ETA_TABLE = {"rapid-answer": (2, "low"), "deep-report": (12, "medium")} @@ -170,11 +171,15 @@ def create_app( @app.get("/api/config") async def config_status() -> ConfigStatus: + latest_probe = None + for run_id in list(runs._actors)[-1:]: + actor_probe = runs.actor(run_id).state.probe + latest_probe = actor_probe return ConfigStatus( model_adapter=type(model).__name__, search_adapter=type(search).__name__, storage_dir=str(storage.run_dir("00000000").parent), - probe=None, + probe=latest_probe, cost_counters=runs.cost_counters, ) @@ -222,6 +227,15 @@ def create_app( registry=registry, event_bus=bus, ) + probe_data: dict[str, Any] | None = None + if supports_probe(model): + try: + probe = await probe_model_adapter(model) + probe_data = probe.model_dump(mode="json") + await actor.attach_probe(probe_data) + except AdapterError: + probe_data = None + task = asyncio.create_task(actor.run()) runs.add(run_id, actor, task, bus) @@ -244,7 +258,13 @@ def create_app( if not runs.has(run_id): raise HTTPException(status_code=404, detail="run not found") actor = runs.actor(run_id) - await actor.approve() + edited_plan: Plan | None = None + if body and body.plan: + try: + edited_plan = Plan.model_validate(body.plan) + except ValidationError: + raise HTTPException(status_code=422, detail="invalid plan") + await actor.approve(edited_plan) return _summary(run_id) @app.post("/api/runs/{run_id}/pause") diff --git a/backend/src/researchx/engine/actor.py b/backend/src/researchx/engine/actor.py index b67c9bd..7b54f9e 100644 --- a/backend/src/researchx/engine/actor.py +++ b/backend/src/researchx/engine/actor.py @@ -112,6 +112,15 @@ class RunActor: except AdapterError as exc: await self._fail(str(exc)) raise EngineError(str(exc)) from exc + except httpx.HTTPError as exc: + await self._fail(f"network error ({type(exc).__name__})") + raise EngineError(f"network error ({type(exc).__name__})") from exc + except EngineError: + await self._fail("engine error") + raise + except Exception as exc: + await self._fail(f"unexpected engine failure ({type(exc).__name__})") + raise EngineError(f"unexpected engine failure ({type(exc).__name__})") from exc return self.state async def attach_probe(self, probe_data: dict[str, Any]) -> None: @@ -177,10 +186,21 @@ class RunActor: data = json.loads(raw) except json.JSONDecodeError as exc: raise AdapterError("planner returned non-JSON plan") from exc + if not isinstance(data, dict): + raise AdapterError("planner plan is not a JSON object") + sub_questions = data.get("sub_questions", []) + source_universe = data.get("source_universe", []) + outline = data.get("outline", []) + if not ( + isinstance(sub_questions, list) + and isinstance(source_universe, list) + and isinstance(outline, list) + ): + raise AdapterError("planner plan has malformed lists") return Plan( - sub_questions=[str(q) for q in data.get("sub_questions", [])], - source_universe=[str(u) for u in data.get("source_universe", [])], - outline=[str(o) for o in data.get("outline", [])], + sub_questions=[str(q) for q in sub_questions if isinstance(q, str)], + source_universe=[str(u) for u in source_universe if isinstance(u, str)], + outline=[str(o) for o in outline if isinstance(o, str)], ) async def _approval_phase(self) -> None: diff --git a/backend/src/researchx/trust/anchoring.py b/backend/src/researchx/trust/anchoring.py index 7059749..c2ae8e3 100644 --- a/backend/src/researchx/trust/anchoring.py +++ b/backend/src/researchx/trust/anchoring.py @@ -49,17 +49,41 @@ def anchor_quote(quote: str, passages: list[Passage]) -> AnchorVerdict: ) -def extract_quotes_from_report(report_text: str, evidence: EvidenceStore) -> list[str]: - marker_re = re.compile(r"\[p-(\d+)\]") - quotes: list[str] = [] - for match in marker_re.finditer(report_text): +_MARKER_RE = re.compile(r"\[p-(\d+)\]") +_HEADER_RE = re.compile(r"^#{1,6}\s.*$", re.MULTILINE) + + +def _claim_fragment(before: str) -> str: + cleaned = _HEADER_RE.sub("", before).rstrip() + fragment_start = max( + cleaned.rfind("."), cleaned.rfind("!"), cleaned.rfind("?"), cleaned.rfind("\n") + ) + quote_text = cleaned[fragment_start + 1 :].strip(" \t>\"'*-") + if not quote_text: + quote_text = cleaned[-200:].strip(" \t>\"'*-") + return quote_text + + +def extract_report_claims(report_text: str) -> list[tuple[str, str]]: + """Extract (passage_id, quote_text) claims from the report. + + The quote text is what the WRITER actually wrote adjacent to the + citation marker — the last sentence fragment before each [p-N] + marker — NOT the stored passage's own text. This is the adversarial + core of REQ-20: a fabricated quote riding a real passage ID must + be visible to the anchorer. + """ + claims: list[tuple[str, str]] = [] + for match in _MARKER_RE.finditer(report_text): passage_id = f"p-{match.group(1)}" - passage = evidence.get(passage_id) - if passage: - quotes.append(passage.quote) - return quotes + quote_text = _claim_fragment(report_text[: match.start()]) + claims.append((passage_id, quote_text)) + return claims + + +def extract_quotes_from_report(report_text: str, evidence: EvidenceStore) -> list[str]: + return [quote for _pid, quote in extract_report_claims(report_text) if evidence.get(_pid)] def extract_cited_passage_ids(report_text: str) -> list[str]: - marker_re = re.compile(r"\[p-(\d+)\]") - return [f"p-{m.group(1)}" for m in marker_re.finditer(report_text)] \ No newline at end of file + return [f"p-{m.group(1)}" for m in _MARKER_RE.finditer(report_text)] \ No newline at end of file diff --git a/backend/src/researchx/trust/ledger.py b/backend/src/researchx/trust/ledger.py index 4d0cd56..9944cff 100644 --- a/backend/src/researchx/trust/ledger.py +++ b/backend/src/researchx/trust/ledger.py @@ -7,7 +7,7 @@ from researchx.evidence.store import EvidenceStore, Passage from researchx.trust.anchoring import ( anchor_quote, extract_cited_passage_ids, - extract_quotes_from_report, + extract_report_claims, normalize, ) from researchx.trust.liveness import LivenessStatus @@ -80,17 +80,22 @@ def build_ledger( ) -> CitationLedger: claims = _claims_per_passage(report_text) all_passages = evidence.writer_context() - report_quotes = extract_quotes_from_report(report_text, evidence) + report_claims = extract_report_claims(report_text) cited_ids = extract_cited_passage_ids(report_text) known_ids = {p.passage_id for p in all_passages} + cited_urls: list[str] = [] entries: list[CitationEntry] = [] anchored = 0 unanchored = 0 weak = 0 confidence: dict[str, str] = {} - for quote in report_quotes: - verdict = anchor_quote(quote, all_passages) + for passage_id, writer_quote in report_claims: + passage = evidence.get(passage_id) + if passage is None: + unanchored += 1 + continue + verdict = anchor_quote(writer_quote, [passage]) if verdict.matched: anchored += 1 elif verdict.weak: @@ -101,8 +106,13 @@ def build_ledger( for cited_id in cited_ids: if cited_id not in known_ids: unanchored += 1 + passage = evidence.get(cited_id) + if passage is not None and passage.url not in cited_urls: + cited_urls.append(passage.url) for page in evidence.pages(): + if page.url not in cited_urls: + continue passages = evidence.passages_for_url(page.url) passage_ids = [p.passage_id for p in passages] page_claims = sum(claims.get(pid, 0) for pid in passage_ids) diff --git a/backend/tests/test_adversarial.py b/backend/tests/test_adversarial.py index d61ca5d..69e7b55 100644 --- a/backend/tests/test_adversarial.py +++ b/backend/tests/test_adversarial.py @@ -12,7 +12,7 @@ from pathlib import Path import httpx from researchx.adapters.fakes import CORPUS, FakeModelAdapter, FakeSearchAdapter -from researchx.adapters.types import ChatResult +from researchx.adapters.types import AdapterError, ChatResult from researchx.api.app import create_app from researchx.evidence.store import EvidenceStore from researchx.trust.anchoring import anchor_quote @@ -261,8 +261,9 @@ def test_export_round_trip_json_validates_into_backend_model() -> None: from researchx.trust.ledger import CitationLedger store = EvidenceStore() - store.add_passage("https://example.org/a", "A passage with real content " * 6) - report = "# Report\n\nClaim [p-1].\n" + stored_text = "A passage with real content for the round-trip proof." + store.add_passage("https://example.org/a", stored_text) + report = f"# Report\n\n{stored_text} [p-1].\n" ledger = build_ledger( run_id="attack003", report_text=report, @@ -276,3 +277,83 @@ def test_export_round_trip_json_validates_into_backend_model() -> None: checklist_md = export_checklist(ledger) assert "Anchoring gate" in checklist_md assert "PASS" in checklist_md + + +def test_attack_fabricated_quote_on_real_passage_id_fails() -> None: + store = EvidenceStore() + stored = store.add_passage( + "https://example.org/real", + "The genuine finding about verification pipelines is that anchoring works. " * 3, + ) + report = ( + "# Report\n\n" + f"THIS FABRICATED QUOTE WAS NEVER STORED ANYWHERE AT ALL [{stored.passage_id}]\n" + ) + ledger = build_ledger( + run_id="attack004", + report_text=report, + evidence=store, + liveness={"https://example.org/real": LivenessStatus.LIVE}, + visited_at=datetime.now(UTC), + ) + assert ledger.checklist.unanchored_claims >= 1, ( + "a fabricated quote riding a real passage ID must be flagged (REQ-20)" + ) + passed, _ = liveness_export_gate(ledger) + assert passed is False + + +def test_attack_real_verbatim_quote_on_real_passage_anchors() -> None: + store = EvidenceStore() + stored_text = ( + "The genuine finding about verification pipelines is that anchoring works." + ) + stored = store.add_passage("https://example.org/real", stored_text + " " + stored_text) + report = f"# Report\n\n{stored_text} [{stored.passage_id}]\n" + ledger = build_ledger( + run_id="attack005", + report_text=report, + evidence=store, + liveness={"https://example.org/real": LivenessStatus.LIVE}, + visited_at=datetime.now(UTC), + ) + assert ledger.checklist.anchored_claims == 1 + assert ledger.checklist.unanchored_claims == 0 + passed, _ = liveness_export_gate(ledger) + assert passed is True + + +async def test_attack_search_result_redirecting_to_private_ip_refused(monkeypatch) -> None: + import tempfile + from pathlib import Path as PathMod + + import httpx as httpx_mod + + from researchx.adapters.secrets import EnvFileSecretsProvider + from researchx.adapters.tavily import TavilySearchAdapter + + with tempfile.TemporaryDirectory() as tmpdir: + env_file = PathMod(tmpdir) / ".env.secrets" + env_file.write_text("TAVILY_API_KEY=t-key\n", encoding="utf-8") + secrets = EnvFileSecretsProvider(paths=[env_file]) + adapter = TavilySearchAdapter(secrets) + + def handler(request: httpx_mod.Request) -> httpx_mod.Response: + if "boobytrap" in str(request.url): + return httpx_mod.Response( + 302, headers={"location": "http://169.254.169.254/latest/meta-data"} + ) + return httpx_mod.Response(200, text="ok") + + transport = httpx_mod.MockTransport(handler) + original = httpx_mod.AsyncClient + monkeypatch.setattr( + httpx_mod, "AsyncClient", lambda **kw: original(transport=transport, **kw) + ) + + try: + await adapter.fetch("https://example.org/boobytrap") + except AdapterError as exc: + assert "refused" in str(exc) or "not allowed" in str(exc) + else: + raise AssertionError("fetch of a private-redirecting URL must be refused") diff --git a/backend/tests/test_p0_regressions.py b/backend/tests/test_p0_regressions.py index fd95b99..b225b71 100644 --- a/backend/tests/test_p0_regressions.py +++ b/backend/tests/test_p0_regressions.py @@ -190,10 +190,11 @@ def test_ledger_export_gate() -> None: from researchx.trust.liveness import LivenessStatus store = EvidenceStore() - store.add_passage("https://a", "passage one " * 20) + stored_text = "The verified sentence used for the export gate test." + store.add_passage("https://a", stored_text) ledger = build_ledger( run_id="abcd2345", - report_text="[p-1]", + report_text=f"{stored_text} [p-1]", evidence=store, liveness={"https://a": LivenessStatus.LIVE}, visited_at=datetime.now(UTC), @@ -204,7 +205,7 @@ def test_ledger_export_gate() -> None: ledger_dead = build_ledger( run_id="abcd2345", - report_text="[p-1]", + report_text=f"{stored_text} [p-1]", evidence=store, liveness={"https://a": LivenessStatus.DEAD}, visited_at=datetime.now(UTC), diff --git a/backend/tests/test_trust.py b/backend/tests/test_trust.py index e49e431..2f34fd1 100644 --- a/backend/tests/test_trust.py +++ b/backend/tests/test_trust.py @@ -97,7 +97,10 @@ def _store_with_report() -> tuple[EvidenceStore, str]: store = EvidenceStore() for page in CORPUS[:3]: store.add_page(page.url, page.title, page.text) - report = f"# Report\n\nClaim one [{store.pages()[0].passage_ids[0]}] and claim two [p-1].\n" + first_passage = store.pages()[0].passage_ids[0] + stored = store.get(first_passage) + sentence = (stored.text if stored else "").split(".")[0] + "." + report = f"# Report\n\n{sentence} [{first_passage}]\n\nMore context [{first_passage}]\n" return store, report @@ -121,9 +124,10 @@ async def test_ledger_build_and_checklist_export(monkeypatch) -> None: visited_at=datetime.now(UTC), ) assert ledger.run_id == "abcd2345" - assert len(ledger.entries) == len(store.pages()) - assert ledger.checklist.citations_checked == len(store.pages()) - assert ledger.checklist.live == len(store.pages()) + cited_url = store.pages()[0].url + assert [e.url for e in ledger.entries] == [cited_url] + assert ledger.checklist.citations_checked == 1 + assert ledger.checklist.live == 1 assert ledger.checklist.dead == 0 assert ledger.checklist.anchored_claims >= 1 assert all(