From 7f7868f7cfa88f660289f2624bc2aba2fd1cb203 Mon Sep 17 00:00:00 2001 From: coreci Date: Sat, 10 Oct 2026 15:30:14 +0000 Subject: [PATCH] docs(milestone): complete coreci-broker ---ci--- phase: 5 milestone: v0.1 status: complete requirements: covered: [REQ-01, REQ-02, REQ-03, REQ-04, REQ-05, REQ-06, REQ-07, REQ-08, REQ-09, REQ-10, REQ-11, REQ-12, REQ-13, REQ-14, REQ-15, REQ-16, REQ-17, REQ-18, REQ-19, REQ-20] partial: [] ---/ci--- --- .ciagent/ARCHITECTURE.md | 48 + .ciagent/CHECKPOINT.json | 8 + .ciagent/GRILL.md | 129 ++ .ciagent/PERSONAS.md | 60 + .ciagent/PLAN.md | 158 ++ .ciagent/PROJECT.md | 72 + .ciagent/REQUIREMENTS.md | 99 ++ .ciagent/RESEARCH.md | 82 + .ciagent/ROADMAP.md | 60 + .ciagent/config.json | 54 + .gitignore | 13 + README.md | 180 +++ coreci_broker.py | 1390 +++++++++++++++++ coreci_broker_client.py | 161 ++ coreci_broker_ui/app_registrations.html | 271 ++++ coreci_broker_ui/audit.html | 280 ++++ coreci_broker_ui/dashboard.html | 64 + coreci_broker_ui/index.html | 13 + coreci_broker_ui/login.html | 78 + coreci_broker_ui/nav.js | 84 + coreci_broker_ui/shared.css | 369 +++++ coreci_broker_ui/user_allowlist.html | 271 ++++ reference_tools/README.md | 25 + reference_tools/minimal_auth_showcase.py | 45 + reference_tools/proxmox_readonly_reference.py | 104 ++ tests/test_acceptance_full.py | 345 ++++ tests/test_acceptance_p1.py | 114 ++ tests/test_acceptance_p2.py | 241 +++ tests/test_acceptance_p3.py | 309 ++++ tests/test_acceptance_p4.py | 290 ++++ tests/test_admin_auth.py | 251 +++ tests/test_agent_meta.py | 134 ++ tests/test_allowlist.py | 177 +++ tests/test_app_auth.py | 263 ++++ tests/test_apps.py | 187 +++ tests/test_audit.py | 213 +++ tests/test_audit_query.py | 305 ++++ tests/test_auth_audit.py | 188 +++ tests/test_client.py | 308 ++++ tests/test_config.py | 105 ++ tests/test_creds_source.py | 312 ++++ tests/test_hardening.py | 162 ++ tests/test_hasher.py | 207 +++ tests/test_mounts.py | 148 ++ tests/test_p3_remediation.py | 249 +++ tests/test_p5_remediation.py | 207 +++ tests/test_reference_tools.py | 271 ++++ tests/test_register.py | 347 ++++ tests/test_router.py | 242 +++ tests/test_secrets.py | 233 +++ tests/test_state.py | 318 ++++ tests/test_static_disclosure.py | 181 +++ tests/test_store.py | 155 ++ tests/test_token.py | 182 +++ 54 files changed, 10762 insertions(+) create mode 100644 .ciagent/ARCHITECTURE.md create mode 100644 .ciagent/CHECKPOINT.json create mode 100644 .ciagent/GRILL.md create mode 100644 .ciagent/PERSONAS.md create mode 100644 .ciagent/PLAN.md create mode 100644 .ciagent/PROJECT.md create mode 100644 .ciagent/REQUIREMENTS.md create mode 100644 .ciagent/RESEARCH.md create mode 100644 .ciagent/ROADMAP.md create mode 100644 .ciagent/config.json create mode 100644 .gitignore create mode 100644 README.md create mode 100644 coreci_broker.py create mode 100644 coreci_broker_client.py create mode 100644 coreci_broker_ui/app_registrations.html create mode 100644 coreci_broker_ui/audit.html create mode 100644 coreci_broker_ui/dashboard.html create mode 100644 coreci_broker_ui/index.html create mode 100644 coreci_broker_ui/login.html create mode 100644 coreci_broker_ui/nav.js create mode 100644 coreci_broker_ui/shared.css create mode 100644 coreci_broker_ui/user_allowlist.html create mode 100644 reference_tools/README.md create mode 100644 reference_tools/minimal_auth_showcase.py create mode 100644 reference_tools/proxmox_readonly_reference.py create mode 100644 tests/test_acceptance_full.py create mode 100644 tests/test_acceptance_p1.py create mode 100644 tests/test_acceptance_p2.py create mode 100644 tests/test_acceptance_p3.py create mode 100644 tests/test_acceptance_p4.py create mode 100644 tests/test_admin_auth.py create mode 100644 tests/test_agent_meta.py create mode 100644 tests/test_allowlist.py create mode 100644 tests/test_app_auth.py create mode 100644 tests/test_apps.py create mode 100644 tests/test_audit.py create mode 100644 tests/test_audit_query.py create mode 100644 tests/test_auth_audit.py create mode 100644 tests/test_client.py create mode 100644 tests/test_config.py create mode 100644 tests/test_creds_source.py create mode 100644 tests/test_hardening.py create mode 100644 tests/test_hasher.py create mode 100644 tests/test_mounts.py create mode 100644 tests/test_p3_remediation.py create mode 100644 tests/test_p5_remediation.py create mode 100644 tests/test_reference_tools.py create mode 100644 tests/test_register.py create mode 100644 tests/test_router.py create mode 100644 tests/test_secrets.py create mode 100644 tests/test_state.py create mode 100644 tests/test_static_disclosure.py create mode 100644 tests/test_store.py create mode 100644 tests/test_token.py diff --git a/.ciagent/ARCHITECTURE.md b/.ciagent/ARCHITECTURE.md new file mode 100644 index 0000000..77b45f0 --- /dev/null +++ b/.ciagent/ARCHITECTURE.md @@ -0,0 +1,48 @@ +# ARCHITECTURE — CoreCI Broker + +## Overview + +One Open WebUI workspace tool, implemented as a single Python file (`coreci_broker.py`), embedding a stdlib HTTP server (`ThreadingHTTPServer`) bound to `BROKER_BIND` (default `127.0.0.1:8765`). It serves a JSON API, a four-page admin web UI (static HTML/CSS/JS from `coreci_broker_ui/`), and a public static client module. SQLite (`broker.db`, mode 0600) is the source of truth; an append-only JSONL file (`broker.audit.log`, mode 0600) records every read and denied attempt. The broker is the trust root with no upstream auth; it reads the `creds-coreci-coo` KB through a pluggable adapter and never writes to it. + +Two runtime modes share the same file: (a) Open WebUI workspace tool — a `Tools` class starts the server in a daemon thread at construction (`__init__`), behind a once-per-process idempotency guard (the OWUI loader has no activate() hook and re-execs modules on edit; paths resolve from env, never `__file__`); (b) standalone — `python3 coreci_broker.py` for development, testing, and direct operation. + +## Components + +| Name | Description | Boundaries | Depends on | +|------|-------------|-----------|------------| +| Config bootstrap | Reads/writes the broker `.env` (spec §5.1 contract + documented additions); first-boot generation of `BROKER_INSTANCE_PEPPER` and `BROKER_ADMIN_PASS` (printed to the startup log once, written to `.env` and SQLite) | Owns the `.env` file and the `broker_config` table; never prints secrets after first boot | SQLite store | +| SQLite store | Tables: `broker_config`, `app_registrations`, `user_allowlist`, `agent_registrations`, `admin_sessions` + indexes (spec §5.2); mode 0600 | Sole writer of `broker.db`; all SQL lives here | — | +| Audit writer/reader | Appends one JSON line per action (register/read/denied/login/logout/add_app/remove_app/add_user/remove_user); filtered query endpoint (since/until/user_id/uuid/action/limit) | Owns `broker.audit.log`; append-only; no rotation in v0.1 | — | +| Hasher | Dual-mode slow hash: bcrypt when importable, else stdlib `hashlib.scrypt`; stored hashes format-prefixed (`$2b$…` / `scrypt$…`) so verification is format-aware | The only component with an optional import | — | +| Token engine | Server-side derivation: `token = base64(sha256(pepper:UUID:user_id:agent_id:','.join(sorted(tags))))` | Pure function of (pepper, inputs); never stores plaintext tokens | Hasher, Config | +| HTTP server + router | stdlib threaded HTTP; JSON body/param handling; `{"error, message}` contract with 400/401/403/404/409/500; static file serving (UI + client module) | No business logic; routes to admin/tool planes | All handlers | +| Admin plane | Session auth (cookie `broker_session` + HTTP Basic); login/logout/whoami; app_registrations CRUD (add/revoke/reinstate); user_allowlist CRUD; audit query | Admin-only; audited | Store, Audit | +| Tool auth plane | `POST /api/broker/agents/register` (allowlist gates, idempotent re-derive, TTL, pepper rule); App-auth verification (`Authorization: App ` — UUID-scoped slow-hash scan + in-memory verified-token cache); `GET /agents/`; state GET/PUT with identity enforcement | Register is callable without a valid token (gates are the allowlists); the cache is never authoritative | Store, Token engine, Hasher | +| Secrets plane | `GET /api/broker/secrets/`; scope check `path == "coreci/" + agent_id`; creds-source adapter (FileCredsSource default \| OwuiApiCredsSource) | The only reader of `creds-coreci-coo`; read-only | Tool auth plane, Creds adapter | +| Agent-state store | Backs `GET/PUT /api/broker/agents//state`; toggle `BROKER_AGENT_STATE_BACKEND=internal` (rows in `broker.db`) `\| owui` (real per-agent KBs via OWUI API) | Broker-owned; tool-facing; identity-checked | Tool auth plane | +| Static UI | Four pages (login, app registrations, user allowlist, audit) + dashboard + persistent nav; server-rendered HTML, no build step | Read-only against the admin API | Admin plane | +| Client module | `coreci_broker_client.py` — dependency-free (stdlib urllib) auth client: ensure-registered, state get/put, secret read with 401→re-register→retry-once; served unauthenticated at `/static/` | Contains no secrets; importable by any OWUI tool | Broker API (external) | +| Reference tools | `reference_tools/` — OWUI tool classes showcasing auth-first-then-use (e.g., proxmox reference: register → fetch `coreci/proxmox` → downstream call, safe dry-run default) | Templates for tool authors; not wired to live infra in tests | Client module | + +## DataFlow + +1. **Admin**: CEO logs in via `/ui/login` (or HTTP Basic) → session row in `admin_sessions` → manages allowlists via UI → every admin action audited. +2. **Registration**: tool calls `POST /api/broker/agents/register` with `{agent_id, app_registration_uuid, user_id, tags}` → broker checks `app_registrations` (active) and `user_allowlist` (active) → 403 on unknown/revoked → broker derives the token from the pepper + inputs → stores slow-hash in `agent_registrations`, sets TTL expiry → returns token (+ pepper when the row is created or the caller presented no valid token) → tool persists via `PUT /agents//state`. +3. **Secret read**: tool calls `GET /api/broker/secrets/coreci/` with `Authorization: App ` → broker parses UUID, scans that tool's registration rows with the slow hash (in-memory cache after first verify) → 401 on failure → scope check `path == coreci/` → 403 on mismatch → creds adapter reads `creds-coreci-coo` → audit line (ok/denied/error) → JSON response. +4. **Expiry**: after `BROKER_TOKEN_TTL` the token verifies but the row is expired → 401 → tool re-registers (step 2) → new token overwrites `token_hash` → audit `register`. + +## BuildOrder + +1. Config bootstrap + SQLite store + audit writer (P1) +2. Hasher + token engine (P1) +3. HTTP server + router + error contract + OWUI/standalone mounts (P1) +4. Admin plane + audit query + static UI (P2) +5. Tool auth plane + agent-state store (P3) +6. Secrets plane + creds adapter + client module + reference tools (P4) +7. Acceptance matrix, security review, README (P5) + +## Notes + +- The in-memory verified-token cache maps `(uuid, token) -> registration row`; it is populated on first successful slow-hash verification and cleared on restart. SQLite remains authoritative; cache entries are re-validated against row status/expiry on every hit. +- Creds adapter contract: lookup by path string (`coreci/`) → JSON object. FileCredsSource reads a single JSON document mapping paths to secret objects (default for dev/tests, `BROKER_CREDS_FILE`). OwuiApiCredsSource calls the Open WebUI HTTP API with `BROKER_OWUI_BASE_URL` + `BROKER_OWUI_API_KEY` and is exercised in production; its exact KB-file layout is resolved in P4 research. +- This file is the init-time snapshot; it is refined at phase boundaries per ci-files discipline. \ No newline at end of file diff --git a/.ciagent/CHECKPOINT.json b/.ciagent/CHECKPOINT.json new file mode 100644 index 0000000..f0b26c5 --- /dev/null +++ b/.ciagent/CHECKPOINT.json @@ -0,0 +1,8 @@ +{ + "phase": 5, + "stage": "complete", + "milestone": "v0.1", + "phase_role": "final", + "attempts": 0, + "updated_at": "2026-10-10T04:50:00Z" +} diff --git a/.ciagent/GRILL.md b/.ciagent/GRILL.md new file mode 100644 index 0000000..4420a3a --- /dev/null +++ b/.ciagent/GRILL.md @@ -0,0 +1,129 @@ +# CIAgent Grill Report + +## Run: 2026-10-09 23:10 (mode: interactive, focus: all) + +### Verdict: Proceed (confidence: 0.85) + +7 of 9 axes resolved on evidence; the four escalated items (Axes 7, 9) were resolved interactively by the owner on 2026-10-09 — decisions G-005..G-008, zero open escalations. + +### Axis 1 — Business Case +- **Q1**: Is the per-tool credential problem real and top-priority? + - Evidence: PROJECT.md:9,47 (four live creds via KB/env/0600 JSON today; proxmox tool exists now — no revocation, no audit, no allowlist); ROADMAP.md:5 (owner-directed 2026-10-09) + - Answer: real and current — the estate's path has no revocation or audit; owner directed same-day. (0.85) +- **Q2**: Named sponsor who has decided under pressure? + - Evidence: D-007 owner-confirmed §13 defaults @0.95 (ffdc539); D-012..D-015 forced under clarify pressure (53d99e8) + - Answer: CEO-owner is sole sponsor and decision-maker; pressure decisions on the record. (0.85) +- **Q3**: What breaks if cancelled — and is ROI evidence-based? + - Evidence: PROJECT.md:9 (counterfactual explicit); no financial deck exists; cost is tokens only + - Answer: cancellation returns the estate to unaudited per-tool reads (capability loss, not continuity); ROI is operational, measured against the documented counterfactual. (0.80) + +### Axis 2 — Scope +- **Q1**: Stable, frozen, owned? + - Evidence: REQ-01..REQ-20 frozen from spec v0.5, full traceability (REQUIREMENTS.md:5,78-99); D-008; "No requirement is deferred" (PROJECT.md:27); owner coreci-coo (REQUIREMENTS.md:3) + - Answer: frozen and traced; one acknowledged in-scope unknown (KB file layout → P4, ARCHITECTURE.md:47) with candidate flows pre-verified (RESEARCH.md:55). (0.85) +- **Q2**: Explicitly out of scope? + - Evidence: 12-row table, each row with reason + spec source (REQUIREMENTS.md:57-74) + - Answer: exhaustive; covers the usual silent creep targets (rotation, HA, MFA, federation). (0.90) +- **Q3**: Hidden requirements (security, ops, accessibility)? + - Evidence: security reviews P3-06/P5-02 (PLAN.md:113,145); traversal research (RESEARCH.md:62); D-013 cookie hardening; REQ-15 0600; ops README P5-04 + - Answer: security/ops surfaced pre-code; accessibility unstated — immaterial for a single-admin internal UI in v0.1. (0.75) + +### Axis 3 — Architecture +- **Q1**: Validated by the builders, not the sellers? + - Evidence: RESEARCH.md §4 source-verified against OWUI source at main (no activate() hook, Valves rule, `__file__` temp-delete; KB endpoints from knowledge.py/files.py); plan-checker PASS_WITH_NOTES (b8122be) + - Answer: yes — the executing pipeline authored and checked the plan against source. (0.80) +- **Q2**: Integration surface bounded? + - Evidence: exactly two external surfaces — OWUI Tools mount (RESEARCH.md:30-38) + OWUI KB read API (RESEARCH.md:40-55); gitea push per sibling convention (RESEARCH.md §3) + - Answer: bounded and source-verified; no third integration exists. (0.80) +- **Q3**: Replaced-system edge cases; inherited debt? + - Evidence: four creds, single host, no volume concern (PROJECT.md:47); spec defects 1-4 found and resolved pre-code (RESEARCH.md §2; D-004/D-005); greenfield repo (acf5557) + - Answer: edge cases (pepper chicken-and-egg, KB-loss, re-reg KeyError) resolved by decision; external debt = OWUI lifecycle quirks, mitigated by the once-per-process guard (ARCHITECTURE.md:7); 800-line budget fails fast via AC14 at P1-08. (0.75) + +### Axis 4 — People +- **Q1**: Key-person dependency — acceptable? + - Evidence: D-007 CEO sole approver, binding (PROJECT.md:61); tokens re-register on TTL; allowlist changes are the only CEO-gated act + - Answer: accepted — CEO absence stalls new onboarding only, not broker continuity; by design for a v0.1 credential broker on a trusted single host. (0.75) +- **Q2**: Resources allocated; PO with authority? + - Evidence: agent pipeline, sequential, parallelization off (config.json:33-37); guided autonomy, escalation_hooks ["*"] (config.json:5-9) + - Answer: no BAU contention applies; owner-CEO is PO with absolute authority. (0.80) +- **Q3**: Building capability they don't have? + - Evidence: sibling precedent on same pipeline + forge (RESEARCH.md §3); the one novel surface (OWUI mount) is source-researched + - Answer: capability precedented; unproven only in that zero code exists — P1 is the cheap falsifier. (0.70) + +### Axis 5 — Timeline +- **Q1**: Deadline set before or after scope? + - Evidence: spec v0.5 (ffdc539, 22:37) precedes plan (b8122be, 23:02); no external deadline exists anywhere in the corpus + - Answer: spec-first; no reverse-engineered date to fail against. (0.85) +- **Q2**: Critical path; single 3-month pusher? + - Evidence: P1→P5 sequential (ROADMAP.md:24-57); only live-environment unknown is OwuiApiCredsSource KB layout (P4), bounded by FileCredsSource default + CEO-controlled KB structure + - Answer: no 3-month pusher; worst case is bounded P4 adapter rework. (0.75) +- **Q3**: Evidence-based estimates; definition of done? + - Evidence: 30 one-commit tasks, tests ship with task (PLAN.md:12); must_haves frontmatter (PLAN.md:2-7); AC1-AC14 + per-phase exits (ROADMAP) + - Answer: slice-sized with checker sign-off; done is test-enforced, not demo-defined. (0.80) + +### Axis 6 — Budget +- **Q1**: Cost profile and burn? + - Evidence: token/compute only; phase 0 complete in ~30 min wall-clock (5 commits); no cash budget exists + - Answer: no runway question; pipeline runs on owned infra. (0.75) +- **Q2**: Predictable drivers outside the budget? + - Evidence: stdlib-only eliminates licensing (D-009); forge org exists (RESEARCH.md:9); OWUI self-hosted; no pip, no npm + - Answer: none uncovered — the design removes the usual cost drivers by construction. (0.80) +- **Q3**: Contingent on something unhappened? + - Evidence: forge repo creation untested (assumption d), but release_blocking false (config.json:24) and manual forge-UI fallback is trivial + - Answer: no financial contingency; the sole contingency is operational (→ Axis 7). (0.70) + +### Axis 7 — Risks +- **Q1**: Top assumptions, evidence per item? + - (a) bcrypt ships in OWUI runtime + daemon-thread tools allowed: RESEARCH.md §4 source+docs — but validated at `main`, deployed version unpinned; absorbed by dual-mode scrypt fallback (0.70) + - (b) stdlib dev parity with the OWUI runtime: D-001 dual-mode covers both environments (RESEARCH.md:8) (0.80) + - (c) FileCredsSource makes tests hermetic: tempdir + env-var state (RESEARCH.md §6; PLAN P1-01/P4-01) — plan-level, proven at P1 (0.70) + - (d) forge repo creatable with current token: token valid, org exists, repo 404 — creation UNTESTED (RESEARCH.md:9). **→ ESCALATION** +- **Q2**: Dependencies outside the team? + - Evidence: OWUI (host + API + process perms), gitea — all in daily estate use (PROJECT.md:47; RESEARCH.md §3) + - Answer: bounded and precedented. (0.75) +- **Q3**: Single killer risk; pre-mortem? + - Evidence: worst case = mount or adapter fails on the live host → standalone mode is functionally equivalent (ARCHITECTURE.md:7); defect table (RESEARCH.md §2) is the pre-mortem substance + - Answer: no project-killing risk found on evidence — but NO plan task validates against the live OWUI host before ship. **→ ESCALATION** (0.65) + +### Axis 8 — Governance +- **Q1**: Decision-maker under disagreement? + - Evidence: single owner-CEO; guided autonomy escalates every gate (ROADMAP.md:9; config.json:6) + - Answer: no committee exists to deadlock; owner is arbiter. (0.85) +- **Q2**: Cadence, escalation, omissions from status? + - Evidence: ---ci--- blocks with confidence on all four commits (ffdc539..b8122be); 15 decisions recorded; checker notes published, not buried + - Answer: decision log is exemplary; the mild contradiction is RESEARCH.md:82 "no genuine blockers" sitting beside untested assumption (d). (0.70) +- **Q3**: Stop-the-project trigger? + - Evidence: MISSING as a formal trigger — but sponsor = approver = decision-maker is one person + - Answer: implicit owner prerogative suffices where one person holds all three roles. (0.70) + +### Axis 9 — Adoption +- **Q1**: Who uses it; what's in it for them? + - Evidence: CEO gets allowlist + audit UI replacing blind KB edits; tool authors get a zero-dep client replacing hand-rolled 0600 JSON (REQ-19; PROJECT.md:9) + - Answer: both user classes shed concrete work — not a "train them and pray" case. (0.75) +- **Q2**: Ops handoff; rollback plan? + - Evidence: ops = owner; README + backup guidance P5-04 (PLAN.md:150); full state = broker.db + broker.audit.log + .env (PROJECT.md:37-39); unmigrated tools never depend on the broker + - Answer: rollback = stop process, restore three files; migrated proxmox tool reverts to direct reads. (0.75) +- **Q3**: Success criteria validated with the judges? + - Evidence: judge = owner via AC1-AC14 (spec §12, owner-owned); but every OWUI-facing test is stub-driven (PLAN P3-04, P4-01), no task touches the live host, and proxmox migration is "post-broker", dateless (PROJECT.md:47) + - Answer: acceptance is owner-validated; live readiness and the adoption criterion were escalated and are now resolved as G-005/G-008. (0.85) + +### Meta +- **Auditor's flag**: no live-host validation before ship; forge repo creation untested; deployed OWUI version unpinned; adoption criterion undefined. All cheap to fix; none fatal. +- **Not being done**: live smoke test on the OWUI host; OWUI version pin; a risk register; post-ship adoption criteria. +- **Simplest 80% version**: drop the admin UI + reference tools (CLI allowlists, file creds only) saves ~one phase — but the UI is the CEO's core value and OwuiApiCredsSource is the production path; at 30 tasks / 800 lines, the over-scope charge does not stick. +- **90-day must-be-true**: broker live on the OWUI host; proxmox tool on the client module; allowlist + audit in CEO daily use. Not true today (no code) — but every gating factor is internal. + +### Binding Decisions +| ID | Axis | Decision | Confidence | +|----|------|----------|------------| +| G-001 | 4 | Key-person dependency accepted for v0.1: sole-approver CEO absence stalls new onboarding only, not continuity (D-007 by design) | 0.75 | +| G-002 | 7 | Assumptions (a)-(c) accepted as mitigated-by-design (dual-mode hashing, hermetic test plan); (d) + live validation escalate | 0.70 | +| G-003 | 9 | Rollback accepted: full state = broker.db + broker.audit.log + .env; stop-and-restore; tools revert to direct reads | 0.75 | +| G-004 | 3 | 800-line budget risk accepted: AC14 enforced from P1-08 (fails fast); spec §12.14 amendment is the owner's pressure valve | 0.70 | +| G-005 | 9 | Post-ship live-OWUI smoke checklist (owner-run: mount, register, read) is the plan of record; v0.1 ships on standalone + stub tests; no real tool migrates before the checklist passes; rollback = disable the tool | 0.85 | +| G-006 | 7 | Deployed OWUI pinned at v0.11.3 (owner-stated 2026-10-09) — same-era as the researched `main` surface, no re-research required; smoke checklist item 1 re-confirms the two KB read endpoints on v0.11.3 | 0.85 | +| G-007 | 7 | Forge repo creation is tested at the P0 ship (POST /orgs/coreci/repos with the existing token), not deferred to P5-06; on scope failure: local-only fallback + auto-escalation, and the owner's 30-second manual forge-UI creation is the recorded fallback | 0.9 | +| G-008 | 9 | 90-day adoption criterion: proxmox_readonly_tool.py runs on coreci_broker_client.py with direct KB reads removed by 2027-01-09; owner owns the date; recorded in ROADMAP as a post-milestone note (out of v0.1 scope) | 0.8 | + +### Escalations +- (none — all four escalated challenges resolved interactively by the owner on 2026-10-09; decisions G-005..G-008 above) \ No newline at end of file diff --git a/.ciagent/PERSONAS.md b/.ciagent/PERSONAS.md new file mode 100644 index 0000000..6440ca0 --- /dev/null +++ b/.ciagent/PERSONAS.md @@ -0,0 +1,60 @@ +# PERSONAS — CoreCI Broker + +RESEARCH-stage roster (2026-10-09, milestone v0.1). Pure stdlib Python 3.11 project: single-file broker, no package.json, no JS framework, no src/ tree; the only UI is server-rendered static HTML/CSS/JS from `coreci_broker_ui/` (P2). Execution is sequential (parallelization off), so phase-specific personas activate only at their phase boundaries per this roster. + +--- +name: backend-engineer +domain: backend +active: true +reason: Primary execution persona — the entire v0.1 product is Python on one file: P1 core skeleton, P2 admin-plane API, P3 tool auth plane, P4 secrets adapter + public client module + reference tools. Owns every security-critical code path (token derivation, slow-hash verification, allowlist gates, audit trail, path-scoped reads) and every test that gates a phase. +frameworks: [python3.11, http.server, sqlite3, hashlib.scrypt, unittest] +constraints: ["coreci_broker.py stays one file under 1400 lines (REQ-17 as amended by D-016..D-018, AC14)", "stdlib-only runtime; bcrypt is the sole optional import; no pip, no npm (D-001/D-009)", "all errors use the {error, message} JSON contract on 400/401/403/404/409/500", "broker.db and broker.audit.log are created mode 0600 (REQ-15)", "the broker never writes to creds-coreci-coo (REQ-08)"] +territory: [coreci_broker.py, tests/**, coreci_broker_client.py, reference_tools/**] +subagent_type: backend-engineer +--- + +--- +name: frontend-engineer +domain: frontend +active: true +phase_specific: true +reason: P2 only (REQ-13 admin UI). Inactive outside P2 — no UI work exists in P1/P3/P4/P5; deactivate at the P2 exit boundary and do not dispatch outside P2. +frameworks: [html, css, vanilla-js] +constraints: ["server-rendered static pages from coreci_broker_ui/ on the same port as the API; no build step, no client framework, no npm (REQ-13)", "pages: /ui/login, /ui/app_registrations, /ui/user_allowlist, /ui/audit plus /ui/dashboard and a persistent nav header with Logout", "pages refresh via the corresponding admin API after each action; the API contract is owned by backend-engineer — frontend adapts", "session cookie broker_session is HttpOnly + SameSite=Lax (D-013); no direct DB or file access — admin API only"] +territory: [coreci_broker_ui/**] +subagent_type: frontend-engineer +--- + +--- +name: security-engineer +domain: security +active: true +phase_specific: true +reason: Custom persona activated for P3 (tool auth plane: register gates, App-token verification, identity enforcement) and P5 (security review pass before ship); inactive in P1/P2/P4 unless the lead escalates a finding. +frameworks: [python3.11, unittest, STRIDE, CWE] +constraints: ["review-only: coreci_broker.py and tests stay owned by backend-engineer; findings route through lead-developer, never direct edits", "findings are STRIDE-classified and CWE-first (cite CWE ids)", "focus: path traversal on /secrets/ (URL-decode before validation), authz (allowlist gates, path scoping, identity match), audit completeness, pepper never re-printed, 0600 modes, token scoping"] +territory: [coreci_broker.py, tests/**] +subagent_type: ci-security-auditor +--- + +--- +name: lead-developer +domain: coordination +active: true +reason: Active across all execution phases (P1–P5): decomposes plans into vertical slices per persona domain, arbitrates territory conflicts, revises the plan at phase boundaries, and sequences tasks across persona dependencies. No direct code writes. +frameworks: [none] +constraints: ["no direct modifications to code files — coordination and conflict resolution only", "arbitration: backend-engineer owns files in cross-persona conflicts and defines the API contract; frontend adapts; security findings flow through backend", "enforce boundaries: no direct database access from the UI, no UI logic inside coreci_broker.py; battle-tested defaults over novel approaches"] +territory: [.ciagent/PLAN.md, .ciagent/*.md] +subagent_type: lead-developer +--- + +--- +name: data-engineer +domain: data +active: false +reason: Deactivated — no database tier exists beyond sqlite3 embedded in the single file; the spec §5.2 schema and all SQL live inside coreci_broker.py and are owned by backend-engineer; no pipelines, ETL, or migrations in v0.1 scope. +frameworks: [sqlite3] +constraints: ["stays inactive for the whole milestone; any residual data work folds into backend-engineer"] +territory: [] +subagent_type: backend-engineer +--- \ No newline at end of file diff --git a/.ciagent/PLAN.md b/.ciagent/PLAN.md new file mode 100644 index 0000000..9cca0bf --- /dev/null +++ b/.ciagent/PLAN.md @@ -0,0 +1,158 @@ +--- +must_haves: + - "python3 -m unittest discover -s tests passes (all phases, cumulative)" + - "coreci_broker.py single file, line count < 1400 per D-016..D-018 (AC14)" + - "python3 -m py_compile coreci_broker.py coreci_broker_client.py" + - "broker boots standalone on an ephemeral port; DB + audit log created mode 0600; admin pass printed exactly once" + - "all AC1-AC14 acceptance criteria green by end of P5" +--- + +# PLAN — CoreCI Broker (Milestone v0.1) + +Execution conventions: sequential waves (within a wave, tasks run in listed order); each task = one subagent commit `feat(P##): …` carrying a `---ci---` block (phase, milestone: v0.1, status: execute, persona, task, requirements.covered). Tests ship with their task — never deferred. Personas per PERSONAS.md: backend-engineer (all phases), frontend-engineer (P2 only), security-engineer (P3+P5, review-only — findings route via lead-developer, never direct edits), lead-developer (coordination, no code writes). Verify commands: `python3 -m py_compile `; `python3 -m unittest discover -s tests`; AC14 line-count via test. Test harness: in-process `ThreadingHTTPServer` on an ephemeral port + `tempfile.TemporaryDirectory()` state, driven with stdlib `urllib` (RESEARCH §6). + +## User-Facing Surface + +- **CEO admin UI** — server-rendered from `coreci_broker_ui/`, same port as the API, persistent nav with Logout: `/ui/login` (the CEO's bookmark), `/ui/app_registrations` (table + add-modal + revoke/reinstate), `/ui/user_allowlist` (same, optional email), `/ui/audit` (filterable, read-only, no export), `/ui/dashboard`. +- **Tool-side client module** — `coreci_broker_client.py`: `ensure_registered()`, `get_state()`, `put_state()`, `read_secret()`; stdlib-urllib-only, importable with zero dependencies in any OWUI tool env; fetched unauthenticated at `/static/coreci_broker_client.py`. +- **Standalone CLI** — `python3 coreci_broker.py`: binds `127.0.0.1:8765`; first boot prints the generated admin pass exactly once. +- **OWUI Tools mount** — `Tools` class (same file): daemon-thread server behind a once-per-process guard, `self.valves`, `_`-prefixed helpers. + +## Happy Path + +1. First boot in a fresh directory: broker writes `.env` (32-byte base64 pepper + 32+-char admin pass), prints the pass to the startup log exactly once (AC1); a second boot prints nothing. +2. CEO opens the `/ui/login` bookmark, logs in with the printed pass (session cookie `broker_session`), lands on the dashboard. +3. CEO adds the tool's app UUID on `/ui/app_registrations` and the user_id on `/ui/user_allowlist`; each page reflects the persisted row right after the action (AC2/AC3); audit gets `add_app`/`add_user`. +4. Tool author fetches `/static/coreci_broker_client.py` (unauthenticated) into the OWUI workspace and imports it — no third-party deps. +5. Tool calls `ensure_registered()`: broker gates on the allowlists, derives the token server-side, returns `{token, token_expires_at, pepper}` (row-create); the client persists via `PUT /agents//state`. +6. Tool reads `coreci/proxmox` with `Authorization: App ` → 200 secret payload (AC5); audit line `read`/`ok`. +7. Same tool requests `coreci/gitea` → scope mismatch → 403 (AC6); audit `read`/`denied`. +8. Token expires (`BROKER_TOKEN_TTL`): next call → 401 (AC10) → client re-registers → new token → retry succeeds; audit `register`. +9. CEO reviews `/ui/audit`: login/logout (AC9), allowlist adds, registers, reads ok/denied — every step audited. + +## UX Acceptance Criteria + +- **AC1** — first boot prints the admin pass exactly once; the CEO logs in at `/ui/login` with it; second boot prints nothing. +- **AC2/AC3** — app registrations and user-allowlist entries are added and removed via the UI; the page reflects persisted state after every action (no stale views). +- **AC9** — every login and logout writes an audit line (`login`/`logout`). +- **UI** — server-rendered static HTML/CSS/JS from `coreci_broker_ui/` on the same port: no build step, no framework, no npm. +- **Audit page** — read-only with filters; NO export button or download control of any kind. +- **Client module** — `coreci_broker_client.py` imports cleanly with zero deps (stdlib urllib only) in any OWUI tool env; contains no secrets. + +## Phase 1 — Core Skeleton + +Branch `phase/01-core-skeleton` → tag `v0.1.1`. Requirements: REQ-03, REQ-11, REQ-14, REQ-15, REQ-17, REQ-18. +Exit: broker boots standalone on an ephemeral port; `broker.db` + `broker.audit.log` created mode 0600; admin pass printed exactly once (silent second boot); §6 token derivation byte-exact; source under the D-018 cap (1400). + +### Wave 1 — persistence layer (independent) + +- **P1-01** · backend-engineer · `coreci_broker.py`, `tests/test_config.py` · `.env` reader/writer at `BROKER_ENV_PATH` (spec §5.1 contract + documented additions); first-boot generation of `BROKER_INSTANCE_PEPPER` (32 random bytes, base64, synced to SQLite when the store boots — sync lands with P1-02's schema) and `BROKER_ADMIN_PASS` (32+ random chars), written to `.env`, printed to the startup log exactly once — never again · REQ-14 · test: first boot writes `.env` + prints pass once; second boot silent; `BROKER_*` overrides honored. +- **P1-02** · backend-engineer · `coreci_broker.py`, `tests/test_store.py` · SQLite schema per spec §5.2 (`broker_config`, `app_registrations`, `user_allowlist`, `agent_registrations`, `admin_sessions` + indexes); DB created mode 0600 via `os.open(..., 0o600)` (umask-independent); sole writer of `broker.db`, all SQL in-file · REQ-15 · test: boot creates `broker.db` at 0600 with all tables + indexes. +- **P1-03** · backend-engineer · `coreci_broker.py`, `tests/test_audit.py` · append-only JSONL audit writer at `BROKER_AUDIT_LOG_PATH`: one JSON object per line, fields `ts, user_id, app_registration_uuid, agent_id, action, path, result` (admin actions carry null user_id/uuid); mode 0600; flush + fsync per write; no rotation (D-007) · REQ-11 · test: each write appends one valid JSON line at mode 0600; all action verbs accepted. + +### Wave 2 — crypto primitives + +- **P1-04** · backend-engineer · `coreci_broker.py`, `tests/test_hasher.py` · dual-mode hasher (D-001): bcrypt when importable, else stdlib `hashlib.scrypt`; stored hashes format-prefixed (`$2b$…` / `scrypt$…`); verification format-aware · REQ-03 · test: scrypt hash/verify roundtrip + prefix dispatch on both formats (bcrypt path via format contract — no bcrypt locally). +- **P1-05** · backend-engineer · `coreci_broker.py`, `tests/test_token.py` · token engine: `base64(sha256(pepper:UUID:user_id:agent_id:','.join(sorted(tags))))` — 44-char ASCII, no prefix; pure function of (pepper, inputs); never stores plaintext tokens · REQ-03 · test: derivation matches the §6 formula byte-for-byte; 44-char assertion; tag order normalized. + +### Wave 3 — transport + mounts (listed in dependency order) + +- **P1-06** · backend-engineer · `coreci_broker.py`, `tests/test_router.py` · stdlib `ThreadingHTTPServer` + JSON router: body/param parsing, `{"error, message}` contract on 400/401/403/404/409/500, static-file serving hook for `/ui/` + `/static/` (fixed root, traversal-rejecting); no business logic in the router · REQ-18 · test: ephemeral-port boot; every error code returns the contract shape; static hook serves a fixture. +- **P1-07** · backend-engineer · `coreci_broker.py`, `tests/test_mounts.py` · OWUI `Tools` mount: `__init__` starts the daemon-thread server behind a once-per-process guard (no activate() hook exists — RESEARCH §4), `self.valves = self.Valves()`, `_`-prefixed helpers, paths resolved from env never `__file__`; standalone `__main__` entry · REQ-18 · test: `Tools()` double-instantiation starts exactly one server; standalone boot serves a 200. + +### Wave 4 — phase acceptance + +- **P1-08** · backend-engineer · `tests/test_acceptance_p1.py` · AC14 line-count test (`coreci_broker.py` under the current cap — 1400 per D-018, wc-style, no comment stripping) + boot acceptance: 0600 on DB + audit log, admin pass printed exactly once (silent second boot), restart persistence of DB rows + audit history · REQ-17, REQ-16 (per REQUIREMENTS traceability), REQ-15 · test: is the acceptance file. + +## Phase 2 — Admin Plane + +Branch `phase/02-admin-plane` → tag `v0.1.2`. Requirements: REQ-01, REQ-05, REQ-06, REQ-12, REQ-13. +Exit: AC1/AC2/AC3/AC9 green; admin-usable (login → manage allowlists → query audit, all via UI). + +### Wave 1 — admin auth + +- **P2-01** · backend-engineer · `coreci_broker.py`, `tests/test_admin_auth.py` · `POST /api/broker/auth/login|logout|whoami`: verifies `BROKER_ADMIN_USER`/`BROKER_ADMIN_PASS`, creates `admin_sessions` row (TTL `BROKER_SESSION_TTL`, default 300s); cookie `broker_session` HttpOnly + SameSite=Lax (D-013); HTTP Basic accepted on all admin API endpoints; lazy expiry + boot cleanup of stale sessions · REQ-01, REQ-11 · test: login sets cookie; wrong pass → 401; expired session → 401; logout invalidates + audits (AC9 `login`/`logout` lines). + +### Wave 2 — admin API + +- **P2-02** · backend-engineer · `coreci_broker.py`, `tests/test_apps.py` · app_registrations CRUD: `GET/POST /api/broker/app_registrations`, `DELETE .../` (marks revoked — new registrations rejected, existing tokens honored until TTL), `POST .../reinstate`; UUID format validation → 400; duplicate → 409; audited `add_app`/`remove_app` · REQ-05 · test: add/list/revoke/reinstate matrix + 400/409 + revoke-keeps-tokens + audit lines. +- **P2-03** · backend-engineer · `coreci_broker.py`, `tests/test_allowlist.py` · user_allowlist CRUD: same shape (`GET/POST`, `DELETE .../` revoked semantics, `POST .../reinstate`), optional `email`, duplicate → 409; audited `add_user`/`remove_user` · REQ-06 · test: add/remove/reinstate + 409 + audit lines. +- **P2-04** · backend-engineer · `coreci_broker.py`, `tests/test_audit_query.py` · admin-only `GET /api/broker/audit`: filters `since`, `until`, `user_id`, `app_registration_uuid`, `action`; `limit` default 100 / max 1000; `next_since` cursor; streams the file line-by-line · REQ-12 · test: seeded audit file → filter combos, limit clamp, cursor pagination. + +### Wave 3 — admin UI (frontend-engineer; pages refresh via the admin API after each action; no direct DB/file access) + +- **P2-05** · frontend-engineer · `coreci_broker_ui/` (login page, dashboard, shared nav/CSS) · `/ui/login` (the CEO's bookmark) + `/ui/dashboard`; persistent nav header with Logout on every page · REQ-13, REQ-01 · verify: pages served 200 on the API port, post-login redirect to dashboard; full AC drive lands in P2-08. +- **P2-06** · frontend-engineer · `coreci_broker_ui/` (app_registrations + user_allowlist pages) · table + add-modal + revoke/reinstate per row on both pages; after each action the page re-fetches via the corresponding API and reflects persisted state · REQ-13, REQ-05, REQ-06 · verify: AC2/AC3 flows (tests in P2-08); interim smoke — pages boot + served. +- **P2-07** · frontend-engineer · `coreci_broker_ui/` (audit page) · filterable audit view (since/until/user_id/uuid/action) wired to `GET /api/broker/audit`; strictly read-only — no export button or control of any kind · REQ-13, REQ-12 · verify: page renders query results; no export element in the markup. + +### Wave 4 — phase acceptance + +- **P2-08** · backend-engineer · `tests/test_acceptance_p2.py` · AC1 (first-boot login at `/ui/login` with the printed pass), AC2/AC3 (add + remove app/user through the UI; `app_registrations`/`user_allowlist` persist; page reflects state after each action), AC9 (login/logout audit lines) · REQ-01, REQ-05, REQ-06, REQ-12, REQ-13 · test: is the acceptance file. + +## Phase 3 — Tool Auth Plane + +Branch `phase/03-tool-auth-plane` → tag `v0.1.3`. Requirements: REQ-02, REQ-04, REQ-09, REQ-10. +Exit: AC4/AC7/AC8/AC10/AC12 green; tool-auth usable (register → App-auth calls → state persistence). + +### Wave 1 — registration + +- **P3-01** · backend-engineer · `coreci_broker.py`, `tests/test_register.py` · `POST /api/broker/agents/register` with `{agent_id, app_registration_uuid, user_id, tags}`: allowlist gates → 403 unknown/revoked UUID or user; malformed → 400; idempotent re-derive (D-005) — overwrite stored hash on input change, same derived token when unexpired + unchanged; TTL from `BROKER_TOKEN_TTL` (default 86400, D-007); pepper returned on row-create or tokenless register (KB-loss recovery); audited `register` · REQ-02, REQ-03 · test: gate matrix (AC7/AC8), idempotency/overwrite/same-token, pepper rule, 400s. + +### Wave 2 — App-auth + agent surface (listed in dependency order) + +- **P3-02** · backend-engineer · `coreci_broker.py`, `tests/test_app_auth.py` · App-auth verification (D-004): parse `Authorization: App `; UUID-scoped slow-hash scan over that tool's rows; in-memory verified-token cache populated on first success, never authoritative — every hit re-checks row status + expiry; 401 missing/malformed/failed · REQ-04 · test: cache-miss scan + cache-hit re-check (revoke + expiry), both hash formats, 401 matrix; revoked-app token honored until TTL. +- **P3-03** · backend-engineer · `coreci_broker.py`, `tests/test_agent_meta.py` · `GET /api/broker/agents/` (D-015): App-auth + identity match; metadata-only response (no token, no hash) · REQ-04, REQ-02 · test: correct-identity 200 with metadata-only payload; identity mismatch → 403; no token material in response. +- **P3-04** · backend-engineer · `coreci_broker.py`, `tests/test_state.py` · `GET/PUT /api/broker/agents//state` (App-auth): broker enforces calling UUID + token-derived user_id match the stored identity; 404 when no state exists; state schema per spec §8 (agent_id, app_registration_uuid, user_id, tags, pepper, current_token, token_expires_at, first_registered_at, last_seen_at); backend toggle `BROKER_AGENT_STATE_BACKEND=internal` (default, rows in `broker.db`) `| owui` (thin OWUI-API adapter, stub-verified) (D-003) · REQ-09, REQ-10 · test: state roundtrip + identity enforcement + 404; owui backend against a stub server. + +### Wave 3 — phase acceptance + +- **P3-05** · backend-engineer · `tests/test_acceptance_p3.py` · AC4 (broker- and tool-side derivation byte-identical; stored as slow-hash), AC7 (revoked/unknown UUID → 403), AC8 (unknown user → 403), AC10 (TTL expiry → 401 → re-register → new token, audited), AC12 (state read/write with identity enforcement) · REQ-02, REQ-04, REQ-09, REQ-10 · test: is the acceptance file. + +### Wave 4 — security review (review-only) + +- **P3-06** · security-engineer · review of `coreci_broker.py`, `tests/**` (no edits — findings STRIDE-classified, CWE-cited, routed via lead-developer) · tool-auth security pass: register gates, UUID-scoped scan + cache authority, identity enforcement, TTL/expiry handling, audit completeness · REQ-02, REQ-04, REQ-09 · verify: findings recorded in the task commit's `---ci---` block; remediation (if any) = follow-up backend commit dispatched by lead-developer. + +## Phase 4 — Secrets + Client Ecosystem + +Branch `phase/04-secrets-client-ecosystem` → tag `v0.1.4`. Requirements: REQ-07, REQ-08, REQ-19, REQ-20. +Exit: AC5/AC6 green; full product — client module fetched from `/static/` drives a reference tool end-to-end against the test broker. + +### Wave 1 — secrets plane (listed in dependency order) + +- **P4-01** · backend-engineer · `coreci_broker.py`, `tests/test_creds_source.py` · creds-source adapter (D-002): `FileCredsSource` default (JSON at `BROKER_CREDS_FILE`, default `data/creds.json` — D-014) + `OwuiApiCredsSource` (`BROKER_OWUI_BASE_URL` + `BROKER_OWUI_API_KEY`; RESEARCH §4 flow: KB lookup → file map → `/api/v1/files/{id}/data/content`); response `{path, data, version, updated_at}` (version synthesized broker-side if absent); broker never writes the KB · REQ-08 · test: file-source hit/miss; owui source against a stub HTTP server. +- **P4-02** · backend-engineer · `coreci_broker.py`, `tests/test_secrets.py` · `GET /api/broker/secrets/` (App-auth): scope check `path == "coreci/" + agent_id` → 403 mismatch, 404 absent, 401 auth failure; every attempt audited (`read` ok/denied/error); traversal defense — URL-decode BEFORE validation, reject `..`, backslash, NUL, absolute paths (decoded form feeds the scope check) · REQ-07 · test: ok/401/403/404 matrix + URL-encoded traversal attempts rejected + audit per attempt. + +### Wave 2 — client ecosystem (listed in dependency order) + +- **P4-03** · backend-engineer · `coreci_broker_client.py`, `tests/test_client.py`, `coreci_broker.py` (static route) · public client module: stdlib `urllib` only (no httpx, no bcrypt — importable in any OWUI tool env); `ensure_registered()` idempotent; `get_state()`/`put_state()`; `read_secret()` with 401 → re-register → retry-once; `data.get("pepper")` (never assumes the key — RESEARCH §2 defect 3); served unauthenticated at `/static/coreci_broker_client.py` via the P1-06 hook; contains no secrets · REQ-19 · test: zero third-party imports asserted; full flow against the in-process broker; 401→re-register→retry path. +- **P4-04** · backend-engineer · `reference_tools/`, `tests/test_reference_tools.py` · reference OWUI tools (D-006): (a) proxmox-style §11 flow — first call registers + persists state, subsequent calls read the token from state and fetch `coreci/proxmox`, downstream call defaults to a safe dry-run; (b) minimal auth-first-then-use-a-second-tool showcase · REQ-20 · test: both tools drive the test broker (FileCredsSource); dry-run default asserted; no live infra. + +### Wave 3 — phase acceptance + +- **P4-05** · backend-engineer · `tests/test_acceptance_p4.py` · AC5 (valid token reads the secret; tampered token → 401), AC6 (proxmox-registered tool reading `coreci/gitea` → 403); end-to-end: fetch client from `/static/` unauthenticated → import → register → state → secret read · REQ-07, REQ-19, REQ-20 · test: is the acceptance file. + +## Phase 5 — Final Review + Ship + +Branch `phase/05-final-review-ship` → tag `v0.1.5` (final phase patch = milestone release, D-008). Requirements: REQ-01..REQ-20 (acceptance). +Exit: AC1–AC14 all green; security review PASS; README complete with deviations; owner approval recorded; milestone merged to main, tagged `v0.1.5`, released on gitea. + +### Wave 1 — acceptance matrix + +- **P5-01** · backend-engineer · `tests/test_acceptance_full.py` · complete §12 matrix: AC1–AC13 as behavioral tests (consolidating the per-phase AC files and completing any remaining §12 criteria — 0600 modes, restart persistence, client-module fetch) + AC14 line-count capstone; cumulative suite green via `python3 -m unittest discover -s tests` · REQ-01..REQ-20 · test: is the acceptance suite. + +### Wave 2 — security review (review-only) + +- **P5-02** · security-engineer · review of `coreci_broker.py`, `coreci_broker_client.py`, `reference_tools/`, `tests/**` (no edits — STRIDE/CWE findings via lead-developer) · pre-ship security pass: path traversal incl. URL-encoded forms, pepper never re-printed (second boot, re-register), 0600 modes, token scoping + identity enforcement, audit completeness, static-serving roots · REQ-07, REQ-11, REQ-14, REQ-15 (+ all) · verify: PASS or CWE-cited findings; remediation follows in P5-03. + +### Wave 3 — remediation + docs + +- **P5-03** · backend-engineer · `coreci_broker.py`, `tests/**` · remediate P5-02 findings (no-op skip if PASS) · affected REQs per finding · test: full suite re-green. +- **P5-04** · backend-engineer · `README.md` · run/deploy notes: standalone + OWUI mount, env/config contract (spec §5.1 + documented additions), first-boot pass discipline, backup guidance (SQLite is the source of truth), client-module bootstrap; documented deviations distilled from D-001..D-015 incl. the spec defects + resolutions (PROJECT.md §Context) · REQ-14, REQ-18 (+ traceability) · test: README covers run/deploy/deviations; suite still green. + +### Wave 4 — review + reconstruction + +- **P5-05** · lead-developer · `.ciagent/*.md` review notes (no code writes) · multi-persona review of the milestone diff (backend, frontend P2 artifacts, security sign-offs); audit/reconstruction: every REQ-01..REQ-20 maps to green tests + traceable commits · REQ-01..REQ-20 · verify: sign-offs + reconstruction recorded in `---ci---` blocks. + +### Wave 5 — ship + +- **P5-06** · lead-developer · no code writes · owner approval gate recorded; squash-merge `milestone/v0.1-coreci-broker` → `main`; tag `v0.1.5`; first push to gitea `coreci/broker` (public, default branch main — forge repo exists before push, D-010/D-012) + release · REQ-01..REQ-20 · verify: `v0.1.5` on `main`; release visible on the forge. \ No newline at end of file diff --git a/.ciagent/PROJECT.md b/.ciagent/PROJECT.md new file mode 100644 index 0000000..415c3dd --- /dev/null +++ b/.ciagent/PROJECT.md @@ -0,0 +1,72 @@ +# PROJECT — CoreCI Broker + +## Name + +CoreCI Broker + +## Core Value + +A single canonical credential flow for autonomous agent tools. The broker mediates access to the `creds-coreci-coo` knowledge base: tools authenticate by a derived token (not a shared secret), the CEO manages a UI allowlist of approved tool UUIDs and user IDs, every read is audited to an append-only file, and per-tool scope is enforced by path (`coreci/` only). It replaces per-tool credential resolution (KB files, env vars, mode-0600 JSON) with one small, readable implementation surface: a single Python file, one SQLite DB, a four-page web UI. + +## Requirements + +### Validated + +Source: Requirements Specification v0.5 (owner: coreci-coo, 2026-10-09; supersedes v0.4). The full v0.5 spec is distilled into REQ-01..REQ-20 in REQUIREMENTS.md, with acceptance criteria drawn from the spec's §12 list. Clarification decisions D-001..D-011 (recorded in the init commit) plus D-012..D-015 (recorded in the clarify stage) resolve the spec's ambiguities and residual defaults: + +- Dual-mode token hashing (bcrypt when importable, stdlib scrypt fallback; format-prefixed hashes) +- Pluggable creds-source adapter (file-based default; Open WebUI HTTP API for production) +- Agent-state backend toggle (`internal` default | `owui`) +- UUID-scoped token verification + in-memory verified-token cache (spec §7.5's token_hash index is impossible with bcrypt) +- Register/pepper semantics including the KB-loss recovery path +- A public, dependency-free client module served as a static asset so agents can bootstrap their own OWUI tools +- Spec §13 defaults: no audit rotation, single instance, 24h token TTL, CEO sole approver + +### Active + +Milestone v0.1 (`milestone/v0.1-coreci-broker`) implements the entire v0.5 spec: REQ-01..REQ-20. No requirement is deferred. + +### Out of Scope + +See REQUIREMENTS.md §Out of Scope — federation, secret versioning, high availability, UI beyond the four pages + dashboard, audit rotation, multiple broker instances, approver agents, key rotation ceremony, HSM, rate limiting, admin MFA, notifications. + +## Constraints + +- **Single-file broker**: `coreci_broker.py` under 1400 lines (amended from 800 via owner decisions D-016..D-018; excluding tests, UI HTML, client module, reference tools) — spec §12.14 as amended +- **Stdlib-only runtime**: no pip in the target dev environment; bcrypt is the sole optional import (dual-mode hashing); tests use unittest +- **SQLite is the source of truth**: single instance, no HA; back it up +- **Bind default** `127.0.0.1:8765`; the bind address controls reachability +- **Mode 0600** on `broker.db` and `broker.audit.log` +- **The broker never writes to `creds-coreci-coo`** — the CEO writes that KB directly through Open WebUI's UI +- **Config contract** = spec §5.1 plus documented additions: `BROKER_ENV_PATH`, `BROKER_CREDS_SOURCE`, `BROKER_CREDS_FILE`, `BROKER_OWUI_BASE_URL`, `BROKER_OWUI_API_KEY`, `BROKER_AGENT_STATE_BACKEND` +- **Token form**: `base64(sha256(pepper:UUID:user_id:agent_id:sorted_tags))` — 44-char ASCII, no prefix; `BROKER_TOKEN_TTL` default 86400s +- **Open WebUI runtime**: bcrypt and httpx exist there; the client module must not depend on either (stdlib urllib) so any tool can import it + +## Context + +CoreCI estate: sibling projects researchx and nextcraft live on gitea (`git.coreci.dev/coreci/*`) and use this same ciagent pipeline. The broker runs on the Open WebUI host as one workspace tool (embedded HTTP server in a daemon thread; also runnable standalone). The `creds-coreci-coo` KB currently mediates four credentials (`coreci/proxmox`, `coreci/gitea`, `coreci/executor` TBD, plus the COO's own login — independent of the broker's `BROKER_ADMIN_PASS`). The existing `proxmox_readonly_tool.py` is updated post-broker to use the public client module; it lives outside this repo, so this repo ships the reference implementation of the tool-side bootstrap instead. + +Known spec defects resolved by clarification (recorded for traceability): §7.5's in-memory `token_hash` index is impossible with bcrypt salted hashes; §6 claims the tool derives a token every call while §9's flow uses the broker-returned token from the agent KB (the broker derives authoritatively; the tool stores); §9's snippet raises `KeyError` on `data["pepper"]` for re-registrations; the pepper chicken-and-egg (a token is needed to read agent state, but the pepper is needed to derive a token) is fixed by returning the pepper when the caller presents no valid token. + +## Key Decisions + +| ID | Decision | Rationale | Outcome | +|----|----------|-----------|---------| +| D-001 | Dual-mode bcrypt/scrypt hashing, format-prefixed hashes | Spec §6 requires bcrypt; dev env is stdlib-only; OWUI runtime ships bcrypt | To be validated in P1 | +| D-002 | Creds-source adapter: file (default) \| OWUI API | Spec never specified the physical read mechanism; both environments must work | To be validated in P4 | +| D-003 | Agent-state backend toggle, internal default | §9.1 endpoints are the only interface; internal storage is functionally identical without the admin-API dependency | To be validated in P3 | +| D-004 | UUID-scoped slow-hash scan + in-memory verified-token cache | §7.5's bcrypt token_hash index is impossible; header UUID bounds the scan; SQLite stays authoritative | To be validated in P3 | +| D-005 | Register re-derives; pepper returned on row-create or tokenless register | Broker holds pepper + all inputs; tokenless register is the only KB-loss recovery path | To be validated in P3 | +| D-006 | Public dependency-free client module at /static/ + reference tools | Owner directive: agents bootstrap their own tools within the OWUI workspace | To be validated in P4 | +| D-007 | Spec §13 defaults accepted | Owner confirmed: no rotation, single instance, 24h TTL, CEO sole approver | Binding for v0.1 | +| D-008 | Milestone v0.1; "v0.5" remains the spec revision label | computeMilestoneTag greenfield convention (researchx, nextcraft precedent) | Binding for v0.1 | +| D-009 | Stdlib-only runtime; unittest for tests | No pip in environment; dual-mode hashing is the sole optional import | Binding for v0.1 | +| D-010 | Single-project mode; gitea release target coreci/broker | Sibling convention on the same forge; first push deferred to a ship phase | Binding for v0.1 | +| D-011 | Empty root commit on main precedes the branch hierarchy | Branches cannot be created from an unborn HEAD; nextcraft precedent | Executed at init | +| D-012 | Forge repo coreci/broker public, default branch main | Sibling convention (researchx/nextcraft both public); code ships no runtime secrets — pepper/admin pass generated at boot, .env.secrets gitignored | Binding for v0.1 | +| D-013 | Session cookie `broker_session` uses HttpOnly + SameSite=Lax | Spec silent on cookie attributes; HttpOnly blocks JS exfiltration, Lax preserves bookmark-login UX | Binding for v0.1 | +| D-014 | `BROKER_CREDS_FILE` defaults to `data/creds.json` | FileCredsSource dev/test default location; `data/` already gitignored | Binding for v0.1 | +| D-015 | `GET /api/broker/agents/`: App-auth + identity match, metadata-only response (no token, no hash) | §7.4 auth/response unspecified; mirrors state-endpoint identity enforcement; token material never leaves the broker | Binding for v0.1 | +| D-016 | AC14 line cap amended: coreci_broker.py < 1000 lines (was < 800) | Owner-approved §12.14 amendment at P2: 11 lines of headroom vs ~275 lines of remaining handler surface; preserves the readable single-file intent without dense-cramming | Binding for v0.1 | +| D-017 | D-016 extended: AC14 cap < 1200 lines | Second §12.14 amendment at P3 (owner's D-016 pattern applied with new data): P3-01 lands at 990/1000 with ~150-180 lines of remaining broker surface (App-auth cache, agent state, creds adapter, secrets endpoint); mild compaction reclaims only ~50 — same readability-over-cramming tradeoff the owner chose at D-016; subject to review at the P5 owner approval gate | Binding for v0.1 | +| D-018 | D-017 extended: AC14 cap < 1400 lines — final projected amendment | Third §12.14 amendment, pre-P4 per the P3 verify planning signal: 1178/1200 after P3 with 21 lines headroom vs P4's ~105-140 line need (creds adapter + secrets endpoint, the LAST broker-surface additions); P5 adds only excluded artifacts (tests, README). Same owner pattern (D-016/D-017), flagged for the P5 owner approval gate | Binding for v0.1 | \ No newline at end of file diff --git a/.ciagent/REQUIREMENTS.md b/.ciagent/REQUIREMENTS.md new file mode 100644 index 0000000..04e1fa3 --- /dev/null +++ b/.ciagent/REQUIREMENTS.md @@ -0,0 +1,99 @@ +# Requirements — CoreCI Broker + +## Overview + +Source: Requirements Specification v0.5 (owner: coreci-coo, 2026-10-09; supersedes the v0.4 in-memory draft). "v0.5" is the spec's revision label; the product milestone is v0.1. REQ-01..REQ-20 constitute the full v0.1 scope — nothing is deferred. Acceptance criteria reference the spec's §12 list (AC1–AC14). Clarification decisions D-001..D-011 live in the init commit's `---ci---` block and are distilled in PROJECT.md. + +## v1 (Milestone v0.1) + +### Auth & Identity + +- **REQ-01 Admin authentication.** `BROKER_ADMIN_USER`/`BROKER_ADMIN_PASS` from the broker `.env`. Session cookie `broker_session` for the UI; HTTP Basic accepted on all admin API endpoints. `POST /api/broker/auth/login` creates a session (SQLite `admin_sessions`, TTL from `BROKER_SESSION_TTL`, default 300s), `logout` invalidates it, `whoami` returns the session info or 401. Admin sessions expire lazily and are cleaned on boot. +- **REQ-02 Tool self-registration.** `POST /api/broker/agents/register` with `{agent_id, app_registration_uuid, user_id, tags}`. The broker rejects unknown or revoked UUIDs and user_ids with 403 (allowlists are the gate); malformed input → 400. Re-registration is idempotent: the broker re-derives from the request inputs and overwrites the stored hash when inputs changed; an unexpired row re-issues the same derived token. The response returns the token, `token_expires_at`, and the pepper when the row is created or the caller presented no valid token (KB-loss recovery). Register actions are audited. +- **REQ-03 Derived token.** `token = base64(sha256(pepper:APP_UUID:user_id:AGENT_ID:','.join(sorted(tags))))` — a 44-char ASCII string, no prefix, no challenge protocol. The broker derives the same token a compliant tool would derive. Stored as a slow hash in `agent_registrations.token_hash`: bcrypt when importable, else stdlib scrypt; hashes are format-prefixed (`$2b$…` / `scrypt$…`) so verification is format-aware. +- **REQ-04 Token verification & TTL.** Tool endpoints require `Authorization: App `. Verification: look up rows scoped to the presented UUID, check the slow hash (in-memory verified-token cache after first success; cache is never authoritative — rows re-checked for status/expiry). Missing/malformed/failed → 401. Tokens live `BROKER_TOKEN_TTL` seconds (default 86400); expiry → 401 → the tool re-registers. Revoked apps keep working until TTL expiry by design. + +### Allowlist Administration + +- **REQ-05 App registrations.** Admin-only `GET/POST /api/broker/app_registrations`, `DELETE /api/broker/app_registrations/` (marks revoked; new registrations rejected; existing tokens honored until TTL), `POST .../reinstate`. UUID format validated (400 on invalid, 409 on duplicate). All actions audited as `add_app`/`remove_app`. +- **REQ-06 User allowlist.** Admin-only `GET/POST /api/broker/user_allowlist`, `DELETE .../` (revoked semantics), `POST .../reinstate`. Optional `email` field. 409 on duplicate. Audited as `add_user`/`remove_user`. + +### Secret Mediation + +- **REQ-07 Path-scoped secret reads.** `GET /api/broker/secrets/`. Scope is the path: a registration may read exactly `coreci/` — cross-path → 403, missing secret → 404, auth failure → 401. Every attempt (ok/denied/error) is audited with `user_id`, `app_registration_uuid`, `agent_id`, `action: read`, `path`, `result`. +- **REQ-08 Creds-source adapter.** The broker reads `creds-coreci-coo` exclusively through a pluggable adapter, `BROKER_CREDS_SOURCE=file|owui`: `FileCredsSource` reads a JSON document at `BROKER_CREDS_FILE` mapping paths to secret objects (default — dev/tests); `OwuiApiCredsSource` reads via the Open WebUI HTTP API using `BROKER_OWUI_BASE_URL` + `BROKER_OWUI_API_KEY` (production). The broker never writes to the KB. Response shape: `{path, data, version, updated_at}`. + +### Agent State + +- **REQ-09 State endpoints.** `GET/PUT /api/broker/agents//state` (App-auth). The broker enforces that the calling UUID and the token-derived user_id match the stored identity; 404 when no state exists. State schema per spec §8 (agent_id, app_registration_uuid, user_id, tags, pepper, current_token, token_expires_at, first_registered_at, last_seen_at). This is the tool's only persistence channel — the broker is the durability layer. +- **REQ-10 State backend toggle.** `BROKER_AGENT_STATE_BACKEND=internal` (default: rows in `broker.db` behind the state endpoints) `| owui` (real per-agent Open WebUI KBs via the OWUI API). The tool cannot tell the difference — the state endpoints are the only interface. + +### Audit + +- **REQ-11 Append-only audit log.** One JSON object per line at `BROKER_AUDIT_LOG_PATH` (mode 0600, append-only, no rotation in v0.1). Fields: `ts`, `user_id`, `app_registration_uuid`, `agent_id`, `action` (register/read/denied/login/logout/add_app/remove_app/add_user/remove_user), `path`, `result` (ok/denied/error); admin actions carry null user_id/uuid. +- **REQ-12 Audit query.** Admin-only `GET /api/broker/audit` with `since`, `until`, `user_id`, `app_registration_uuid`, `action` filters and `limit` (default 100, max 1000); returns entries plus `next_since` cursor. The broker streams the file line-by-line with filtering. + +### Web UI + +- **REQ-13 Four pages + dashboard.** Server-rendered HTML from `coreci_broker_ui/`, no build step, same port as the API. `/ui/login` (the CEO's bookmark), `/ui/app_registrations` (table + add-modal + revoke/reinstate per row), `/ui/user_allowlist` (same), `/ui/audit` (filterable, read-only, no export), plus `/ui/dashboard` and a persistent nav header with Logout. After each action the page refreshes via the corresponding API. + +### Ops & Bootstrap + +- **REQ-14 `.env` config + first boot.** The broker reads its `.env` at `BROKER_ENV_PATH` on boot (spec §5.1 contract plus the documented additions). On first boot it generates `BROKER_INSTANCE_PEPPER` (32 random bytes, base64 — synced to SQLite) and `BROKER_ADMIN_PASS` (32+ random chars), writes them to `.env`, and prints the admin pass to the startup log exactly once — never again. +- **REQ-15 Mode 0600.** `broker.db` and `broker.audit.log` are created mode 0600, owned by the Open WebUI process user. `.env.secrets` discipline (ciagent) mirrors this. +- **REQ-16 Persistence across restarts.** SQLite and the audit file are the durable state. Restarts preserve registrations, allowlist entries, sessions' data, and audit history; in-memory caches (verified-token cache) rebuild lazily; existing tokens still verify. +- **REQ-17 Single-file constraint.** `coreci_broker.py` is one Python file under **1400** lines (amended 800→1000 D-016, →1200 D-017, →1400 D-018 — owner-approved §12.14 amendment pattern, final projected), excluding tests, UI HTML, the client module, and reference tools. A line-count test enforces it (AC14). +- **REQ-18 Runtime modes.** Importable as an Open WebUI workspace tool (daemon-thread server + a `Tools` class surface) and runnable standalone (`python3 coreci_broker.py`). Stdlib-only at runtime; bcrypt is the sole optional import (D-001/D-009). Tests use unittest against an in-process server on an ephemeral port. + +### Client Ecosystem + +- **REQ-19 Public client module.** `coreci_broker_client.py` — dependency-free (stdlib urllib; no httpx, no bcrypt) — implements the tool-side bootstrap of spec §9: ensure-registered (idempotent), state get/put, secret read with 401 → re-register → retry-once, and never assumes the pepper key is present in register responses (`data.get("pepper")`). Served unauthenticated at `/static/coreci_broker_client.py` so any OWUI tool author can fetch it into their workspace and import it. It contains no secrets. +- **REQ-20 Reference tools.** `reference_tools/` ships at least: (a) a proxmox-style reference implementing §11's flow — first call registers and persists state, subsequent calls read the token from state and fetch `coreci/proxmox`, downstream call defaults to a safe dry-run; (b) a minimal showcase of auth-first-then-use-a-second-tool. These are templates for tool authors, wired to the test broker in tests. + +## v2 (Deferred) + +None defined. Future candidates held as out-of-scope decisions, not requirements: audit rotation with retention, multiple broker instances, approver-agent delegation, OWUI-KB credential sync. + +## Out of Scope + +| Feature | Reason | +|---------|--------| +| Multi-instance federation | Spec §3 — one broker on the Open WebUI host | +| Secret versioning / history | Spec §3 — PUT overwrites; broker is read-only on the KB | +| High availability | Spec §3 — SQLite is the source of truth; back it up | +| Web UI beyond the four pages + dashboard | Spec §3, §10 | +| Broker writes to `creds-coreci-coo` | Spec §3 — the CEO writes the KB via Open WebUI's UI | +| Audit log rotation | Spec §13 Q1 — CEO default: never rotate; manual `mv` when large | +| Multiple broker instances | Spec §13 Q2 — confirmed out of scope for v0.5 | +| Approver agents (delegated allowlist adds) | Spec §13 Q4 — CEO is sole approver | +| Key rotation ceremony | Spec §14 — pepper and tool UUIDs are stable | +| HSM integration for the pepper | Spec §14 | +| Rate limiting on auth endpoints | Spec §14 — assumes trusted network | +| Multi-factor admin auth | Spec §14 — single password sufficient | +| Email/notification on allowlist changes | Spec §14 | +| bcrypt as a hard dependency; pip-installable packages | D-001/D-009 — environment has no pip; stdlib-only runtime | + +## Traceability + +| Requirement | Phase | Status | +|-------------|-------|--------| +| REQ-01 | 2 | complete | +| REQ-02 | 3 | complete | +| REQ-03 | 1 | complete | +| REQ-04 | 3 | complete | +| REQ-05 | 2 | complete | +| REQ-06 | 2 | complete | +| REQ-07 | 4 | complete | +| REQ-08 | 4 | complete | +| REQ-09 | 3 | complete | +| REQ-10 | 3 | complete | +| REQ-11 | 1 | complete | +| REQ-12 | 2 | complete | +| REQ-13 | 2 | complete | +| REQ-14 | 1 | complete | +| REQ-15 | 1 | complete | +| REQ-16 | 1 | complete | +| REQ-17 | 1 | complete | +| REQ-18 | 1 | complete | +| REQ-19 | 4 | complete | +| REQ-20 | 4 | complete | \ No newline at end of file diff --git a/.ciagent/RESEARCH.md b/.ciagent/RESEARCH.md new file mode 100644 index 0000000..347a650 --- /dev/null +++ b/.ciagent/RESEARCH.md @@ -0,0 +1,82 @@ +# RESEARCH — CoreCI Broker + +Date: 2026-10-09 · Milestone: v0.1 · Phase: 0 (pre-execution) · Status: complete +Sources: `.ciagent/` PROJECT/REQUIREMENTS/ARCHITECTURE/ROADMAP, git log (init ffdc5392, clarify 53d99e80), docs.openwebui.com + open-webui source at `main` (fetched 2026-10-09). No scope changes; D-001..D-015 reconfirmed; no secrets read or printed. + +## 1. Environment Constraints (verified facts — do not re-verify) + +- Python 3.11.2 on the dev host; NO pip, NO bcrypt locally → D-001 dual-mode hashing active: stdlib `hashlib.scrypt` in dev/tests, bcrypt in the OWUI runtime; hashes format-prefixed (`$2b$…` / `scrypt$…`). Tests = stdlib unittest (D-009). Pure-Python repo, no package.json. +- Forge: gitea `git.coreci.dev`, org `coreci`; release target `coreci/broker` — repo created at first push/ship, public, default branch `main` (D-010/D-012). `GITEA_TOKEN` resolves from `.ciagent/.env.secrets` only (gitignored, mode 0600, never printed). +- Stdlib components carrying the runtime: `http.server.ThreadingHTTPServer`, `sqlite3`, `hashlib` (scrypt/sha256), `secrets`, `json`, `base64`, `urllib` (client module), `threading`. +- Repo state verified at research time: branches `main`, `milestone/v0.1-coreci-broker`, `phase/00-pre-execution`; `.gitignore` at root; `.ciagent/` holds PROJECT/REQUIREMENTS/ARCHITECTURE/ROADMAP/config.json/CHECKPOINT.json + `.env.secrets` (unread, 0600). + +## 2. Spec Defects & Resolutions (PROJECT.md §Context; decisions in init/clarify commits) + +| # | Defect | Resolution | Test implication | +|---|--------|------------|------------------| +| 1 | §7.5 in-memory `token_hash` index impossible with bcrypt (salted hashes neither comparable nor indexable) | D-004: UUID-scoped slow-hash scan + in-memory verified-token cache; SQLite authoritative, cache never authoritative | Cover both paths: cache-miss scan and cache-hit re-check of row status/expiry; both hash formats | +| 2 | §6 (tool derives every call) vs §9 (tool stores broker-returned token) contradiction | Broker derives authoritatively; tool stores via state endpoints (ARCHITECTURE.md DataFlow 2–4) | AC4: broker- and tool-side derivation byte-identical (§6 formula); state GET/PUT roundtrip | +| 3 | §9 snippet `KeyError` on `data["pepper"]` for re-registrations | D-005: pepper returned only on row-create or tokenless register; REQ-19 mandates `data.get("pepper")` in the client | Re-registration without a pepper key must not crash the client; register still audited | +| 4 | Pepper chicken-and-egg (token needed to read state; pepper needed to derive) | Tokenless register returns the pepper — the only KB-loss recovery path (D-005) | Recovery scenario: no valid token → register → pepper returned → tool can re-derive | + +## 3. Sibling Conventions (data provided — not fetched) + +- researchx: root = `.ciagent/`, `.gitignore`, `README.md`, `backend/`, `frontend/` — public repo, default branch main. +- nextcraft: root = `.ciagent/`, `apps/`, `packages/`, `deploy/`, `scripts/`, pnpm workspace, `Dockerfile`, `.gitea/` — public repo, default branch main. +- Implications: flat single-project `.ciagent/` (matches D-010); `README.md` at repo root expected at ship (P5); per-phase tags on the v0.1.x line (D-008); `---ci---` block in every commit; ships via gitea releases once the forge repo exists. + +## 4. Open WebUI Integration Surface (researched 2026-10-09; docs live, post-v0.10 era) + +### Workspace Tools — `Tools` class surface (docs: /features/extensibility/plugin/tools/development) +- Single Python file; frontmatter docstring keys: `title`, `author`, `author_url`, `git_url`, `description`, `required_open_webui_version`, `requirements`, `version`, `licence`. Broker ships `requirements:` empty — stdlib-only + bcrypt already present in the OWUI runtime (PROJECT.md Constraints). +- `Tools` class rules: `__init__` MUST set `self.valves = self.Valves()` (stored admin valves load only onto an instance that has a `valves` attribute); optional `Valves`/`UserValves` (pydantic BaseModel); every public callable is exposed to the model → helpers must be `_`-prefixed; method docstrings (reST `:param name:` form) + type hints generate the JSON schema; return str/dict/list; async methods preferred. +- **Lifecycle: there is NO `activate()`/`deactivate()` hook for Tools.** The loader (`backend/open_webui/utils/plugin.py`) execs the source into a `tool_{id}` module in `sys.modules`, instantiates `module.Tools()` at load (create/update) and caches it on `app.state.TOOLS`; re-exec occurs on source change; module-top-level side effects persist, and disabling a tool does NOT undo them. ⇒ ARCHITECTURE.md's "mounts the server at activation" maps to constructor/module-load time: start the daemon-thread `ThreadingHTTPServer` in `__init__` (or module top level) behind a once-per-process idempotency guard. +- Loader sets the module's `__file__` to a temp file deleted after exec ⇒ the broker must resolve DB/audit/env paths from config/env vars, never from `__file__`. +- User context: OWUI provides no `self.user` attribute; the per-call reserved arg `__user__` is a dict (`id`, `email`, `name`, `role`, plus `valves` when UserValves defined) (docs: /features/extensibility/plugin/development/reserved-args). Broker admin auth (`BROKER_ADMIN_PASS`) is independent of OWUI identity; ARCHITECTURE.md's "self.user supplies admin-agnostic context" is the broker's own convention — the broker's Tools class can stash `__user__` or ignore it. Nuance only, no architecture change. +- Background infra: OWUI's documented pattern is `asyncio.create_task` (Under the Hood §Patterns 4); the broker's stdlib analog is `threading.Thread(target=serve_forever, daemon=True)` — same once-per-process guard required (module re-exec on edits). +- Asset serving: a tool may register FastAPI routes via `app.add_api_route` with an idempotency guard (Under the Hood §Patterns 3) — but such routes are unauthenticated unless gated. The broker deliberately serves UI + `/static/` from its OWN `ThreadingHTTPServer` on `127.0.0.1:8765` — no FastAPI coupling; `/static/coreci_broker_client.py` is intentionally unauthenticated (REQ-19). +- Security posture: workspace tools execute unsandboxed Python in the server process (docs Plugin Security Warning) — the broker mount inherits that trust boundary. + +### Knowledge Bases API — read path (source-verified: `backend/open_webui/routers/knowledge.py`, `files.py` at main, 2026-10-09) +- Auth: `Authorization: Bearer ` (sk-… key from Settings→Account; JWT also accepted; `x-api-key` alt header via `CUSTOM_API_KEY_HEADER`) — feeds `BROKER_OWUI_BASE_URL` + `BROKER_OWUI_API_KEY` (REQ-08/D-002). Exact layout deferred to P4 per ARCHITECTURE.md §Notes; candidates below. +- Router `/api/v1/knowledge` (all endpoints `get_verified_user`): + - `GET /api/v1/knowledge/?page=N` — paginated KB list (30/page) → `{items:[{id,name,description,write_access,…}], total}` + - `GET /api/v1/knowledge/search?query=&page=` — same shape; candidate for finding `creds-coreci-coo` by name + - `GET /api/v1/knowledge/{id}` — KB detail + `files[]` (per-file metadata) + `write_access` + - `GET /api/v1/knowledge/{id}/files?query=&include_content=&page=` — file list; `include_content=true` embeds extracted content (docs flag it as expensive) + - `GET /api/v1/knowledge/{id}/files/pending`, `GET /api/v1/knowledge/{id}/export` (zip) — auxiliary + - Write ops exist (`/create`, `/{id}/update`, `/{id}/file/add|update|remove|move`, `/{id}/files/batch/add`, `/{id}/delete`, `/{id}/reset`, `/{id}/sync/*`, `/{id}/dirs/*`) — the broker never calls them (read-only, REQ-08). +- Router `/api/v1/files`: + - `GET /api/v1/files/{id}` → FileModel; `data.content` holds extracted text + - `GET /api/v1/files/{id}/data/content` → `{"content": ""}` — JSON form, best fit for credential JSON files + - `GET /api/v1/files/{id}/content` → raw bytes (FileResponse) + - `GET /api/v1/files/{id}/process/status` → `{"status": "pending"|"completed"|"failed"}` + - `GET /api/v1/files/?page=` (`content=true` default), `GET /api/v1/files/search?filename=` +- Candidate P4 flow: search KB by name → map path→file (filename convention TBD in P4) → `GET /api/v1/files/{file_id}/data/content` → parse JSON → `{path, data, version, updated_at}`. Note: OWUI exposes `meta.size`/`updated_at` but no version field → `version` may be synthesized broker-side (P4 detail; REQ-08 response shape unchanged). + +## 5. Security Prior Art + +- Peppered derivation + slow-hash storage: the token derives from a 256-bit random pepper (`BROKER_INSTANCE_PEPPER`, 32 bytes), so an attacker holding `token_hash` faces an offline brute force over the pepper — infeasible. Online guessing is rate-limit-free (spec §14: trusted network + `127.0.0.1` bind), but each guess costs one scrypt/bcrypt op, and the UUID-scoped scan bounds verification work (D-004). Storing only the salted slow hash (never the plaintext token) follows OWASP password-storage guidance; format prefixes keep verification format-aware (D-001). +- Append-only JSONL audit: one JSON object per line; flush-per-write (+ `os.fsync` for the process-restart case) is affordable at the broker's write volume (register/read/admin actions) and preserves the no-loss guarantee; file mode 0600; no rotation — manual `mv` (D-007, spec §13 Q1). Exact flush/fsync choice is a P1 plan decision, not research. +- Session cookie `broker_session`: HttpOnly + SameSite=Lax (D-013); no Secure flag (loopback HTTP by default); Path scoped to the broker. +- Path traversal on `/secrets/`: URL-decode (`urllib.parse.unquote`) BEFORE validation; reject `..` segments, absolute paths, backslashes, NUL; the scope check `path == "coreci/" + agent_id` runs on the decoded value (REQ-07). Static serving of `/static/` and UI must likewise resolve against a fixed root and reject traversal. +- OWUI-runtime boundary: broker binds `127.0.0.1:8765` — bind address controls reachability (PROJECT.md Constraints). The OWUI process hosts both the tool (broker mount) and the KB the broker reads; `BROKER_OWUI_API_KEY` is the only OWUI-side credential the broker holds, and it lives in the broker `.env` (0600 discipline, REQ-15). + +## 6. Testing Prior Art + +- In-process server harness (stdlib, matches D-009/REQ-18): `ThreadingHTTPServer(("127.0.0.1", 0), Handler)` for an ephemeral port in `setUp`; `threading.Thread(target=server.serve_forever, daemon=True)`; drive requests with `urllib.request` against `http://127.0.0.1:{port}`; `shutdown()` + `server_close()` via `addCleanup`. +- Temp state: `tempfile.TemporaryDirectory()` per test for `broker.db`, `broker.audit.log`, `.env`, `data/creds.json`; point the broker at them via env vars (`BROKER_ENV_PATH`, `BROKER_CREDS_FILE`, audit path). +- 0600 modes: `os.stat(path).st_mode & 0o777 == 0o600` after first boot; create with `os.open(path, O_CREAT|O_WRONLY, 0o600)` (umask-independent) in the broker, so the test just asserts. +- AC14 line-count test (REQ-17): `coreci_broker.py` is a single file — tests/UI HTML/client module/reference tools are separate files, so a plain line count of that one file is the spec's measure (< 800, spec §12.14); test reads the file and asserts `sum(1 for _ in f) <= 800` (wc -l-style, no comment stripping). +- TTL tests: set `BROKER_TOKEN_TTL` to 1–2 s and `time.sleep`, or inject an expired `token_expires_at` directly in SQLite (faster, deterministic). +- Dual-hash tests: both `scrypt$…` and `$2b$…` verification paths; CI can only execute the scrypt path (no bcrypt locally) — the bcrypt path's format-awareness is tested via the prefix and mocked/verified against the format contract. + +## 7. Research Conclusions + +- P1 (skeleton): bootstrap `.env` + first-boot generation (pepper/admin pass printed exactly once — log once, never again on second boot); SQLite schema per spec §5.2 at 0600; audit writer (JSONL, 0600); dual-mode hasher with format prefixes; token engine pure-function of (pepper, inputs); ThreadingHTTPServer + `{"error, message}` router; OWUI mount = `Tools.__init__` starts the daemon-thread server behind a once-per-process guard (no activate() hook exists), `self.valves = self.Valves()` required, `_`-prefixed helpers; standalone `__main__`; path resolution from env, never `__file__`; keep the 800-line budget from the first commit (UI/client/reference are separate files). +- P2 (admin plane): cookie `broker_session` HttpOnly+SameSite=Lax; HTTP Basic + session cookie both accepted on all admin API endpoints; four pages + dashboard served from `coreci_broker_ui/` on the same port; audit query streams the file line-by-line with `next_since` cursor (limit default 100/max 1000). +- P3 (tool auth): register semantics per D-005 (idempotent re-derive, overwrite on input change, pepper on row-create-or-tokenless); verification per D-004 (UUID-scoped slow-hash scan; cache hit re-checks row status/expiry); `GET /agents/` per D-015 (metadata-only); state endpoints with identity enforcement; `BROKER_AGENT_STATE_BACKEND=internal` default. +- P4 (secrets + ecosystem): `OwuiApiCredsSource` builds on the KB read endpoints in §4 (Bearer key; `/{id}` + `/{id}/files` for mapping, `/api/v1/files/{id}/data/content` for content); `FileCredsSource` default `data/creds.json` (D-014); client module stdlib-urllib-only, `data.get("pepper")`, served unauthenticated at `/static/`; reference tools with safe dry-run default. +- P5 (review + ship): AC1–AC13 behavioral + AC14 line-count; security checklist (traversal incl. URL-decoded forms, pepper never re-printed, 0600 modes, token scoping, audit completeness); README with documented deviations; ship to gitea (repo exists before first push). +- ARCHITECTURE.md verification: **no assumption changes required.** The adapter-contract note (OWUI KB-file layout resolved in P4, ARCHITECTURE.md §Notes) stands — candidate endpoints captured in §4. One nuance for P1, not a defect: "mounts the server at activation" maps to constructor-time in the actual OWUI loader (no activate()/deactivate() lifecycle for Tools), and OWUI supplies `__user__` per call rather than a `self.user` attribute — the broker's Tools class owns that convention. +- No genuine blockers found. Environment, OWUI surface, and forge conventions all support the v0.1 scope as specified. \ No newline at end of file diff --git a/.ciagent/ROADMAP.md b/.ciagent/ROADMAP.md new file mode 100644 index 0000000..3c4342a --- /dev/null +++ b/.ciagent/ROADMAP.md @@ -0,0 +1,60 @@ +# Roadmap — Milestone v0.1 (coreci-broker) + +## Overview + +One milestone, owner-directed 2026-10-09: implement Requirements Specification v0.5 in full. **Milestone v0.1 — coreci-broker** (feature type): the single-file broker, its SQLite store, audit log, four-page admin UI, tool-auth plane, secrets mediation, public client module, and reference tools, with the spec's §12 acceptance matrix (AC1–AC14) green. The spec's "v0.5" is its requirements-doc revision label, not the product version (D-008). + +Versioning (greenfield deviation, researchx/nextcraft precedent): the milestone branch is `milestone/v0.1-coreci-broker`; phase tags run on the milestone's own line — phase 0 ships `v0.1.0`, execution phases ship `v0.1.1`..`v0.1.4`, and the final phase's patch (`v0.1.5`) IS the milestone release; no separate minor tag. MAJOR remains 0 until an owner-approved v1 release. Ships go to gitea (`git.coreci.dev/coreci/broker`); the forge repo must exist before the first push. + +Autonomy: **guided** — escalate on every decision gate; threshold 0.85; 1 revision iteration; clarify budget 20. + +## Phase 0 — Pre-Execution + +- **Description**: Specification parsing, clarification (decisions D-001..D-011 recorded in the init commit), `.ciagent/` reference files, and the pre-execution pipeline stages: specify → clarify → research → plan → grill. Init has landed the reference files; `/ci-run` phase 0 completes RESEARCH.md (environment constraints, spec defects, sibling conventions), PLAN.md (task-level breakdown), PERSONAS.md, and the GRILL verdict before any code. +- **Requirements covered**: none (planning artifacts) +- **Success criteria**: `.ciagent/` files reflect the 6-phase structure; REQ-01..REQ-20 defined and traceable; milestone v0.1 established in config.json and PROJECT.md; PLAN.md + PERSONAS.md committed; grill verdict recorded; phase tagged `v0.1.0` +- **Depends on**: — +- **Status**: complete + +## Phase 1 — Core Skeleton + +- **Description**: `coreci_broker.py` foundation: `.env` read/write + first-boot generation (pepper + admin pass, printed once), SQLite schema + indexes per spec §5.2, mode-0600 on DB and audit log, append-only JSONL audit writer per §5.3, dual-mode bcrypt/scrypt hasher (D-001), server-side token derivation per §6, stdlib `ThreadingHTTPServer` + JSON router + `{"error, message}` contract per §7, Open WebUI `Tools`-class mount (daemon thread) + standalone `__main__` entry. +- **Requirements covered**: REQ-03, REQ-11, REQ-14, REQ-15, REQ-16, REQ-17, REQ-18 +- **Success criteria**: broker boots standalone on an ephemeral port, creates DB + audit log at 0600, prints the generated admin pass exactly once (not on second boot); token derivation matches the §6 formula byte-for-byte; source under 800 lines +- **Depends on**: 0 +- **Status**: complete + +## Phase 2 — Admin Plane + +- **Description**: Session auth (cookie `broker_session` + HTTP Basic both accepted), `POST /api/broker/auth/login`, `logout`, `whoami` (§7.1); app_registrations list/add/revoke/reinstate (§7.2, UUID format validation, 409 on duplicate, revoke keeps existing tokens until TTL); user_allowlist list/add/revoke/reinstate (§7.3); audit query with filters + `next_since` cursor (§7.6); the four UI pages + dashboard + persistent nav (§10) served from `coreci_broker_ui/`; session cleanup. +- **Requirements covered**: REQ-01, REQ-05, REQ-06, REQ-12, REQ-13 +- **Success criteria**: AC1 (first-boot login with the printed pass), AC2 (app registration add/remove via UI, persisted in `app_registrations`, UI updates after each action), AC3 (user allowlist same), AC9's login/logout audit lines +- **Depends on**: 1 +- **Status**: complete + +## Phase 3 — Tool Auth Plane + +- **Description**: `POST /api/broker/agents/register` (§7.4): allowlist gates → 403 on unknown/revoked UUID or user; idempotent re-registration re-derives from request inputs and overwrites on input change (D-005); pepper returned on row-create or when the caller presents no valid token; TTL set from `BROKER_TOKEN_TTL`. App-auth verification: parse `Authorization: App `, UUID-scoped slow-hash scan + in-memory verified-token cache (D-004). `GET /api/broker/agents/` (§7.4). State endpoints `GET/PUT /api/broker/agents//state` (§9.1) with identity enforcement (calling UUID + token-derived user_id must match stored identity) and the backend toggle `BROKER_AGENT_STATE_BACKEND=internal` (default) `| owui` (D-003). +- **Requirements covered**: REQ-02, REQ-04, REQ-09, REQ-10 +- **Success criteria**: AC4 (token derived + stored as slow-hash, same value computed both sides), AC7 (revoked/unknown UUID → 403), AC8 (unknown user → 403), AC10 (TTL expiry → 401 → re-register → new token, audited), AC12 (state read/write with identity enforcement) +- **Depends on**: 2 +- **Status**: complete + +## Phase 4 — Secrets + Client Ecosystem + +- **Description**: Creds-source adapter (D-002): `FileCredsSource` (default; JSON document at `BROKER_CREDS_FILE` mapping paths to secret objects) and `OwuiApiCredsSource` (`BROKER_OWUI_BASE_URL` + `BROKER_OWUI_API_KEY`). `GET /api/broker/secrets/` (§7.5): App-auth, scope check `path == "coreci/" + agent_id` → 403 on mismatch, 404 when absent, audit every attempt. Public client module `coreci_broker_client.py` (stdlib urllib only): ensure-registered, state get/put, secret read with 401 → re-register → retry-once; served unauthenticated at `/static/coreci_broker_client.py`. Reference OWUI tools (`reference_tools/`): proxmox reference + a minimal auth-then-use showcase, safe dry-run default (§9, §11; D-006). +- **Requirements covered**: REQ-07, REQ-08, REQ-19, REQ-20 +- **Success criteria**: AC5 (valid token reads the secret; tampered token → 401), AC6 (proxmox-registered tool reading `coreci/gitea` → 403); client module fetchable unauthenticated and drives a reference tool end-to-end against a test broker +- **Depends on**: 3 +- **Status**: complete + +## Phase 5 — Final Review + Ship + +- **Description**: Full §12 acceptance matrix as tests (AC1–AC13 as behavioral tests, AC14 as a line-count test); security review pass (path traversal on ``, pepper never re-printed, 0600 modes, token scoping, audit completeness); README with run/deploy notes and documented spec deviations, including the owner-run post-ship live-OWUI smoke checklist (mount/register/read on the deployed v0.11.3 host — G-005/G-006); multi-persona review of the milestone diff; owner approval gate; milestone merge to main and tag `v0.1.5` (final phase patch = milestone release); first push to gitea once the forge repo exists. +- **Requirements covered**: REQ-01..REQ-20 (acceptance) +- **Success criteria**: AC1–AC14 all green (`python3 -m unittest discover -s tests`); security review PASS; README complete with deviations; owner approval recorded; milestone squash-merged to main, tagged `v0.1.5`, release created +- **Depends on**: 4 +- **Status**: complete +## Post-Milestone Notes (owner-promoted from GRILL G-008) + +- **90-day adoption criterion**: `proxmox_readonly_tool.py` runs on `coreci_broker_client.py` with direct KB reads removed by **2027-01-09**; the owner owns the date. No real tool migrates to the broker before the post-ship OWUI smoke checklist (G-005) passes on the deployed v0.11.3 host. diff --git a/.ciagent/config.json b/.ciagent/config.json new file mode 100644 index 0000000..b624003 --- /dev/null +++ b/.ciagent/config.json @@ -0,0 +1,54 @@ +{ + "version": "0.8.0", + "autonomy": { + "level": "guided", + "decision_confidence_threshold": 0.85, + "escalation_hooks": ["*"], + "max_revision_iterations": 1, + "clarify_budget": 20 + }, + "release": { + "forge": "gitea", + "base_url": "https://git.coreci.dev", + "owner": "coreci", + "repo": "broker" + }, + "secrets": { + "scopes": ["gitea"], + "env_files": [".env", ".env.secrets", ".env.*"] + }, + "ship": { + "per_phase": true, + "allow_skip": false, + "max_release_retries": 3, + "release_blocking": false + }, + "verification": { + "bdd_default": false + }, + "personas": { + "enabled": true, + "territory_enforcement": "warn" + }, + "parallelization": { + "enabled": false, + "max_concurrent_agents": 1 + }, + "ideation": { + "max_ideas": 20, + "categories": ["security", "quality", "architecture", "coverage", "improvement", "spec", "chaos", "bdd"] + }, + "security": { + "bash_allowlist": { + "blocked_env_vars": ["GITEA_TOKEN", "GITHUB_TOKEN", "GITLAB_TOKEN"] + } + }, + "projects": [], + "active_project": null, + "milestone": { + "version": "v0.1", + "name": "coreci-broker", + "type": "feature", + "branch": "milestone/v0.1-coreci-broker" + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0736f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +# Secrets (ciagent credential discipline) +.env +.env.secrets +.env.* + +# Python +__pycache__/ +*.pyc + +# Broker runtime state +data/ +*.db +*.log \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a5666fd --- /dev/null +++ b/README.md @@ -0,0 +1,180 @@ +# CoreCI Broker + +A single-file credential broker for autonomous agent tools on the Open WebUI +host. Tools authenticate with derived tokens instead of shared secrets, the +CEO gates access through UI-managed allowlists of app UUIDs and user IDs, +every read lands in an append-only audit log, and secret reads are path-scoped +to `coreci/` — one Python file, one SQLite DB, a four-page admin UI, +zero pip installs, no build step. + +## Quickstart (standalone) + +```bash +python3 coreci_broker.py +``` + +- Stdlib-only runtime, Python 3.11+. Zero pip installs: bcrypt when + importable, stdlib scrypt otherwise (format-prefixed hashes). +- First boot generates `.env`: a 32-byte base64 pepper plus a 32+-char admin + password. The password is printed to the startup log as `BROKER_ADMIN_PASS=…` + **exactly once** — save it immediately; it is never printed again. Lost pass: + remove `BROKER_ADMIN_PASS` from `.env` and reboot — a new one prints once. +- The server binds `127.0.0.1:8765`. Open `http://127.0.0.1:8765/ui/login` and + log in with the printed pass. +- Stop with Ctrl+C. Durable state is exactly three files: `broker.db`, + `broker.audit.log`, `.env` — back them up. + +## Configuration + +Read on boot from the `.env` at `BROKER_ENV_PATH`. Precedence: +**process env → `.env` → defaults**. + +| Variable | Default | Purpose | +|----------|---------|---------| +| `BROKER_BIND` | `127.0.0.1:8765` | Listen address; the bind controls reachability | +| `BROKER_ADMIN_USER` | `admin` | Admin login user name | +| `BROKER_ADMIN_PASS` | generated first boot | Admin login password (32+ random chars) | +| `BROKER_INSTANCE_PEPPER` | generated first boot | 32-byte base64 pepper; token derivation input, synced to SQLite | +| `BROKER_ENV_PATH` | `.env` | Path of the `.env` file the broker reads/writes | +| `BROKER_TOKEN_TTL` | `86400` | Tool token lifetime, seconds (24 h) | +| `BROKER_SESSION_TTL` | `300` | Admin UI session cookie lifetime, seconds | +| `BROKER_DB_PATH` | `broker.db` | SQLite database (mode 0600) | +| `BROKER_AUDIT_LOG_PATH` | `broker.audit.log` | Append-only JSONL audit log (mode 0600) | +| `BROKER_CREDS_SOURCE` | `file` | Credential source adapter: `file` (dev/tests) or `owui` (production) | +| `BROKER_CREDS_FILE` | `data/creds.json` | `FileCredsSource` JSON document: paths → secret objects | +| `BROKER_OWUI_BASE_URL` | — | Open WebUI base URL for the OWUI creds source | +| `BROKER_OWUI_API_KEY` | — | Open WebUI API key (Bearer) for KB read access | +| `BROKER_AGENT_STATE_BACKEND` | `internal` | Agent-state storage: `internal` (rows in broker.db) or `owui` | +| `BROKER_MAX_AGENTS_PER_APP` | `50` | Cap on agent registrations per app UUID | +| `BROKER_UI_ROOT` | `coreci_broker_ui` | Directory served for `/ui/*` pages | +| `BROKER_STATIC_ROOT` | `.` | Directory served for `/static/*` (client module) | + +Production: `BROKER_CREDS_SOURCE=owui` with `BROKER_OWUI_BASE_URL` + +`BROKER_OWUI_API_KEY` (0600 `.env` discipline). Keep `BROKER_BIND` on +loopback unless you deliberately widen reachability. + +## Running as an Open WebUI workspace tool + +Paste `coreci_broker.py` into Open WebUI as a workspace Tool (frontmatter +`requirements:` stays empty — the runtime already ships everything). Notes +that matter: + +- The server starts in `Tools.__init__` behind a once-per-process guard; the + OWUI loader has no activate/deactivate hook, and disabling a tool does not + stop a previously started server. +- **After editing the broker tool source, restart Open WebUI.** The loader + re-execs the module on source change, but the previously loaded server keeps + its port — the new instance fails loud with a bind-error message telling you + to restart, rather than silently serving old code. +- Paths (DB, audit log, `.env`, UI roots) resolve from environment config, + never from `__file__` — the OWUI loader deletes the temp file it execs. +- Durable state: `broker.db` + `broker.audit.log` + `.env` — **back those + three up**; SQLite is the source of truth, no HA, restore is a file copy. +- `Tools.status()` reports the serving address; call it from a chat to verify + the mount, or hit `/healthz` on the bind. + +## Admin UI + +Four pages + dashboard, server-rendered on the same port as the API: + +- `/ui/login` — the CEO's bookmark. First-boot pass discipline: the printed + pass is shown once; record it in your secret store on first login. +- `/ui/app_registrations` — allowlist of tool app UUIDs: add, revoke, reinstate. +- `/ui/user_allowlist` — allowlist of user IDs (optional email): same actions. +- `/ui/audit` — filterable, read-only view of the audit log (no export in v0.1). +- `/ui/dashboard` — landing page with persistent nav header and Logout. + +Allowlist semantics: **revoke keeps existing tokens working until TTL expiry — +by design.** Revocation immediately blocks new registrations; already-issued +tokens live at most one `BROKER_TOKEN_TTL` (24 h by default). + +## Tool authors + +Fetch the public client module — unauthenticated, from +`http:///static/coreci_broker_client.py` — into your workspace and +import it. Flow: `ensure_registered(state)` → `put_state(state)` → +`read_secret("coreci/", state)`. Stdlib-urllib-only (no httpx, no +bcrypt), so any tool can import it; on a 401 it re-registers and retries +once. State endpoints are the tool's only persistence channel — keep `state` +there, not in local files. See `reference_tools/` for the proxmox-style §11 +flow and a minimal auth-then-use showcase; downstream calls default to a +safe dry-run. + +## Security model + +- Derived token: `base64(sha256(pepper:UUID:user_id:agent_id:sorted_tags))` — + 44 ASCII chars, no prefix, no challenge protocol. Stored only as a slow + hash (`$2b$…` / `scrypt$…`), never in plaintext. +- Allowlists are the gate: unknown or revoked app UUIDs and unknown users get + 403 at registration; the token is the consequence, not the authority. +- Every secret-read attempt is audited (ok/denied/error) with user, UUID, + agent, and path. +- Secret reads are path-scoped: a registration reads exactly + `coreci/` — cross-path → 403, absent → 404. +- The broker never writes to `creds-coreci-coo`; the CEO maintains that KB + through the Open WebUI UI. +- `broker.db`, `broker.audit.log`, and `.env` are mode 0600. +- Binds `127.0.0.1` by default; trusted-network posture — no rate limiting by + design (spec §14). + +## Documented spec deviations + +Honest list; full detail in `.ciagent/PROJECT.md`: + +- **Line-cap amendments (D-016/D-017/D-018)**: the single-file constraint grew + 800 → 1000 → 1200 → 1400 lines via the owner-approved §12.14 amendment + pattern (readability over dense cramming). Current source: 1390 lines; AC14 + enforces the cap. +- **Dual-mode hashing (D-001)**: spec §6 mandates bcrypt; the runtime is + stdlib-only, so scrypt fallback with format-prefixed hashes. +- **Token verification (D-004)**: spec §7.5's in-memory `token_hash` index is + impossible with salted slow hashes; the broker scans rows scoped to the + presented UUID and keeps a non-authoritative verified-token cache. +- **Pepper rule (D-005)**: the pepper is returned only on row creation or + tokenless registration (the KB-loss recovery path); spec §9's snippet would + raise `KeyError` on re-registration. +- **Spec defects resolved**: §7.5 index, §6-vs-§9 (the broker derives + authoritatively; the tool stores), §9's `KeyError` — see PROJECT.md §Context. +- `BROKER_ADMIN_USER` defaults to `admin` (spec silent on a default). +- Register cap `BROKER_MAX_AGENTS_PER_APP` (50) — a documented addition. +- Reinstate actions audit as `add_app`/`add_user` (no separate audit action). +- In-scope but absent secrets audit as `read` with `result: error` (404s are + audited, not silent). + +## Post-ship live-OWUI smoke checklist (owner-run) + +Before migrating any real tool (G-005; deployed host pinned v0.11.3, G-006): + +1. Confirm the deployed Open WebUI version (expected **v0.11.3**) and that the + KB read endpoints match the researched surface: `/api/v1/knowledge/{id}` and + `/api/v1/files/{id}/data/content`. +2. Add the broker as a workspace tool; check `/healthz` on the bind. +3. CEO first-boot login at `/ui/login` with the printed pass. +4. Add one app UUID and one user to the allowlists via the UI. +5. In a scratch tool: import the client module from `/static/` → + `ensure_registered` → read `coreci/proxmox` via the OWUI creds source. +6. Verify every step in `/ui/audit`. +7. Rollback = disable the tool + restore the three state files. + +No real tool migrates before this passes; the 90-day adoption criterion +(proxmox tool on the client, direct KB reads removed) is due 2027-01-09. + +## Tests + +```bash +timeout 300 python3 -m unittest discover -s tests +``` + +295 tests, stdlib unittest only (~2 min; one skip where bcrypt is not +importable). The AC1–AC14 acceptance matrix is green. + +## Repository layout + +| Path | What it is | +|------|------------| +| `coreci_broker.py` | The broker: single file — server, store, audit, auth, UI serving | +| `coreci_broker_client.py` | Public stdlib-only client module (served at `/static/`) | +| `coreci_broker_ui/` | Server-rendered HTML for the four pages + dashboard | +| `reference_tools/` | Tool-author templates (proxmox-style §11 flow + minimal showcase) | +| `tests/` | Stdlib-unittest suite incl. the AC1–AC14 acceptance matrix | +| `.ciagent/` | CI pipeline artifacts (project, requirements, roadmap, decisions) | \ No newline at end of file diff --git a/coreci_broker.py b/coreci_broker.py new file mode 100644 index 0000000..87a0d83 --- /dev/null +++ b/coreci_broker.py @@ -0,0 +1,1390 @@ +"""title: CoreCI Broker +author: coreci +description: Single-file credential mediation broker for CoreCI agent tools. +version: 0.1.0 +requirements: [] +licence: MIT +""" +import base64 +import datetime +import errno +import hashlib +import hmac +import http.cookies +import json +import os +import secrets +import sqlite3 +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +try: + import bcrypt +except ImportError: + bcrypt = None + +ENV_PATH_DEFAULT = ".env" + +ENV_KEYS = ( + "BROKER_ENV_PATH", + "BROKER_BIND", + "BROKER_ADMIN_USER", + "BROKER_ADMIN_PASS", + "BROKER_INSTANCE_PEPPER", + "BROKER_TOKEN_TTL", + "BROKER_SESSION_TTL", + "BROKER_DB_PATH", + "BROKER_AUDIT_LOG_PATH", + "BROKER_UI_ROOT", + "BROKER_STATIC_ROOT", + "BROKER_AGENT_STATE_BACKEND", + "BROKER_CREDS_SOURCE", + "BROKER_CREDS_FILE", + "BROKER_OWUI_BASE_URL", + "BROKER_OWUI_API_KEY", + "BROKER_MAX_AGENTS_PER_APP", +) + +ENV_DEFAULTS = { + "BROKER_BIND": "127.0.0.1:8765", + "BROKER_ADMIN_USER": "admin", + "BROKER_TOKEN_TTL": "86400", + "BROKER_SESSION_TTL": "300", + "BROKER_DB_PATH": "broker.db", + "BROKER_AUDIT_LOG_PATH": "broker.audit.log", + "BROKER_UI_ROOT": "coreci_broker_ui", + "BROKER_STATIC_ROOT": ".", + "BROKER_AGENT_STATE_BACKEND": "internal", + "BROKER_CREDS_SOURCE": "file", + "BROKER_CREDS_FILE": "data/creds.json", + "BROKER_MAX_AGENTS_PER_APP": "50", +} + +AUDIT_ACTIONS = ( + "register", + "read", + "denied", + "login", + "logout", + "add_app", + "remove_app", + "add_user", + "remove_user", +) + +AUDIT_RESULTS = ("ok", "denied", "error") + +BCRYPT_PREFIXES = ("$2a$", "$2b$", "$2y$") +SCRYPT_PREFIX = "scrypt$" +SCRYPT_SALT_BYTES = 16 +SCRYPT_N = 2 ** 14 +SCRYPT_R = 8 +SCRYPT_P = 1 + +STATIC_PREFIXES = ( + ("ui", "ui_root"), + ("static", "static_root"), +) + +STATIC_CONTENT_TYPES = { + ".html": "text/html; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".py": "text/x-python; charset=utf-8", + ".txt": "text/plain; charset=utf-8", + ".json": "application/json; charset=utf-8", +} + +MAX_BODY_BYTES = 1024 * 1024 + +MAX_STATE_BYTES = 256 * 1024 + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS broker_config ( + key TEXT PRIMARY KEY, + value TEXT +); +CREATE TABLE IF NOT EXISTS app_registrations ( + uuid TEXT PRIMARY KEY, + name TEXT, + status TEXT DEFAULT 'active', + created_at TEXT, + revoked_at TEXT +); +CREATE TABLE IF NOT EXISTS user_allowlist ( + user_id TEXT PRIMARY KEY, + email TEXT, + status TEXT DEFAULT 'active', + created_at TEXT, + revoked_at TEXT +); +CREATE TABLE IF NOT EXISTS agent_registrations ( + agent_id TEXT, + app_registration_uuid TEXT, + user_id TEXT, + tags TEXT, + token_hash TEXT, + token_expires_at TEXT, + first_registered_at TEXT, + last_seen_at TEXT, + PRIMARY KEY (agent_id, app_registration_uuid) +); +CREATE INDEX IF NOT EXISTS idx_agent_registrations_app_uuid + ON agent_registrations (app_registration_uuid); +CREATE TABLE IF NOT EXISTS admin_sessions ( + session_id TEXT PRIMARY KEY, + created_at TEXT, + expires_at TEXT +); +CREATE TABLE IF NOT EXISTS agent_state ( + agent_id TEXT, + app_registration_uuid TEXT, + state TEXT, + updated_at TEXT, + PRIMARY KEY (agent_id, app_registration_uuid) +); +""" + + +def _parse_env_file(path): + entries = [] + if not os.path.isfile(path): + return entries + with open(path, "r", encoding="utf-8") as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = (part.strip() for part in line.partition("=")) + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + if key: + entries.append((key, value)) + return entries + + +def _append_env_entries(path, entries): + needs_newline = os.path.exists(path) and os.path.getsize(path) > 0 + if needs_newline: + with open(path, "rb") as handle: + handle.seek(-1, os.SEEK_END) + needs_newline = handle.read(1) != b"\n" + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + try: + os.fchmod(fd, 0o600) + except BaseException: + os.close(fd) + raise + with os.fdopen(fd, "a", encoding="utf-8") as handle: + if needs_newline: + handle.write("\n") + for key, value in entries: + handle.write(f"{key}={value}\n") + + +def _generate_pepper(): + return base64.b64encode(secrets.token_bytes(32)).decode("ascii") + + +def _generate_admin_pass(): + return secrets.token_urlsafe(32) + + +def _utc_now_iso(): + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def hash_token(token): + token_bytes = token.encode("utf-8") + if bcrypt is not None: + return bcrypt.hashpw(token_bytes, bcrypt.gensalt()).decode("ascii") + salt = secrets.token_bytes(SCRYPT_SALT_BYTES) + digest = hashlib.scrypt(token_bytes, salt=salt, n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P) + encoded = base64.urlsafe_b64encode + return f"{SCRYPT_PREFIX}{encoded(salt).decode('ascii')}${encoded(digest).decode('ascii')}" + + +def verify_token(token, stored): + if not isinstance(stored, str) or not stored: + return False + token_bytes = token.encode("utf-8") + if stored.startswith(BCRYPT_PREFIXES): + try: + return bcrypt is not None and bcrypt.checkpw(token_bytes, stored.encode("ascii")) + except (ValueError, TypeError): + return False + if stored.startswith(SCRYPT_PREFIX): + fields = stored[len(SCRYPT_PREFIX):].split("$") + if len(fields) != 2: + return False + try: + salt = base64.urlsafe_b64decode(fields[0].encode("ascii")) + expected = base64.urlsafe_b64decode(fields[1].encode("ascii")) + actual = hashlib.scrypt(token_bytes, salt=salt, n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P) + except (ValueError, TypeError): + return False + return hmac.compare_digest(actual, expected) + return False + + +def derive_token(pepper, app_uuid, user_id, agent_id, tags): + material = f"{pepper}:{app_uuid}:{user_id}:{agent_id}:{','.join(sorted(tags))}" + return base64.b64encode(hashlib.sha256(material.encode("utf-8")).digest()).decode("ascii") + + +def load_config(env=None): + if env is None: + env = os.environ + env_path = str(env.get("BROKER_ENV_PATH") or ENV_PATH_DEFAULT) + file_values = dict(_parse_env_file(env_path)) + generated = [] + if "BROKER_INSTANCE_PEPPER" not in env and "BROKER_INSTANCE_PEPPER" not in file_values: + generated.append(("BROKER_INSTANCE_PEPPER", _generate_pepper())) + if "BROKER_ADMIN_PASS" not in env and "BROKER_ADMIN_PASS" not in file_values: + generated.append(("BROKER_ADMIN_PASS", _generate_admin_pass())) + if generated: + _append_env_entries(env_path, generated) + file_values.update(generated) + config = {"BROKER_ENV_PATH": env_path} + for key in ENV_KEYS: + if key == "BROKER_ENV_PATH": + continue + config[key] = str(env.get(key) or file_values.get(key) or ENV_DEFAULTS.get(key, "")) + for key, value in generated: + if key == "BROKER_ADMIN_PASS": + print(f"BROKER_ADMIN_PASS={value}", flush=True) + return config + + +class Store: + def __init__(self, path): + self._lock = threading.Lock() + fd = os.open(path, os.O_CREAT | os.O_WRONLY, 0o600) + try: + os.fchmod(fd, 0o600) + finally: + os.close(fd) + self.conn = sqlite3.connect(path, check_same_thread=False) + self.conn.executescript(SCHEMA_SQL) + self._run( + "DELETE FROM admin_sessions WHERE expires_at IS NULL OR expires_at <= ?", + (_utc_now_iso(),), + ) + + def _run(self, sql, params=()): + with self._lock: + rows = self.conn.execute(sql, params).fetchall() + self.conn.commit() + return rows + + def set_config(self, key, value): + self._run("INSERT OR REPLACE INTO broker_config VALUES (?, ?)", (key, str(value))) + + def get_config(self, key): + rows = self._run("SELECT value FROM broker_config WHERE key = ?", (key,)) + return rows[0][0] if rows else None + + def create_admin_session(self, session_id, created_at, expires_at): + self._run( + "INSERT OR REPLACE INTO admin_sessions VALUES (?, ?, ?)", + (session_id, created_at, expires_at), + ) + + def get_admin_session(self, session_id): + rows = self._run("SELECT * FROM admin_sessions WHERE session_id = ?", (session_id,)) + return rows[0] if rows else None + + def delete_admin_session(self, session_id): + self._run("DELETE FROM admin_sessions WHERE session_id = ?", (session_id,)) + + def list_rows(self, table): + return self._run("SELECT * FROM %s ORDER BY created_at" % table) + + def get_row(self, table, key, value): + rows = self._run("SELECT * FROM %s WHERE %s = ?" % (table, key), (value,)) + return rows[0] if rows else None + + def add_row(self, table, value, field): + self._run( + "INSERT INTO %s VALUES (?, ?, 'active', ?, NULL)" % table, + (value, field, _utc_now_iso()), + ) + + def set_row_status(self, table, key, value, status): + self._run( + "UPDATE %s SET status = ?, revoked_at = ? WHERE %s = ?" % (table, key), + (status, _utc_now_iso() if status == "revoked" else None, value), + ) + + def get_agent(self, agent_id, app_uuid): + rows = self._run( + "SELECT * FROM agent_registrations WHERE agent_id = ? AND app_registration_uuid = ?", + (agent_id, app_uuid), + ) + return rows[0] if rows else None + + def agents_by_uuid(self, app_uuid): + return self._run( + "SELECT * FROM agent_registrations WHERE app_registration_uuid = ?", (app_uuid,) + ) + + def save_agent(self, agent_id, app_uuid, user_id, tags, token_hash, expires_at, now): + self._run( + "INSERT INTO agent_registrations VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (agent_id, app_uuid, user_id, tags, token_hash, expires_at, now, now), + ) + + def update_agent(self, agent_id, app_uuid, user_id, tags, token_hash, expires_at, now): + self._run( + "UPDATE agent_registrations SET user_id = ?, tags = ?, token_hash = ?, " + "token_expires_at = ?, last_seen_at = ? WHERE agent_id = ? AND app_registration_uuid = ?", + (user_id, tags, token_hash, expires_at, now, agent_id, app_uuid), + ) + + def touch_agent(self, agent_id, app_uuid, now): + self._run( + "UPDATE agent_registrations SET last_seen_at = ? WHERE agent_id = ? AND app_registration_uuid = ?", + (now, agent_id, app_uuid), + ) + + def get_agent_state(self, agent_id, app_uuid): + rows = self._run("SELECT state FROM agent_state WHERE agent_id = ? AND app_registration_uuid = ?", + (agent_id, app_uuid)) + return json.loads(rows[0][0]) if rows else None + + def save_agent_state(self, agent_id, app_uuid, state): + self._run("INSERT OR REPLACE INTO agent_state VALUES (?, ?, ?, ?)", + (agent_id, app_uuid, state, _utc_now_iso())) + + def close(self): + with self._lock: + self.conn.close() + + +class InternalAgentStateBackend: + def __init__(self, store): + self.store = store + + def get_state(self, agent_id, app_uuid): + return self.store.get_agent_state(agent_id, app_uuid) + + def put_state(self, agent_id, app_uuid, state): + self.store.save_agent_state(agent_id, app_uuid, json.dumps(state)) + + +class OwuiAgentStateBackend: + """Agent-state documents over HTTP: GET/PUT {base}/api/v1/broker-agent-state/{agent_id}.""" + + def __init__(self, base_url, api_key): + self.base_url = str(base_url or "").rstrip("/") + self.api_key = str(api_key or "") + + def _request(self, method, agent_id, payload=None): + url = self.base_url + "/api/v1/broker-agent-state/" + urllib.parse.quote(agent_id, safe="") + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + url, data=data, method=method, + headers={"Authorization": "Bearer %s" % self.api_key, "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return response.read() + + def get_state(self, agent_id, app_uuid): + try: + raw = self._request("GET", agent_id) + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + return json.loads(raw.decode("utf-8")) if raw else None + + def put_state(self, agent_id, app_uuid, state): + self._request("PUT", agent_id, state) + + +def _build_agent_state_backend(config, store): + if str(config.get("BROKER_AGENT_STATE_BACKEND") or "") == "owui": + return OwuiAgentStateBackend( + config.get("BROKER_OWUI_BASE_URL"), config.get("BROKER_OWUI_API_KEY") + ) + return InternalAgentStateBackend(store) + + +class _SecretNotFound(Exception): + pass + + +class _CredsSourceError(Exception): + pass + + +def _secret_response(path, data, updated_at=None): + serialized = json.dumps(data, sort_keys=True, separators=(",", ":")).encode("utf-8") + return {"path": path, "data": data, "version": hashlib.sha256(serialized).hexdigest()[:8], + "updated_at": updated_at} + + +class FileCredsSource: + def __init__(self, path): + self.path = str(path or "data/creds.json") + + def get_secret(self, path): + try: + with open(self.path, "r", encoding="utf-8") as handle: + document = json.load(handle) + except FileNotFoundError: + raise _SecretNotFound(path) + except (OSError, ValueError) as error: + raise _CredsSourceError(f"creds file unusable: {error}") + if not isinstance(document, dict): + raise _CredsSourceError("creds file must map paths to secret objects") + if path not in document: + raise _SecretNotFound(path) + return _secret_response(path, document[path]) + + +class OwuiApiCredsSource: + """Secrets from the `creds-coreci-coo` KB via the OWUI HTTP API; a KB file's + name is its secret path (e.g. `coreci/proxmox`), its content the secret JSON.""" + KB_NAME = "creds-coreci-coo" + + def __init__(self, base_url, api_key): + self.base_url = str(base_url or "").rstrip("/") + self.api_key = str(api_key or "") + self._file_map = None + + def _fetch_json(self, url_path): + try: + url = self.base_url + url_path + request = urllib.request.Request(url, headers={"Authorization": "Bearer %s" % self.api_key}) + with urllib.request.urlopen(request, timeout=30) as response: + raw = response.read() + except urllib.error.HTTPError as error: + if error.code == 404: + raise _SecretNotFound(url_path) + raise _CredsSourceError(f"creds source returned HTTP {error.code} for {url_path}") + except (OSError, ValueError) as error: + raise _CredsSourceError(f"creds source unavailable: {error}") + try: + body = json.loads(raw.decode("utf-8")) + except ValueError: + raise _CredsSourceError(f"creds source returned invalid JSON for {url_path}") + if not isinstance(body, dict): + raise _CredsSourceError(f"creds source returned a non-object for {url_path}") + return body + + def _kb_id(self): + for page in range(1, 1001): + body = self._fetch_json("/api/v1/knowledge/?page=%d" % page) + items = body.get("items") or [] + for item in items: + if item.get("name") == self.KB_NAME: + return item.get("id") + total = body.get("total") + if not items or (isinstance(total, int) and page * 30 >= total): + raise _SecretNotFound(self.KB_NAME) + raise _CredsSourceError("creds source pagination did not terminate") + + def _load_file_map(self): + kb_id = self._kb_id() + body = self._fetch_json("/api/v1/knowledge/%s" % urllib.parse.quote(str(kb_id), safe="")) + file_map = {} + for meta in body.get("files") or []: + name, file_id = meta.get("name"), meta.get("id") + if isinstance(name, str) and name and file_id is not None: + file_map[name] = (str(file_id), meta.get("updated_at")) + return file_map + + def get_secret(self, path): + if self._file_map is None or path not in self._file_map: + self._file_map = self._load_file_map() + if path not in self._file_map: + raise _SecretNotFound(path) + file_id, updated_at = self._file_map[path] + body = self._fetch_json("/api/v1/files/%s/data/content" % urllib.parse.quote(file_id, safe="")) + content = body.get("content") + if not isinstance(content, str): + raise _CredsSourceError(f"creds file for {path} returned no content") + try: + data = json.loads(content) + except ValueError: + raise _CredsSourceError(f"creds file for {path} is invalid JSON") + return _secret_response(path, data, updated_at) + + +def _build_creds_source(config): + source = str(config.get("BROKER_CREDS_SOURCE") or "file") + if source == "file": + return FileCredsSource(config.get("BROKER_CREDS_FILE")) + if source == "owui": + return OwuiApiCredsSource(config.get("BROKER_OWUI_BASE_URL"), config.get("BROKER_OWUI_API_KEY")) + raise ValueError(f"BROKER_CREDS_SOURCE must be file or owui, got {source!r}") + + +class Audit: + def __init__(self, path): + self.path = path + self._lock, self._handle = threading.Lock(), None + fd = os.open(path, os.O_CREAT | os.O_WRONLY, 0o600) + try: + os.fchmod(fd, 0o600) + finally: + os.close(fd) + + def write(self, action, path="", result="ok", user_id=None, + app_registration_uuid=None, agent_id=None): + if action not in AUDIT_ACTIONS: + raise ValueError(f"unknown audit action: {action!r}") + if result not in AUDIT_RESULTS: + raise ValueError(f"unknown audit result: {result!r}") + entry = { + "ts": _utc_now_iso(), "user_id": user_id, "app_registration_uuid": app_registration_uuid, + "agent_id": agent_id, "action": action, "path": path, "result": result, + } + line = json.dumps(entry, separators=(",", ":")) + "\n" + with self._lock: + if self._handle is None: + self._handle = open(self.path, "a", encoding="utf-8") + self._handle.write(line) + self._handle.flush() + os.fsync(self._handle.fileno()) + + def close(self): + with self._lock: + if self._handle is not None: + self._handle.close() + self._handle = None + + +class Broker: + def __init__(self, config, store, audit): + self.config, self.store, self.audit = config, store, audit + self.ui_root = config.get("BROKER_UI_ROOT") or "coreci_broker_ui" + self.static_root = config.get("BROKER_STATIC_ROOT") or "." + self.token_cache = {} + self.token_cache_lock = threading.Lock() + self.agent_state_backend = _build_agent_state_backend(config, store) + self.creds_source = _build_creds_source(config) + self.router = Router() + self.router.add("GET", "/healthz", _healthz) + self.router.add("POST", "/api/broker/auth/login", _auth_login) + self.router.add("POST", "/api/broker/auth/logout", _auth_logout) + self.router.add("GET", "/api/broker/auth/whoami", _auth_whoami) + self.router.add("GET", "/api/broker/app_registrations", _apps_list) + self.router.add("POST", "/api/broker/app_registrations", _apps_add) + self.router.add("DELETE", "/api/broker/app_registrations/", _apps_revoke) + self.router.add("POST", "/api/broker/app_registrations//reinstate", _apps_reinstate) + self.router.add("GET", "/api/broker/user_allowlist", _users_list) + self.router.add("POST", "/api/broker/user_allowlist", _users_add) + self.router.add("DELETE", "/api/broker/user_allowlist/", _users_revoke) + self.router.add("POST", "/api/broker/user_allowlist//reinstate", _users_reinstate) + self.router.add("GET", "/api/broker/audit", _audit_query) + self.router.add("POST", "/api/broker/agents/register", _agents_register) + self.router.add("GET", "/api/broker/agents/", _require_app(_agent_meta)) + self.router.add("GET", "/api/broker/agents//state", _require_app(_agent_state_get)) + self.router.add("PUT", "/api/broker/agents//state", _require_app(_agent_state_put)) + self.router.add("GET", "/api/broker/secrets/", _require_app(_secrets_get)) + + def clear_token_cache(self): + with self.token_cache_lock: + self.token_cache.clear() + + +def boot(env=None): + config = load_config(env) + store = Store(config["BROKER_DB_PATH"]) + store.set_config("BROKER_INSTANCE_PEPPER", config["BROKER_INSTANCE_PEPPER"]) + audit = Audit(config["BROKER_AUDIT_LOG_PATH"]) + return Broker(config, store, audit) + + +class _BadRequest(Exception): + pass + + +def _healthz(request): + request.send_json(200, {"ok": True}) + + +def _check_admin_credentials(broker, username, password): + if not isinstance(username, str) or not isinstance(password, str): + return False + config = broker.config + user = str(config.get("BROKER_ADMIN_USER") or "").encode("utf-8") + expected = str(config.get("BROKER_ADMIN_PASS") or "").encode("utf-8") + return (hmac.compare_digest(username.encode("utf-8"), user) + and hmac.compare_digest(password.encode("utf-8"), expected)) + + +def _basic_credentials(request): + header = request.headers.get("Authorization") or "" + if header[:6].lower() != "basic ": + return "", "" + try: + decoded = base64.b64decode(header[6:].strip(), validate=True).decode("utf-8") + except ValueError: + return "", "" + username, sep, password = decoded.partition(":") + return (username, password) if sep else ("", "") + + +def _request_session(request): + jar = http.cookies.SimpleCookie() + try: + jar.load(request.headers.get("Cookie") or "") + except http.cookies.CookieError: + return None + morsel = jar.get("broker_session") + if morsel is None or not morsel.value: + return None + row = request.broker.store.get_admin_session(morsel.value) + if row is not None and row[2] > _utc_now_iso(): + return row + return None + + +def _require_admin(request): + if _request_session(request) is not None: + return True + if not request.headers.get("Authorization"): + request.send_error_json(401, "unauthorized") + return False + username, password = _basic_credentials(request) + if _check_admin_credentials(request.broker, username, password): + return True + request.send_error_json(401, "invalid credentials") + return False + + +def _auth_login(request): + broker = request.broker + body = request.body if isinstance(request.body, dict) else {} + username = body.get("username") + if not _check_admin_credentials(broker, username, body.get("password")): + broker.audit.write("login", path="/api/broker/auth/login", result="denied") + request.send_error_json(401, "invalid credentials") + return + session_id = secrets.token_urlsafe(32) + raw_ttl = str(broker.config.get("BROKER_SESSION_TTL") or "") + seconds = int(raw_ttl) if raw_ttl.isdigit() else 300 + now = datetime.datetime.now(datetime.timezone.utc) + expires = (now + datetime.timedelta(seconds=seconds)).isoformat() + broker.store.create_admin_session(session_id, now.isoformat(), expires) + broker.audit.write("login", path="/api/broker/auth/login", result="ok") + cookie = "broker_session=%s; Path=/; HttpOnly; SameSite=Lax" % session_id + request.send_json(200, {"ok": True, "username": username}, [("Set-Cookie", cookie)]) + + +def _auth_logout(request): + broker = request.broker + if not _require_admin(request): + return + row = _request_session(request) + if row is not None: + broker.store.delete_admin_session(row[0]) + broker.audit.write("logout", path="/api/broker/auth/logout", result="ok") + request.send_json(200, {"ok": True}) + + +def _auth_whoami(request): + if not _require_admin(request): + return + row = _request_session(request) + payload = {"username": request.broker.config.get("BROKER_ADMIN_USER") or "", + "created_at": row[1] if row else None, "expires_at": row[2] if row else None} + request.send_json(200, payload) + + +def _list_rows(request, table, keys, field): + if not _require_admin(request): + return + rows = request.broker.store.list_rows(table) + request.send_json(200, {field: [dict(zip(keys, row)) for row in rows]}) + + +def _apps_list(request): + _list_rows(request, "app_registrations", + ("uuid", "name", "status", "created_at", "revoked_at"), "registrations") + + +def _users_list(request): + _list_rows(request, "user_allowlist", + ("user_id", "email", "status", "created_at", "revoked_at"), "allowlist") + + +def _canonical_uuid(value): + try: + canonical = str(uuid.UUID(value)) + except (ValueError, TypeError, AttributeError): + return None + return canonical if canonical == value else None + + +def _required_string(value): + return value if isinstance(value, str) and value else None + + +def _has_control_chars(value): + return any(ord(ch) < 32 or ord(ch) == 127 for ch in value) + + +def _valid_id(value): + if not _required_string(value): + return None + if value in (".", ".."): + return None + if ":" in value or "/" in value or "\\" in value or _has_control_chars(value): + return None + return value + + +def _add_row(request, table, key, field, action, validate): + if not _require_admin(request): + return + broker = request.broker + body = request.body if isinstance(request.body, dict) else {} + value = validate(body.get(key)) + if value is None: + request.send_error_json(400, "invalid " + key) + return + if broker.store.get_row(table, key, value): + request.send_error_json(409, "duplicate " + key) + return + optional = body.get(field) + broker.store.add_row(table, value, optional if isinstance(optional, str) else None) + broker.audit.write(action, path="/api/broker/%s/%s" % (table, value)) + request.send_json(201, {key: value, "status": "active"}) + + +def _apps_add(request): + _add_row(request, "app_registrations", "uuid", "name", "add_app", _canonical_uuid) + + +def _users_add(request): + _add_row(request, "user_allowlist", "user_id", "email", "add_user", _valid_id) + + +def _set_row_status(request, table, key, status, action): + if not _require_admin(request): + return + broker = request.broker + value = request.path_params[key] + row = broker.store.get_row(table, key, value) + if row is None: + request.send_error_json(404, "not found") + return + if row[2] != status: + broker.store.set_row_status(table, key, value, status) + broker.audit.write(action, path="/api/broker/%s/%s" % (table, value)) + request.send_json(200, {key: value, "status": status}) + + +def _apps_revoke(request): + _set_row_status(request, "app_registrations", "uuid", "revoked", "remove_app") + + +def _apps_reinstate(request): + _set_row_status(request, "app_registrations", "uuid", "active", "add_app") + + +def _users_revoke(request): + _set_row_status(request, "user_allowlist", "user_id", "revoked", "remove_user") + + +def _users_reinstate(request): + _set_row_status(request, "user_allowlist", "user_id", "active", "add_user") + + +def _parse_app_header(header): + parts = (header or "").split() + if len(parts) != 3 or parts[0] != "App": + return None, None + app_uuid = _canonical_uuid(parts[1]) + return (app_uuid, parts[2]) if app_uuid is not None else (None, None) + + +def _app_auth_context(row): + return { + "agent_id": row[0], + "user_id": row[2], + "app_registration_uuid": row[1], + "token_expires_at": row[5], + } + + +def _verify_app_token(broker, header): + app_uuid, token = _parse_app_header(header) + if app_uuid is None: + return None + now = _utc_now_iso() + key = (app_uuid, token) + with broker.token_cache_lock: + cached = broker.token_cache.get(key) + if cached is not None: + row = broker.store.get_agent(cached["agent_id"], app_uuid) + if row is not None and row[5] and row[5] > now: + return _app_auth_context(row) + with broker.token_cache_lock: + broker.token_cache.pop(key, None) + return None + for row in broker.store.agents_by_uuid(app_uuid): + if row[5] and row[5] > now and verify_token(token, row[4]): + context = _app_auth_context(row) + with broker.token_cache_lock: + broker.token_cache[key] = context + return context + return None + + +def _app_token_valid(broker, header): + return _verify_app_token(broker, header) is not None + + +def _audit_app_denied(request): + parts = (request.headers.get("Authorization") or "").split() + if len(parts) < 2 or parts[0] != "App": + return + request.broker.audit.write( + "denied", path=request.path, result="denied", + app_registration_uuid=_canonical_uuid(parts[1]), + ) + + +def _require_app(handler): + def wrapped(request): + context = _verify_app_token(request.broker, request.headers.get("Authorization")) + if context is None: + _audit_app_denied(request) + request.send_error_json(401, "invalid app credentials") + return + request.app_auth = context + request.broker.store.touch_agent( + context["agent_id"], context["app_registration_uuid"], _utc_now_iso() + ) + handler(request) + return wrapped + + +def _agents_register(request): + broker = request.broker + body = request.body if isinstance(request.body, dict) else {} + agent_id = body.get("agent_id") + app_uuid = body.get("app_registration_uuid") + user_id = body.get("user_id") + tags = body.get("tags") + strings = [v for v in (agent_id, app_uuid, user_id) if isinstance(v, str) and v] + if len(strings) != 3 or not isinstance(tags, list) or not all(isinstance(v, str) for v in tags): + request.send_error_json(400, "invalid registration") + return + if _valid_id(agent_id) is None: + request.send_error_json(400, "invalid agent_id") + return + if any(":" in tag or _has_control_chars(tag) for tag in tags): + request.send_error_json(400, "invalid tags") + return + path = "/api/broker/agents/register" + identity = {"user_id": user_id, "app_registration_uuid": app_uuid, "agent_id": agent_id} + app_row = broker.store.get_row("app_registrations", "uuid", app_uuid) + user_row = broker.store.get_row("user_allowlist", "user_id", user_id) + if app_row is None or app_row[2] != "active" or user_row is None or user_row[2] != "active": + broker.audit.write("register", path=path, result="denied", **identity) + request.send_error_json(403, "registration not allowed") + return + max_raw = str(broker.config.get("BROKER_MAX_AGENTS_PER_APP") or "") + max_agents = int(max_raw) if max_raw.isdigit() else 50 + if len(broker.store.agents_by_uuid(app_uuid)) >= max_agents: + request.send_error_json(400, "too many agents for this app") + return + pepper = broker.store.get_config("BROKER_INSTANCE_PEPPER") or broker.config["BROKER_INSTANCE_PEPPER"] + token = derive_token(pepper, app_uuid, user_id, agent_id, tags) + raw_ttl = str(broker.config.get("BROKER_TOKEN_TTL") or "") + ttl = int(raw_ttl) if raw_ttl.isdigit() else 86400 + now = datetime.datetime.now(datetime.timezone.utc) + expires = (now + datetime.timedelta(seconds=ttl)).isoformat() + tags_json = json.dumps(sorted(tags)) + row = broker.store.get_agent(agent_id, app_uuid) + created = row is None + if created: + try: + broker.store.save_agent(agent_id, app_uuid, user_id, tags_json, hash_token(token), expires, now.isoformat()) + except sqlite3.IntegrityError: + row = broker.store.get_agent(agent_id, app_uuid) + if row is None: + raise + created = False + if not created: + if row[2] == user_id and row[3] == tags_json and row[5] and row[5] > now.isoformat(): + expires = row[5] + broker.store.touch_agent(agent_id, app_uuid, now.isoformat()) + else: + broker.store.update_agent(agent_id, app_uuid, user_id, tags_json, hash_token(token), expires, now.isoformat()) + with broker.token_cache_lock: + for cached_key in [k for k in broker.token_cache if k[0] == app_uuid]: + broker.token_cache.pop(cached_key, None) + payload = {"token": token, "token_expires_at": expires, "created": created} + if created or not _app_token_valid(broker, request.headers.get("Authorization")): + payload["pepper"] = pepper + broker.audit.write("register", path=path, **identity) + request.send_json(201 if created else 200, payload) + + +def _agent_identity_row(request): + auth = request.app_auth + row = request.broker.store.get_agent( + request.path_params["agent_id"], auth["app_registration_uuid"] + ) + if row is None: + request.send_error_json(404, "not found") + return None + if row[0] != auth["agent_id"] or row[2] != auth["user_id"]: + request.send_error_json(403, "agent identity mismatch") + return None + return row + + +def _agent_meta(request): + row = _agent_identity_row(request) + if row is None: + return + request.send_json(200, { + "agent_id": row[0], + "app_registration_uuid": row[1], + "user_id": row[2], + "tags": json.loads(row[3]), + "token_expires_at": row[5], + "first_registered_at": row[6], + "last_seen_at": row[7], + }) + + +def _agent_state_get(request): + row = _agent_identity_row(request) + if row is None: + return + state = request.broker.agent_state_backend.get_state(row[0], row[1]) + if state is None: + request.send_error_json(404, "not found") + return + request.send_json(200, state) + + +def _agent_state_put(request): + row = _agent_identity_row(request) + if row is None: + return + body = request.body + if not isinstance(body, dict): + request.send_error_json(400, "state must be a JSON object") + return + if len(json.dumps(body).encode("utf-8")) > MAX_STATE_BYTES: + request.send_error_json(400, "state too large") + return + request.broker.agent_state_backend.put_state(row[0], row[1], body) + request.send_json(200, {"ok": True}) + + +def _secret_path_unsafe(path): + return (path.startswith("/") or "\\" in path or "\x00" in path + or ".." in path.split("/")) + + +def _secrets_denied(request, path, identity): + request.broker.audit.write("read", path=path, result="denied", **identity) + request.send_error_json(403, "forbidden") + + +def _secrets_get(request): + broker = request.broker + auth = request.app_auth + path = urllib.parse.unquote(request.path_params["path"]) + identity = {"user_id": auth["user_id"], + "app_registration_uuid": auth["app_registration_uuid"], + "agent_id": auth["agent_id"]} + if _secret_path_unsafe(path): + _secrets_denied(request, path, identity) + return + if path != "coreci/" + auth["agent_id"]: + _secrets_denied(request, path, identity) + return + try: + secret = broker.creds_source.get_secret(path) + except _SecretNotFound: + broker.audit.write("read", path=path, result="error", **identity) + request.send_error_json(404, "not found") + except _CredsSourceError: + broker.audit.write("read", path=path, result="error", **identity) + request.send_error_json(500, "creds source unavailable") + else: + broker.audit.write("read", path=path, **identity) + request.send_json(200, secret) + + +def _parse_iso_ts(value): + try: + parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except (ValueError, AttributeError): + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=datetime.timezone.utc) + return parsed + + +def _audit_entry_matches(entry, query, since, until): + for field in ("user_id", "app_registration_uuid", "action"): + if query.get(field) and entry.get(field) != query[field]: + return False + if since is None and until is None: + return True + parsed = _parse_iso_ts(entry.get("ts") or "") + if parsed is None: + return False + if since is not None and parsed <= since: + return False + if until is not None and parsed > until: + return False + return True + + +def _audit_query(request): + if not _require_admin(request): + return + query = request.query + since = until = None + if query.get("since"): + since = _parse_iso_ts(query["since"]) + if since is None: + request.send_error_json(400, "invalid since") + return + if query.get("until"): + until = _parse_iso_ts(query["until"]) + if until is None: + request.send_error_json(400, "invalid until") + return + try: + limit = int(query.get("limit") or "") + except ValueError: + limit = 100 + limit = min(max(limit, 1), 1000) + entries = [] + with open(request.broker.audit.path, "r", encoding="utf-8") as handle: + for raw in handle: + if len(entries) >= limit: + break + line = raw.strip() + if not line: + continue + try: + entry = json.loads(line) + except ValueError: + continue + if not isinstance(entry, dict): + continue + if not _audit_entry_matches(entry, query, since, until): + continue + entries.append(entry) + next_since = entries[-1].get("ts") if len(entries) >= limit else None + request.send_json(200, {"entries": entries, "next_since": next_since}) + + +def _parse_query(query): + values = urllib.parse.parse_qs(query, keep_blank_values=True) + return {key: value[0] for key, value in values.items()} + + +def _match_pattern(pattern, path): + pattern_parts = pattern.split("/") + path_parts = path.split("/") + if len(pattern_parts) > len(path_parts): + return None + params = {} + for index, expected in enumerate(pattern_parts): + if not (len(expected) > 2 and expected.startswith("<") and expected.endswith(">")): + if expected != path_parts[index]: + return None + continue + name = expected[1:-1] + if name.endswith("..."): + if index + 1 != len(pattern_parts) or not path_parts[index:]: + return None + params[name[:-3]] = "/".join(path_parts[index:]) + return params + if not path_parts[index]: + return None + params[name] = path_parts[index] + if len(pattern_parts) != len(path_parts): + return None + return params + + +def _static_content_type(name): + extension = os.path.splitext(name)[1].lower() + return STATIC_CONTENT_TYPES.get(extension, "application/octet-stream") + + +def _broker_protected_paths(broker): + protected = set() + for key in ("BROKER_ENV_PATH", "BROKER_DB_PATH", "BROKER_AUDIT_LOG_PATH", "BROKER_CREDS_FILE"): + value = broker.config.get(key) or "" + if not value: + continue + base = os.path.realpath(value) + protected.add(base) + for suffix in ("-journal", "-wal", "-shm"): + protected.add(base + suffix) + return protected + + +def _resolve_static_file(root, rest, protected=()): + if "\x00" in rest or "\\" in rest or rest.startswith("/"): + return None + parts = rest.split("/") + if ".." in parts or any(part.startswith(".") for part in parts): + return None + stripped = rest.strip("/") + names = [] + if stripped: + names.append(stripped) + if "." not in os.path.basename(stripped): + names.append(stripped + ".html") + else: + names.append("index.html") + root_real = os.path.realpath(root) + for name in names: + target = os.path.realpath(os.path.join(root_real, name)) + if not target.startswith(root_real + os.sep): + continue + if target in protected: + continue + if os.path.isfile(target): + return target + return None + + +class Router: + def __init__(self): + self._routes = [] + + def add(self, method, pattern, handler): + self._routes.append((method.upper(), pattern, handler)) + + def resolve(self, method, path): + for route_method, pattern, handler in self._routes: + if route_method != method: + continue + params = _match_pattern(pattern, path) + if params is not None: + return handler, params + return None, None + + +class BrokerRequestHandler(BaseHTTPRequestHandler): + broker = None + timeout = 30 + responded = False + path_params = None + query = None + body = None + + def log_message(self, format, *args): + pass + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_PUT(self): + self._dispatch("PUT") + + def do_DELETE(self): + self._dispatch("DELETE") + + def send_json(self, status, payload, headers=()): + if self.responded: + return + self.responded = True + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + for key, value in headers: + self.send_header(key, value) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def send_error_json(self, status, message): + self.send_json(status, {"error": status, "message": message}) + + def _reject_oversized_body(self): + header = self.headers.get("Content-Length") + if not header: + return + try: + length = int(header) + except ValueError: + return + if length > MAX_BODY_BYTES: + raise _BadRequest("body too large") + + def _read_body(self): + header = self.headers.get("Content-Length") or "0" + try: + length = int(header) + except ValueError: + raise _BadRequest("invalid Content-Length") + if length > MAX_BODY_BYTES: + raise _BadRequest("body too large") + if length <= 0: + return b"" + return self.rfile.read(length) + + def _dispatch(self, method): + self.path_params, self.query, self.body = {}, {}, None + try: + self._reject_oversized_body() + parsed = urllib.parse.urlsplit(self.path) + path = urllib.parse.unquote(parsed.path) + self.query = _parse_query(parsed.query) + raw = self._read_body() if method in ("POST", "PUT") else b"" + if self._serve_static(method, path): + return + handler, params = self.broker.router.resolve(method, path) + if handler is None: + self.send_error_json(404, "not found") + return + if raw: + try: + self.body = json.loads(raw) + except (ValueError, RecursionError): + self.send_error_json(400, "invalid JSON body") + return + self.path_params = params + handler(self) + except _BadRequest as error: + self.send_error_json(400, str(error)) + except Exception: + if not self.responded: + self.send_error_json(500, "internal error") + + def _serve_static(self, method, path): + if method != "GET": + return False + for prefix, attr in STATIC_PREFIXES: + if path.startswith("/" + prefix + "/"): + self._send_static_file(getattr(self.broker, attr), path[len(prefix) + 2:]) + return True + return False + + def _send_static_file(self, root, rest): + target = _resolve_static_file( + root, rest, _broker_protected_paths(self.broker) + ) + if target is None: + self.send_error_json(404, "not found") + return + with open(target, "rb") as handle: + data = handle.read() + self.send_response(200) + self.send_header("Content-Type", _static_content_type(target)) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +def serve(host, port, broker): + handler = type("BoundHandler", (BrokerRequestHandler,), {"broker": broker}) + server = ThreadingHTTPServer((host, port), handler) + server.broker = broker + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +_server_lock = threading.Lock() +_started_server = None + + +def _parse_bind(bind): + host, _, port = str(bind).rpartition(":") + if not port.isdigit(): + raise ValueError(f"BROKER_BIND must be host:port, got {bind!r}") + return host or "127.0.0.1", int(port) + + +def _start_guarded_server(env=None): + global _started_server + with _server_lock: + if _started_server is not None: + return _started_server + broker = boot(env) + host, port = _parse_bind(broker.config["BROKER_BIND"]) + _started_server = serve(host, port, broker) + return _started_server + + +def _reset_server_guard(): + global _started_server + with _server_lock: + server = _started_server + _started_server = None + if server is None: + return None + server.shutdown() + server.server_close() + server.broker.clear_token_cache() + server.broker.audit.close() + server.broker.store.close() + return server + + +class Tools: + class Valves: + pass + + def __init__(self): + self.valves = self.Valves() + try: + self._server = _start_guarded_server() + except OSError as error: + if error.errno != errno.EADDRINUSE: + raise + raise RuntimeError( + f"broker bind failed ({error}): a broker from previously loaded " + "module code is still serving the configured bind; restart " + "Open WebUI to load the updated broker" + ) from error + + def status(self) -> str: + """Report the broker server state as a one-line string.""" + host, port = self._server.server_address[:2] + return f"coreci-broker serving on http://{host}:{port}" + + +def run_standalone(env=None): + broker = boot(env) + host, port = _parse_bind(broker.config["BROKER_BIND"]) + server = serve(host, port, broker) + bound_host, bound_port = server.server_address[:2] + print(f"coreci-broker listening on http://{bound_host}:{bound_port}", flush=True) + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + server.shutdown() + server.server_close() + broker.clear_token_cache() + broker.audit.close() + broker.store.close() + print("coreci-broker shut down", flush=True) + + +if __name__ == "__main__": + run_standalone() diff --git a/coreci_broker_client.py b/coreci_broker_client.py new file mode 100644 index 0000000..bf3320f --- /dev/null +++ b/coreci_broker_client.py @@ -0,0 +1,161 @@ +"""Public stdlib-only client for the CoreCI Broker: idempotent tool registration, agent-state persistence, and path-scoped secret reads.""" +import json +import time +import urllib.parse +import urllib.request + +REGISTER_PATH = "/api/broker/agents/register" +SECRETS_PREFIX = "/api/broker/secrets/" +CLIENT_MODULE_PATH = "/static/coreci_broker_client.py" + + +class BrokerError(Exception): + def __init__(self, status, message): + super().__init__(message) + self.status = status + self.message = message + + +def _broker_error(status, body): + message = "broker returned HTTP %s" % status + if isinstance(body, dict) and body.get("message"): + message = str(body["message"]) + return BrokerError(status, message) + + +def _epoch_from_iso(value): + try: + year, month, day = int(value[0:4]), int(value[5:7]), int(value[8:10]) + hour, minute, second = int(value[11:13]), int(value[14:16]), int(value[17:19]) + rest = value[19:] + micro = 0 + if rest[:1] == ".": + digits = "" + for char in rest[1:]: + if not char.isdigit(): + break + digits += char + micro = int((digits + "000000")[:6]) + rest = rest[1 + len(digits):] + offset = 0 + if rest[:1] in ("+", "-"): + sign = 1 if rest[0] == "+" else -1 + offset = sign * (int(rest[1:3]) * 3600 + int(rest[4:6]) * 60) + if not 1 <= month <= 12 or not 1 <= day <= 31: + return None + if hour > 23 or minute > 59 or second > 59: + return None + year -= month <= 2 + era = (year if year >= 0 else year - 399) // 400 + yoe = year - era * 400 + doy = (153 * (month - 3 if month > 2 else month + 9) + 2) // 5 + day - 1 + doe = yoe * 365 + yoe // 4 - yoe // 100 + doy + return (era * 146097 + doe - 719468) * 86400 + hour * 3600 + minute * 60 + second - offset + micro / 1000000.0 + except (ValueError, TypeError, IndexError): + return None + + +def _token_live(state): + token = state.get("current_token") + expires = _epoch_from_iso(state.get("token_expires_at")) + return isinstance(token, str) and bool(token) and expires is not None and expires > time.time() + + +class BrokerClient: + def __init__(self, base_url, agent_id, app_uuid, user_id, tags=None, timeout=10.0): + self.base_url = str(base_url).rstrip("/") + self.agent_id = agent_id + self.app_uuid = app_uuid + self.user_id = user_id + self.tags = list(tags or []) + self.timeout = timeout + + def _app_headers(self, state=None): + token = (state or {}).get("current_token") + if not isinstance(token, str) or not token: + return {} + return {"Authorization": "App %s %s" % (self.app_uuid, token)} + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base_url + path, data=data, method=method) + if data is not None: + request.add_header("Content-Type", "application/json") + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + status, raw = response.status, response.read() + except urllib.request.HTTPError as error: + status, raw = error.code, error.read() + body = None + text = raw.decode("utf-8", "replace") + if text: + try: + body = json.loads(text) + except ValueError: + body = None + return status, body + + def _authorized_request(self, method, path, state, payload=None): + status, body = self._request(method, path, payload, self._app_headers(state)) + if status == 401: + self.register(state) + status, body = self._request(method, path, payload, self._app_headers(state)) + return status, body + + def _state_path(self): + return "/api/broker/agents/%s/state" % urllib.parse.quote(self.agent_id, safe="") + + def register(self, state): + if state is None: + state = {} + payload = {"agent_id": self.agent_id, "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": list(self.tags)} + status, data = self._request("POST", REGISTER_PATH, payload, self._app_headers(state)) + if status not in (200, 201) or not isinstance(data, dict): + raise _broker_error(status, data) + state["agent_id"] = self.agent_id + state["app_registration_uuid"] = self.app_uuid + state["user_id"] = self.user_id + state["tags"] = list(self.tags) + state["current_token"] = data["token"] + state["token_expires_at"] = data["token_expires_at"] + pepper = data.get("pepper") + if pepper is not None: + state["pepper"] = pepper + return state + + def ensure_registered(self, state): + if state is None: + state = {} + if _token_live(state): + return state + return self.register(state) + + def get_state(self, state): + status, body = self._authorized_request("GET", self._state_path(), state) + if status == 200: + return body + if status == 404: + return None + raise _broker_error(status, body) + + def put_state(self, state): + status, body = self._authorized_request("PUT", self._state_path(), state, state) + if status == 200: + return True + raise _broker_error(status, body) + + def read_secret(self, path, state): + suffix = urllib.parse.quote(str(path), safe="/") + status, body = self._authorized_request("GET", SECRETS_PREFIX + suffix, state) + if status == 200: + return body + raise _broker_error(status, body) + + +def fetch_client_module(base_url, timeout=10.0): + url = str(base_url).rstrip("/") + CLIENT_MODULE_PATH + with urllib.request.urlopen(url, timeout=timeout) as response: + return response.read().decode("utf-8") \ No newline at end of file diff --git a/coreci_broker_ui/app_registrations.html b/coreci_broker_ui/app_registrations.html new file mode 100644 index 0000000..f17e540 --- /dev/null +++ b/coreci_broker_ui/app_registrations.html @@ -0,0 +1,271 @@ + + + + + +App Registrations — CoreCI Broker + + + + + +
+
+
+
+

App Registrations

+

Tool app UUIDs allowed to register against the broker.

+
+ +
+ + + + + + + + + + + + + +
UUIDNameStatusCreatedRevokedActions
+
+
+ + + + + \ No newline at end of file diff --git a/coreci_broker_ui/audit.html b/coreci_broker_ui/audit.html new file mode 100644 index 0000000..a08d5ad --- /dev/null +++ b/coreci_broker_ui/audit.html @@ -0,0 +1,280 @@ + + + + + +Audit — CoreCI Broker + + + + + +
+
+

Audit

+

Read-only trail of broker activity. Set filters and apply to query the audit log.

+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+
+ + + + + + + + + + + + + +
TimestampUser IDApp UUIDAgent IDActionPathResult
+
+ +
+ +
+
+
+ + + + \ No newline at end of file diff --git a/coreci_broker_ui/dashboard.html b/coreci_broker_ui/dashboard.html new file mode 100644 index 0000000..e7ca6dc --- /dev/null +++ b/coreci_broker_ui/dashboard.html @@ -0,0 +1,64 @@ + + + + + +Dashboard — CoreCI Broker + + + + +
+
+

Dashboard

+

Signed in as …

+
+
+ + Broker status: checking… +
+
+

Quick links

+
+ + App Registrations + Add, revoke, and reinstate tool app UUIDs + + + User Allowlist + Manage the user ids allowed to register + + + Audit Log + Review broker activity, read-only + +
+
+
+ + + + \ No newline at end of file diff --git a/coreci_broker_ui/index.html b/coreci_broker_ui/index.html new file mode 100644 index 0000000..616a46f --- /dev/null +++ b/coreci_broker_ui/index.html @@ -0,0 +1,13 @@ + + + + + +CoreCI Broker + + + +

Continue to CoreCI Broker sign-in

+ + + \ No newline at end of file diff --git a/coreci_broker_ui/login.html b/coreci_broker_ui/login.html new file mode 100644 index 0000000..8d1d4ae --- /dev/null +++ b/coreci_broker_ui/login.html @@ -0,0 +1,78 @@ + + + + + +CoreCI Broker + + + +
+

CoreCI Broker

+ +
+
+ + +
+
+ + +
+ + +
+
+ + + \ No newline at end of file diff --git a/coreci_broker_ui/nav.js b/coreci_broker_ui/nav.js new file mode 100644 index 0000000..772d275 --- /dev/null +++ b/coreci_broker_ui/nav.js @@ -0,0 +1,84 @@ +var NAV_ITEMS = [ + {label: "Dashboard", href: "/ui/dashboard"}, + {label: "App Registrations", href: "/ui/app_registrations"}, + {label: "Users", href: "/ui/user_allowlist"}, + {label: "Audit", href: "/ui/audit"} +]; + +async function api(path, opts) { + opts = opts || {}; + var init = {method: opts.method || "GET", credentials: "same-origin", headers: {}}; + if (opts.body !== undefined) { + init.headers["Content-Type"] = "application/json"; + init.body = typeof opts.body === "string" ? opts.body : JSON.stringify(opts.body); + } + if (opts.headers) { + for (var name in opts.headers) { + init.headers[name] = opts.headers[name]; + } + } + var response = null; + try { + response = await fetch(path, init); + } catch (networkError) { + return {ok: false, status: 0, data: null}; + } + var data = null; + try { + data = await response.json(); + } catch (parseError) { + data = null; + } + if (response.status === 401) { + location.href = "/ui/login"; + } + return {ok: response.ok, status: response.status, data: data}; +} + +async function navLogout() { + try { + await api("/api/broker/auth/logout", {method: "POST"}); + } catch (error) {} + location.href = "/ui/login"; +} + +function navRender() { + var mount = document.getElementById("nav"); + if (!mount) { + return; + } + var current = location.pathname.replace(/\.html$/, "").replace(/\/+$/, ""); + var header = document.createElement("header"); + header.className = "nav"; + var brand = document.createElement("a"); + brand.className = "nav-brand"; + brand.href = "/ui/dashboard"; + brand.textContent = "CoreCI Broker"; + header.appendChild(brand); + var links = document.createElement("nav"); + links.setAttribute("aria-label", "Primary"); + NAV_ITEMS.forEach(function (item) { + var link = document.createElement("a"); + link.href = item.href; + link.textContent = item.label; + if (current === item.href) { + link.className = "active"; + link.setAttribute("aria-current", "page"); + } + links.appendChild(link); + }); + header.appendChild(links); + var logout = document.createElement("button"); + logout.type = "button"; + logout.className = "btn btn-subtle nav-logout"; + logout.textContent = "Logout"; + logout.addEventListener("click", navLogout); + header.appendChild(logout); + mount.appendChild(header); +} + +if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", navRender); +} else { + navRender(); +} \ No newline at end of file diff --git a/coreci_broker_ui/shared.css b/coreci_broker_ui/shared.css new file mode 100644 index 0000000..6e33edb --- /dev/null +++ b/coreci_broker_ui/shared.css @@ -0,0 +1,369 @@ +:root { + color-scheme: dark; + --bg: #131820; + --card: #1a212b; + --card-alt: #232d3a; + --border: #2d3947; + --text: #e7edf4; + --muted: #96a3b4; + --accent: #2dd4bf; + --accent-ink: #062421; + --accent-soft: rgba(45, 212, 191, 0.12); + --danger: #f07a72; + --radius: 10px; +} + +* { + box-sizing: border-box; +} + +[hidden] { + display: none !important; +} + +html, +body { + margin: 0; + padding: 0; +} + +body { + background: var(--bg); + color: var(--text); + font: 15px/1.55 system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; +} + +h1 { + font-size: 24px; + line-height: 1.25; + margin: 0 0 12px; +} + +h2 { + font-size: 17px; + line-height: 1.25; + margin: 0 0 12px; +} + +h3 { + font-size: 15px; + line-height: 1.25; + margin: 0 0 10px; +} + +p { + margin: 0 0 10px; +} + +a { + color: var(--accent); + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +a:focus-visible, +button:focus-visible, +input:focus-visible, +select:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} + +.wrap { + max-width: 1040px; + margin: 0 auto; + padding: 24px; +} + +.card { + background: var(--card); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 20px 24px; + margin-bottom: 20px; +} + +.nav { + position: sticky; + top: 0; + z-index: 10; + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 8px 20px; + padding: 10px 24px; + background: var(--card); + border-bottom: 1px solid var(--border); +} + +.nav-brand { + color: var(--text); + font-size: 16px; + font-weight: 700; +} + +.nav-brand:hover { + color: var(--accent); + text-decoration: none; +} + +.nav nav { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 4px; +} + +.nav nav a { + color: var(--muted); + font-size: 14px; + padding: 6px 10px; + border-radius: 6px; +} + +.nav nav a:hover { + color: var(--text); + background: var(--card-alt); + text-decoration: none; +} + +.nav nav a.active { + color: var(--accent); + background: var(--accent-soft); +} + +.nav-logout { + margin-left: auto; +} + +.btn { + display: inline-block; + font: inherit; + font-size: 14px; + color: var(--text); + background: var(--card-alt); + border: 1px solid transparent; + border-radius: 8px; + padding: 9px 16px; + cursor: pointer; +} + +.btn:hover { + filter: brightness(1.12); +} + +.btn:disabled { + opacity: 0.6; + cursor: default; +} + +.btn-primary { + color: var(--accent-ink); + background: var(--accent); + font-weight: 600; +} + +.btn-subtle { + background: transparent; + border-color: var(--border); +} + +.btn-block { + width: 100%; +} + +.field { + margin-bottom: 16px; +} + +.field label { + display: block; + color: var(--muted); + font-size: 13px; + margin-bottom: 6px; +} + +.field input, +.field select { + width: 100%; + font: inherit; + color: var(--text); + background: var(--bg); + border: 1px solid var(--border); + border-radius: 8px; + padding: 10px 12px; +} + +.field input:focus, +.field select:focus { + border-color: var(--accent); +} + +.form-error { + color: var(--danger); + background: rgba(240, 122, 114, 0.08); + border: 1px solid rgba(240, 122, 114, 0.35); + border-radius: 8px; + padding: 10px 12px; + font-size: 14px; + margin: 0 0 14px; +} + +.login-body { + min-height: 100vh; + display: grid; + place-items: center; + padding: 24px; +} + +.login-card { + width: 100%; + max-width: 380px; + margin-bottom: 0; +} + +.login-sub { + color: var(--muted); + font-size: 14px; + margin-bottom: 20px; +} + +.status-strip { + display: flex; + align-items: center; + gap: 10px; + padding: 12px 24px; + font-size: 14px; + color: var(--muted); +} + +.dot { + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--muted); +} + +.dot-ok { + background: var(--accent); +} + +.dot-down { + background: var(--danger); +} + +.link-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 16px; +} + +.link-card { + display: block; + background: var(--card-alt); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 16px; + color: var(--text); +} + +.link-card:hover { + border-color: var(--accent); + text-decoration: none; +} + +.link-card strong { + display: block; + font-size: 15px; + margin-bottom: 4px; +} + +.link-card span { + color: var(--muted); + font-size: 13px; +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 14px; +} + +th { + text-align: left; + color: var(--muted); + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.06em; + padding: 8px 12px; + border-bottom: 1px solid var(--border); +} + +td { + padding: 10px 12px; + border-bottom: 1px solid var(--border); +} + +tbody tr:hover { + background: var(--card-alt); +} + +.mono { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; +} + +.badge { + display: inline-block; + color: var(--muted); + border: 1px solid var(--border); + border-radius: 999px; + padding: 2px 10px; + font-size: 12px; +} + +.badge-active { + color: var(--accent); + border-color: var(--accent); + background: var(--accent-soft); +} + +.modal-backdrop { + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.55); + display: grid; + place-items: center; + padding: 24px; + z-index: 20; +} + +.modal { + width: 100%; + max-width: 460px; + background: var(--card); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 20px 24px; + margin-bottom: 0; +} + +.modal-actions { + display: flex; + justify-content: flex-end; + gap: 10px; + margin-top: 20px; +} + +@media (max-width: 900px) { + .wrap { + padding: 16px; + } + + .nav { + padding: 10px 16px; + } +} \ No newline at end of file diff --git a/coreci_broker_ui/user_allowlist.html b/coreci_broker_ui/user_allowlist.html new file mode 100644 index 0000000..8c92afe --- /dev/null +++ b/coreci_broker_ui/user_allowlist.html @@ -0,0 +1,271 @@ + + + + + +User Allowlist — CoreCI Broker + + + + + +
+
+
+
+

User Allowlist

+

User ids allowed to register agents against the broker.

+
+ +
+ + + + + + + + + + + + + +
User IDEmailStatusCreatedRevokedActions
+
+
+ + + + + \ No newline at end of file diff --git a/reference_tools/README.md b/reference_tools/README.md new file mode 100644 index 0000000..b783d53 --- /dev/null +++ b/reference_tools/README.md @@ -0,0 +1,25 @@ +# Reference Tools + +Templates for Open WebUI tool authors wiring tools to the CoreCI Broker. +They contain no secrets and no live-infrastructure client. + +- `proxmox_readonly_reference.py` - the full broker flow (spec section 11): + first call registers and persists agent state, then fetches + `coreci/proxmox`; later calls load the persisted token via `get_state`. +- `minimal_auth_showcase.py` - the minimal pattern: authenticate with the + broker, then use a second tool. + +The flow both templates follow: + +1. The CEO allowlists the tool's app UUID and user id in the broker UI. +2. Fetch the client from `/static/coreci_broker_client.py` and import it. +3. `ensure_registered(state)` registers (idempotent) and merges the token + into your state dict; `put_state(state)` persists it as the tool's + only durable state channel. +4. `read_secret("coreci/", state)` fetches the credential; on a + 401 the client re-registers and retries once. + +Downstream calls default to a safe dry-run with no live credential use; +replace the dry-run block with real calls in your own tool. + +Client module: `coreci_broker_client.py` (stdlib urllib only, zero deps). \ No newline at end of file diff --git a/reference_tools/minimal_auth_showcase.py b/reference_tools/minimal_auth_showcase.py new file mode 100644 index 0000000..2ed621d --- /dev/null +++ b/reference_tools/minimal_auth_showcase.py @@ -0,0 +1,45 @@ +"""title: Minimal Auth Showcase [CoreCI] +author: coreci +description: Minimal Open WebUI tool template showing the CoreCI Broker pattern - authenticate with the broker first, then use a second tool. +version: 0.1.0 +requirements: [] +licence: MIT +""" +import coreci_broker_client + + +class Tools: + class Valves: + base_url = "" + agent_id = "" + app_uuid = "" + user_id = "" + + def __init__(self): + self.valves = self.Valves() + self._state = {} + + def _client(self): + return coreci_broker_client.BrokerClient( + self.valves.base_url, self.valves.agent_id, + self.valves.app_uuid, self.valves.user_id, ["coreci"], + ) + + def demo(self, context=None) -> dict: + """Demonstrate broker auth before any downstream tool call. + + Ensures the agent is registered with the broker (idempotent), then + reads the agent's own secret through the client: the credential + fetch itself proves the auth path works before any second tool + would use the credential. The secret material never appears in the + returned marker. + + :param context: optional Open WebUI call context, unused + :return: marker dict with registered and secret_read flags + """ + client = self._client() + client.ensure_registered(self._state) + client.read_secret("coreci/%s" % self.valves.agent_id, self._state) + return {"registered": True, "secret_read": True, + "path": "coreci/%s" % self.valves.agent_id, + "note": "auth first, then use"} \ No newline at end of file diff --git a/reference_tools/proxmox_readonly_reference.py b/reference_tools/proxmox_readonly_reference.py new file mode 100644 index 0000000..a1be687 --- /dev/null +++ b/reference_tools/proxmox_readonly_reference.py @@ -0,0 +1,104 @@ +"""title: Proxmox Read-Only Reference [CoreCI] +author: coreci +description: Open WebUI tool template for the CoreCI Broker flow (spec section 11) - register, persist agent state, read coreci/proxmox, then act; downstream Proxmox calls default to a safe dry-run. +version: 0.1.0 +requirements: [] +licence: MIT +""" +import datetime + +import coreci_broker_client + +SECRET_PATH = "coreci/proxmox" +DRY_RUN_NOTE = ( + "reference dry-run: the broker credential was fetched but stays unused; " + "replace this block with a real read-only Proxmox call to go live" +) + + +class Tools: + class Valves: + base_url = "" + agent_id = "" + app_uuid = "" + user_id = "" + + def __init__(self): + self.valves = self.Valves() + self._state = {} + + def _client(self): + return coreci_broker_client.BrokerClient( + self.valves.base_url, self.valves.agent_id, + self.valves.app_uuid, self.valves.user_id, ["proxmox", "readonly"], + ) + + def _persist(self, client): + now = datetime.datetime.now(datetime.timezone.utc).isoformat() + self._state.setdefault("first_registered_at", now) + self._state["last_seen_at"] = now + client.put_state(self._state) + + def _load_persisted(self, client): + document = client.get_state(self._state) + if document is not None: + for key in document: + self._state.setdefault(key, document[key]) + + def _credential(self): + client = self._client() + first_call = self._state.get("current_token") is None + fingerprint = (self._state.get("current_token"), + self._state.get("token_expires_at")) + if first_call: + client.ensure_registered(self._state) + self._persist(client) + else: + self._load_persisted(client) + document = client.read_secret(SECRET_PATH, self._state) + current = (self._state.get("current_token"), + self._state.get("token_expires_at")) + if not first_call and current != fingerprint: + self._persist(client) + return document + + def _dry_run(self, action, document): + return { + "dry_run": True, + "action": action, + "credential_present": isinstance(document, dict) and "data" in document, + "note": DRY_RUN_NOTE, + } + + def list_nodes(self, context=None) -> dict: + """List the Proxmox cluster nodes as a safe dry-run by default. + + First call registers with the broker (idempotent) and persists the + agent state; subsequent calls load the persisted token and reuse + it. The credential is fetched from coreci/proxmox before any + downstream action, and on a 401 the client re-registers with a + single retry after which the new token is persisted again. The + reference template then returns a dry-run marker and performs no + live Proxmox call: replace the dry-run block with a real read-only + API call when wiring live infrastructure; this template ships no + live-credential use path. + + :param context: optional Open WebUI call context, unused + :return: dry-run response dict with credential presence + """ + return self._dry_run("list_nodes", self._credential()) + + def vm_status(self, vm_id: str, context=None) -> dict: + """Report one Proxmox VM status as a safe dry-run by default. + + Follows the same broker flow as list_nodes: register and persist + the agent state on the first call, load the persisted token on + later calls, fetch the coreci/proxmox credential, then act. The + dry-run block is the default and no live Proxmox client is wired + in - replace it with a real read-only status call for live use. + + :param vm_id: Proxmox VM identifier to look up + :param context: optional Open WebUI call context, unused + :return: dry-run response dict with credential presence + """ + return self._dry_run("vm_status", self._credential()) \ No newline at end of file diff --git a/tests/test_acceptance_full.py b/tests/test_acceptance_full.py new file mode 100644 index 0000000..86a4b25 --- /dev/null +++ b/tests/test_acceptance_full.py @@ -0,0 +1,345 @@ +import ast +import base64 +import contextlib +import datetime +import importlib.util +import io +import json +import os +import sys +import tempfile +import time +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +BROKER_SOURCE_PATH = os.path.join(REPO_ROOT, "coreci_broker.py") +CLIENT_SOURCE_PATH = os.path.join(REPO_ROOT, "coreci_broker_client.py") +CLIENT_STATIC_PATH = "/static/coreci_broker_client.py" +LOGIN_PATH = "/api/broker/auth/login" +LOGOUT_PATH = "/api/broker/auth/logout" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +AUDIT_PATH = "/api/broker/audit" +SECRETS_PREFIX = "/api/broker/secrets/" +PROXMOX_PATH = "coreci/proxmox" +GITEA_PATH = "coreci/gitea" +PROXMOX_SECRET = {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"} +GITEA_SECRET = {"url": "https://git.coreci.dev", "token": "git-tok"} + +AC_PIN_IDS = { + "AC1": "test_acceptance_p2.AcceptanceP2Tests.test_ac1_first_boot_printed_pass_login_via_ui_login_page", + "AC2": "test_acceptance_p2.AcceptanceP2Tests.test_ac2_app_registration_lifecycle_via_session_cookie", + "AC3": "test_acceptance_p2.AcceptanceP2Tests.test_ac3_user_allowlist_lifecycle_via_session_cookie", + "AC4": "test_acceptance_p3.AcceptanceP3Tests.test_ac4_derivation_identical_both_sides_stored_as_slow_hash", + "AC5": "test_acceptance_p4.AcceptanceP4Tests.test_ac5_valid_token_reads_secret_tampered_token_is_401", + "AC6": "test_acceptance_p4.AcceptanceP4Tests.test_ac6_cross_path_read_is_403_with_denied_audit", + "AC7": "test_acceptance_p3.AcceptanceP3Tests.test_ac7_revoked_or_unknown_uuid_denied_existing_token_honored", + "AC8": "test_acceptance_p3.AcceptanceP3Tests.test_ac8_unknown_or_revoked_user_denied", + "AC9": "test_acceptance_p2.AcceptanceP2Tests.test_ac9_login_and_logout_audit_lines_with_null_identity", + "AC10": "test_acceptance_p3.AcceptanceP3Tests.test_ac10_expiry_401_reregister_refreshes_expiry_and_audits", + "AC11": "test_acceptance_full.AcceptanceFullTests.test_ac11_fresh_boot_state_files_0600_umask_independent", + "AC12": "test_acceptance_p3.AcceptanceP3Tests.test_ac12_state_roundtrip_with_identity_enforcement", + "AC13": "test_acceptance_full.AcceptanceFullTests.test_ac13_client_module_fetch_byte_identical_importable_standalone", + "AC14": "test_acceptance_full.AcceptanceFullTests.test_ac14_coreci_broker_line_count_under_1400", +} + + +def _basic_header(password): + raw = "%s:%s" % (ADMIN_USER, password) + return {"Authorization": "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii")} + + +class AcceptanceFullTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.creds_path = os.path.join(self.tmp, "creds.json") + with open(self.creds_path, "w", encoding="utf-8") as handle: + handle.write(json.dumps({PROXMOX_PATH: PROXMOX_SECRET, GITEA_PATH: GITEA_SECRET})) + + def _env(self, root=None, extra=None): + root = self.tmp if root is None else root + env = { + "BROKER_ENV_PATH": os.path.join(root, ".env"), + "BROKER_DB_PATH": os.path.join(root, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(root, "broker.audit.log"), + "BROKER_CREDS_SOURCE": "file", + "BROKER_CREDS_FILE": self.creds_path, + "BROKER_UI_ROOT": os.path.join(REPO_ROOT, "coreci_broker_ui"), + "BROKER_STATIC_ROOT": REPO_ROOT, + } + env.update(extra or {}) + return env + + def _boot(self, env): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + broker = coreci_broker.boot(dict(env)) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + return broker, buffer.getvalue() + + def _serve(self, broker): + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return "http://127.0.0.1:%d" % server.server_address[1] + + def _printed_pass(self, output): + printed = [ + line.split("=", 1)[1] + for line in output.splitlines() + if line.startswith("BROKER_ADMIN_PASS=") + ] + self.assertEqual(1, len(printed)) + return printed[0] + + def _request(self, base, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, response.read(), response.headers + except urllib.error.HTTPError as error: + return error.code, error.read(), error.headers + + def _request_json(self, base, method, path, payload=None, headers=None): + status, body, response_headers = self._request(base, method, path, payload, headers) + return status, json.loads(body.decode("utf-8")), response_headers + + def _assert_null_identity(self, entry): + self.assertIsNone(entry["user_id"]) + self.assertIsNone(entry["app_registration_uuid"]) + self.assertIsNone(entry["agent_id"]) + + def _assert_tool_identity(self, entry, app_uuid): + self.assertEqual("ceo", entry["user_id"]) + self.assertEqual(app_uuid, entry["app_registration_uuid"]) + self.assertEqual("proxmox", entry["agent_id"]) + + def _assert_state_files_0600(self, env): + for key in ("BROKER_DB_PATH", "BROKER_AUDIT_LOG_PATH", "BROKER_ENV_PATH"): + path = env[key] + self.assertTrue(os.path.isfile(path), path) + self.assertEqual(0o600, os.stat(path).st_mode & 0o777, path) + + def test_ac11_fresh_boot_state_files_0600_umask_independent(self): + env = self._env() + self._boot(env) + self._assert_state_files_0600(env) + other = tempfile.TemporaryDirectory() + self.addCleanup(other.cleanup) + other_env = self._env(root=other.name) + previous_umask = os.umask(0o777) + try: + probe = os.path.join(other.name, "umask-probe") + os.close(os.open(probe, os.O_CREAT | os.O_WRONLY, 0o666)) + self.assertEqual(0o000, os.stat(probe).st_mode & 0o777) + self._boot(other_env) + finally: + os.umask(previous_umask) + self._assert_state_files_0600(other_env) + + def test_ac13_client_module_fetch_byte_identical_importable_standalone(self): + broker, _ = self._boot(self._env()) + base = self._serve(broker) + with open(CLIENT_SOURCE_PATH, "rb") as handle: + expected = handle.read() + with urllib.request.urlopen(base + CLIENT_STATIC_PATH, timeout=10) as response: + self.assertEqual(200, response.status) + fetched = response.read() + self.assertEqual(expected, fetched) + imports = set() + for node in ast.walk(ast.parse(fetched.decode("utf-8"))): + if isinstance(node, ast.Import): + imports.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imports.add(node.module or "") + self.assertTrue(imports) + for name in imports: + self.assertIn(name.split(".")[0], sys.stdlib_module_names, name) + copy_path = os.path.join(self.tmp, "fetched_coreci_broker_client.py") + with open(copy_path, "wb") as handle: + handle.write(fetched) + spec = importlib.util.spec_from_file_location("full_fetched_client", copy_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.assertTrue(callable(module.BrokerClient)) + self.assertTrue(callable(module.fetch_client_module)) + for method in ("ensure_registered", "get_state", "put_state", "read_secret"): + self.assertTrue(callable(getattr(module.BrokerClient, method)), method) + + def test_ac14_coreci_broker_line_count_under_1400(self): + with open(BROKER_SOURCE_PATH, "r", encoding="utf-8") as handle: + line_count = sum(1 for _ in handle) + self.assertLess(line_count, 1400) + + def test_milestone_story_full_happy_path_consolidates_ac1_ac10_ac12(self): + env = self._env(extra={"BROKER_TOKEN_TTL": "1"}) + first, output = self._boot(env) + printed_pass = self._printed_pass(output) + self.assertGreaterEqual(len(printed_pass), 32) + self.assertEqual(1, output.count(printed_pass)) + pepper = None + with open(env["BROKER_ENV_PATH"], "r", encoding="utf-8") as handle: + for line in handle: + if line.startswith("BROKER_INSTANCE_PEPPER="): + pepper = line.split("=", 1)[1].strip() + self.assertIsNotNone(pepper) + self.assertNotIn(pepper, output) + second, silent = self._boot(env) + self.assertEqual("", silent) + second.audit.close() + second.store.close() + base = self._serve(first) + status, body, headers = self._request(base, "GET", "/ui/login") + self.assertEqual(200, status) + self.assertTrue(headers.get("Content-Type", "").startswith("text/html")) + self.assertIn('name="password"', body.decode("utf-8")) + status, body, response_headers = self._request_json( + base, "POST", LOGIN_PATH, {"username": ADMIN_USER, "password": printed_pass} + ) + self.assertEqual(200, status) + self.assertEqual({"ok": True, "username": ADMIN_USER}, body) + raw_cookie = response_headers.get("Set-Cookie") + self.assertTrue(raw_cookie) + self.assertTrue(raw_cookie.startswith("broker_session=")) + self.assertIn("HttpOnly", raw_cookie) + self.assertIn("SameSite=Lax", raw_cookie) + cookie = {"Cookie": raw_cookie.split(";", 1)[0]} + app_uuid = str(uuidlib.uuid4()) + status, body, _ = self._request_json( + base, "POST", APPS_PATH, + {"uuid": app_uuid, "name": "proxmox-tool"}, headers=cookie, + ) + self.assertEqual(201, status) + self.assertEqual({"uuid": app_uuid, "status": "active"}, body) + status, body, _ = self._request_json( + base, "POST", USERS_PATH, + {"user_id": "ceo", "email": "ceo@coreci.dev"}, headers=cookie, + ) + self.assertEqual(201, status) + self.assertEqual({"user_id": "ceo", "status": "active"}, body) + with urllib.request.urlopen(base + CLIENT_STATIC_PATH, timeout=10) as response: + self.assertEqual(200, response.status) + client_source = response.read() + with open(CLIENT_SOURCE_PATH, "rb") as handle: + self.assertEqual(handle.read(), client_source) + copy_path = os.path.join(self.tmp, "story_coreci_broker_client.py") + with open(copy_path, "wb") as handle: + handle.write(client_source) + spec = importlib.util.spec_from_file_location("full_story_client", copy_path) + client_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(client_module) + client = client_module.BrokerClient(base, "proxmox", app_uuid, "ceo", ["coreci"]) + state = {} + client.ensure_registered(state) + self.assertEqual(44, len(state["current_token"])) + self.assertTrue(state["current_token"].isascii()) + self.assertIn("pepper", state) + self.assertEqual("proxmox", state["agent_id"]) + self.assertEqual(app_uuid, state["app_registration_uuid"]) + self.assertEqual("ceo", state["user_id"]) + self.assertTrue(client.put_state(state)) + snapshot = dict(state) + self.assertEqual(snapshot, client.get_state(state)) + document = client.read_secret(PROXMOX_PATH, state) + self.assertEqual({"path", "data", "version", "updated_at"}, set(document)) + self.assertEqual(PROXMOX_PATH, document["path"]) + self.assertEqual(PROXMOX_SECRET, document["data"]) + with self.assertRaises(client_module.BrokerError) as caught: + client.read_secret(GITEA_PATH, state) + self.assertEqual(403, caught.exception.status) + expired_at = state["token_expires_at"] + time.sleep(1.2) + document = client.read_secret(PROXMOX_PATH, state) + self.assertEqual(PROXMOX_SECRET, document["data"]) + self.assertGreater( + datetime.datetime.fromisoformat(state["token_expires_at"]), + datetime.datetime.fromisoformat(expired_at), + ) + status, body, _ = self._request_json(base, "POST", LOGOUT_PATH, headers=cookie) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + status, body, _ = self._request_json( + base, "GET", AUDIT_PATH + "?limit=1000", headers=_basic_header(printed_pass) + ) + self.assertEqual(200, status) + self.assertIsNone(body["next_since"]) + entries = body["entries"] + + def select(action, result=None): + return [ + entry for entry in entries + if entry["action"] == action and (result is None or entry["result"] == result) + ] + + logins = select("login", "ok") + self.assertEqual(1, len(logins)) + self.assertEqual(LOGIN_PATH, logins[0]["path"]) + self._assert_null_identity(logins[0]) + add_apps = select("add_app", "ok") + self.assertEqual(1, len(add_apps)) + self.assertEqual(APPS_PATH + "/" + app_uuid, add_apps[0]["path"]) + self._assert_null_identity(add_apps[0]) + add_users = select("add_user", "ok") + self.assertEqual(1, len(add_users)) + self.assertEqual(USERS_PATH + "/ceo", add_users[0]["path"]) + self._assert_null_identity(add_users[0]) + registers = select("register", "ok") + self.assertGreaterEqual(len(registers), 2) + for entry in registers: + self.assertEqual(REGISTER_PATH, entry["path"]) + self._assert_tool_identity(entry, app_uuid) + ok_reads = select("read", "ok") + self.assertGreaterEqual(len(ok_reads), 2) + for entry in ok_reads: + self.assertEqual(PROXMOX_PATH, entry["path"]) + self._assert_tool_identity(entry, app_uuid) + denied_reads = select("read", "denied") + self.assertEqual(1, len(denied_reads)) + self.assertEqual(GITEA_PATH, denied_reads[0]["path"]) + self._assert_tool_identity(denied_reads[0], app_uuid) + auth_denied = [ + entry for entry in select("denied", "denied") + if entry["path"] == SECRETS_PREFIX + PROXMOX_PATH + ] + self.assertGreaterEqual(len(auth_denied), 1) + self.assertEqual(app_uuid, auth_denied[0]["app_registration_uuid"]) + self.assertIsNone(auth_denied[0]["user_id"]) + self.assertIsNone(auth_denied[0]["agent_id"]) + logouts = select("logout", "ok") + self.assertEqual(1, len(logouts)) + self.assertEqual(LOGOUT_PATH, logouts[0]["path"]) + self._assert_null_identity(logouts[0]) + + def test_coverage_tally_pins_ac1_through_ac14_across_suite(self): + self.assertEqual(14, len(AC_PIN_IDS)) + loader = unittest.TestLoader() + suite = loader.discover(os.path.dirname(os.path.abspath(__file__))) + stack, ids = [suite], set() + while stack: + item = stack.pop() + if isinstance(item, unittest.TestSuite): + stack.extend(item) + else: + ids.add(item.id()) + for ac in sorted(AC_PIN_IDS): + with self.subTest(ac=ac): + self.assertIn(AC_PIN_IDS[ac], ids) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_acceptance_p1.py b/tests/test_acceptance_p1.py new file mode 100644 index 0000000..6f73b5d --- /dev/null +++ b/tests/test_acceptance_p1.py @@ -0,0 +1,114 @@ +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.request + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + + +class AcceptanceP1Tests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + } + + def _boot(self): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + broker = coreci_broker.boot(dict(self.env)) + return broker, buffer.getvalue() + + def test_ac14_coreci_broker_line_count_under_1400(self): + source_path = os.path.join(os.path.dirname(__file__), "..", "coreci_broker.py") + with open(source_path, "r", encoding="utf-8") as handle: + line_count = sum(1 for _ in handle) + self.assertLess(line_count, 1400) + + def test_first_boot_prints_pass_once_second_boot_silent_with_0600_state(self): + first = io.StringIO() + with contextlib.redirect_stdout(first): + broker = coreci_broker.boot(dict(self.env)) + server = coreci_broker.serve("127.0.0.1", 0, broker) + db_path = self.env["BROKER_DB_PATH"] + audit_path = self.env["BROKER_AUDIT_LOG_PATH"] + env_path = self.env["BROKER_ENV_PATH"] + self.assertTrue(os.path.isfile(db_path)) + self.assertTrue(os.path.isfile(audit_path)) + self.assertTrue(os.path.isfile(env_path)) + self.assertEqual(0o600, os.stat(db_path).st_mode & 0o777) + self.assertEqual(0o600, os.stat(audit_path).st_mode & 0o777) + admin_pass = broker.config["BROKER_ADMIN_PASS"] + pepper = broker.config["BROKER_INSTANCE_PEPPER"] + self.assertIn(admin_pass, first.getvalue()) + self.assertEqual(1, first.getvalue().count(admin_pass)) + self.assertNotIn(pepper, first.getvalue()) + server.shutdown() + server.server_close() + broker.audit.close() + broker.store.close() + second = io.StringIO() + with contextlib.redirect_stdout(second): + rebooted = coreci_broker.boot(dict(self.env)) + self.assertEqual("", second.getvalue()) + rebooted.audit.close() + rebooted.store.close() + + def test_restart_preserves_allowlist_row_and_audit_history(self): + broker, _ = self._boot() + server = coreci_broker.serve("127.0.0.1", 0, broker) + user_id = "acceptance-restart-user" + broker.store.conn.execute( + "INSERT INTO user_allowlist (user_id, email, status, created_at) VALUES (?, ?, ?, ?)", + (user_id, "ceo@coreci.dev", "active", coreci_broker._utc_now_iso()), + ) + broker.store.conn.commit() + broker.audit.write("add_user", user_id=user_id) + server.shutdown() + server.server_close() + broker.audit.close() + broker.store.close() + rebooted, _ = self._boot() + self.addCleanup(rebooted.audit.close) + self.addCleanup(rebooted.store.close) + row = rebooted.store.conn.execute( + "SELECT email, status FROM user_allowlist WHERE user_id = ?", (user_id,) + ).fetchone() + self.assertEqual(("ceo@coreci.dev", "active"), row) + audit_path = self.env["BROKER_AUDIT_LOG_PATH"] + with open(audit_path, "r", encoding="utf-8") as handle: + lines = [line for line in handle if line.strip()] + self.assertEqual(1, len(lines)) + entry = json.loads(lines[0]) + self.assertEqual("add_user", entry["action"]) + self.assertEqual(user_id, entry["user_id"]) + self.assertEqual(0o600, os.stat(audit_path).st_mode & 0o777) + + def test_healthz_end_to_end(self): + broker, _ = self._boot() + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + port = server.server_address[1] + with urllib.request.urlopen( + "http://127.0.0.1:%d/healthz" % port, timeout=5 + ) as response: + self.assertEqual(200, response.status) + payload = json.loads(response.read().decode("utf-8")) + self.assertEqual({"ok": True}, payload) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_acceptance_p2.py b/tests/test_acceptance_p2.py new file mode 100644 index 0000000..fac0381 --- /dev/null +++ b/tests/test_acceptance_p2.py @@ -0,0 +1,241 @@ +import base64 +import contextlib +import io +import json +import os +import sys +import tempfile +import time +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +LOGIN_PATH = "/api/broker/auth/login" +LOGOUT_PATH = "/api/broker/auth/logout" +WHOAMI_PATH = "/api/broker/auth/whoami" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +AUDIT_PATH = "/api/broker/audit" + + +class AcceptanceP2Tests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + } + + def _boot_served(self, extra=None): + env = dict(self.env) + env.update(extra or {}) + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + broker = coreci_broker.boot(env) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + printed = [ + line.split("=", 1)[1] + for line in buffer.getvalue().splitlines() + if line.startswith("BROKER_ADMIN_PASS=") + ] + self.assertEqual(1, len(printed)) + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return "http://127.0.0.1:%d" % server.server_address[1], printed[0] + + def _request(self, base, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, response.read(), response.headers + except urllib.error.HTTPError as error: + return error.code, error.read(), error.headers + + def _request_json(self, base, method, path, payload=None, headers=None): + status, body, response_headers = self._request( + base, method, path, payload, headers + ) + return status, json.loads(body.decode("utf-8")), response_headers + + def _get_page(self, base, path): + status, body, headers = self._request(base, "GET", path) + self.assertEqual(200, status) + self.assertTrue(headers.get("Content-Type", "").startswith("text/html")) + return body.decode("utf-8") + + def _basic(self, password): + raw = "%s:%s" % (ADMIN_USER, password) + encoded = base64.b64encode(raw.encode("utf-8")).decode("ascii") + return {"Authorization": "Basic " + encoded} + + def _login(self, base, password): + status, body, headers = self._request_json( + base, "POST", LOGIN_PATH, {"username": ADMIN_USER, "password": password} + ) + self.assertEqual(200, status) + self.assertEqual({"ok": True, "username": ADMIN_USER}, body) + raw = headers.get("Set-Cookie") + self.assertTrue(raw) + self.assertTrue(raw.startswith("broker_session=")) + self.assertIn("HttpOnly", raw) + self.assertIn("SameSite=Lax", raw) + return {"Cookie": raw.split(";", 1)[0]} + + def _find(self, rows, key, value): + for row in rows: + if row[key] == value: + return row + self.fail("no row with %s=%r" % (key, value)) + + def _ok_entries(self, entries): + return [entry for entry in entries if entry["result"] == "ok"] + + def _assert_null_identity(self, entries): + self.assertTrue(entries) + for entry in entries: + self.assertIsNone(entry["user_id"]) + self.assertIsNone(entry["app_registration_uuid"]) + self.assertIsNone(entry["agent_id"]) + + def test_ac1_first_boot_printed_pass_login_via_ui_login_page(self): + base, printed_pass = self._boot_served() + self._get_page(base, "/ui/login") + cookie = self._login(base, printed_pass) + status, body, _ = self._request_json( + base, "GET", WHOAMI_PATH, headers=cookie + ) + self.assertEqual(200, status) + self.assertEqual(ADMIN_USER, body["username"]) + + def test_ac2_app_registration_lifecycle_via_session_cookie(self): + base, printed_pass = self._boot_served() + cookie = self._login(base, printed_pass) + app_uuid = str(uuidlib.uuid4()) + status, body, _ = self._request_json( + base, "POST", APPS_PATH, + {"uuid": app_uuid, "name": "proxmox-tool"}, headers=cookie, + ) + self.assertEqual(201, status) + self.assertEqual({"uuid": app_uuid, "status": "active"}, body) + status, body, _ = self._request_json(base, "GET", APPS_PATH, headers=cookie) + self.assertEqual(200, status) + row = self._find(body["registrations"], "uuid", app_uuid) + self.assertEqual("proxmox-tool", row["name"]) + self.assertEqual("active", row["status"]) + self._get_page(base, "/ui/app_registrations") + status, body, _ = self._request_json( + base, "DELETE", APPS_PATH + "/" + app_uuid, headers=cookie + ) + self.assertEqual(200, status) + self.assertEqual("revoked", body["status"]) + status, body, _ = self._request_json(base, "GET", APPS_PATH, headers=cookie) + self.assertEqual( + "revoked", self._find(body["registrations"], "uuid", app_uuid)["status"] + ) + status, body, _ = self._request_json( + base, "POST", APPS_PATH + "/" + app_uuid + "/reinstate", headers=cookie + ) + self.assertEqual(200, status) + self.assertEqual("active", body["status"]) + status, body, _ = self._request_json(base, "GET", APPS_PATH, headers=cookie) + self.assertEqual( + "active", self._find(body["registrations"], "uuid", app_uuid)["status"] + ) + + def test_ac3_user_allowlist_lifecycle_via_session_cookie(self): + base, printed_pass = self._boot_served() + cookie = self._login(base, printed_pass) + status, body, _ = self._request_json( + base, "POST", USERS_PATH, + {"user_id": "ceo", "email": "ceo@coreci.dev"}, headers=cookie, + ) + self.assertEqual(201, status) + self.assertEqual({"user_id": "ceo", "status": "active"}, body) + status, body, _ = self._request_json(base, "GET", USERS_PATH, headers=cookie) + self.assertEqual(200, status) + row = self._find(body["allowlist"], "user_id", "ceo") + self.assertEqual("ceo@coreci.dev", row["email"]) + self.assertEqual("active", row["status"]) + self._get_page(base, "/ui/user_allowlist") + status, body, _ = self._request_json( + base, "DELETE", USERS_PATH + "/ceo", headers=cookie + ) + self.assertEqual(200, status) + self.assertEqual("revoked", body["status"]) + status, body, _ = self._request_json(base, "GET", USERS_PATH, headers=cookie) + self.assertEqual( + "revoked", self._find(body["allowlist"], "user_id", "ceo")["status"] + ) + status, body, _ = self._request_json( + base, "POST", USERS_PATH + "/ceo/reinstate", headers=cookie + ) + self.assertEqual(200, status) + self.assertEqual("active", body["status"]) + status, body, _ = self._request_json(base, "GET", USERS_PATH, headers=cookie) + self.assertEqual( + "active", self._find(body["allowlist"], "user_id", "ceo")["status"] + ) + + def test_ac9_login_and_logout_audit_lines_with_null_identity(self): + base, printed_pass = self._boot_served() + cookie = self._login(base, printed_pass) + status, body, _ = self._request_json( + base, "POST", LOGOUT_PATH, headers=cookie + ) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + admin = self._basic(printed_pass) + status, body, _ = self._request_json( + base, "GET", AUDIT_PATH + "?action=login", headers=admin + ) + self.assertEqual(200, status) + logins = self._ok_entries(body["entries"]) + self.assertEqual(["login"], [entry["action"] for entry in logins]) + self._assert_null_identity(logins) + status, body, _ = self._request_json( + base, "GET", AUDIT_PATH + "?action=logout", headers=admin + ) + self.assertEqual(200, status) + logouts = self._ok_entries(body["entries"]) + self.assertEqual(["logout"], [entry["action"] for entry in logouts]) + self._assert_null_identity(logouts) + + def test_ui_pages_served_and_static_hygiene(self): + base, printed_pass = self._boot_served() + dashboard = self._get_page(base, "/ui/dashboard") + self.assertIn("Dashboard", dashboard) + audit_page = self._get_page(base, "/ui/audit") + self.assertNotIn("export", audit_page.lower()) + login_page = self._get_page(base, "/ui/login") + self.assertIn('name="username"', login_page) + self.assertIn('name="password"', login_page) + + def test_session_ttl_expiry_end_to_end(self): + base, printed_pass = self._boot_served({"BROKER_SESSION_TTL": "1"}) + cookie = self._login(base, printed_pass) + status, _, _ = self._request(base, "GET", WHOAMI_PATH, headers=cookie) + self.assertEqual(200, status) + time.sleep(1.2) + status, body, _ = self._request_json( + base, "GET", WHOAMI_PATH, headers=cookie + ) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_acceptance_p3.py b/tests/test_acceptance_p3.py new file mode 100644 index 0000000..fe87d16 --- /dev/null +++ b/tests/test_acceptance_p3.py @@ -0,0 +1,309 @@ +import base64 +import contextlib +import datetime +import hashlib +import io +import json +import os +import sys +import tempfile +import time +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +AUDIT_PATH = "/api/broker/audit" +AGENT_PATH = "/api/broker/agents/%s" +STATE_PATH = "/api/broker/agents/%s/state" +AUTH_401_BODY = {"error": 401, "message": "invalid app credentials"} +REGISTER_403_BODY = {"error": 403, "message": "registration not allowed"} + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +def _derive(pepper, app_uuid, user_id, agent_id, tags): + material = "%s:%s:%s:%s:%s" % ( + pepper, app_uuid, user_id, agent_id, ",".join(sorted(tags)) + ) + digest = hashlib.sha256(material.encode("utf-8")).digest() + return base64.b64encode(digest).decode("ascii") + + +class AcceptanceP3Tests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + self.broker, self.base = self._boot() + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed(self.base, self.app_uuid, self.user_id) + + def _boot(self, extra=None): + env = dict(self.env) + env.update(extra or {}) + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(env) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(server.shutdown) + self.addCleanup(server.server_close) + return broker, "http://127.0.0.1:%d" % server.server_address[1] + + def _request(self, method, path, payload=None, headers=None, base=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + (base or self.base) + path, data=data, method=method + ) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8") + return error.code, json.loads(raw) if raw else None + + def _seed(self, base, app_uuid, user_id): + status, body = self._request( + "POST", APPS_PATH, {"uuid": app_uuid}, headers=self.admin, base=base + ) + self.assertEqual(201, status) + status, body = self._request( + "POST", USERS_PATH, {"user_id": user_id}, headers=self.admin, base=base + ) + self.assertEqual(201, status) + + def _register(self, agent_id, tags, base=None, app_uuid=None, user_id=None): + payload = { + "agent_id": agent_id, + "app_registration_uuid": app_uuid or self.app_uuid, + "user_id": user_id or self.user_id, + "tags": list(tags), + } + return self._request("POST", REGISTER_PATH, payload, base=base) + + def _app_header(self, token, app_uuid=None): + return {"Authorization": "App %s %s" % (app_uuid or self.app_uuid, token)} + + def _audit_entries(self, base, query): + status, body = self._request( + "GET", AUDIT_PATH + query, headers=self.admin, base=base + ) + self.assertEqual(200, status) + return body["entries"] + + def test_ac4_derivation_identical_both_sides_stored_as_slow_hash(self): + agent_id = "px-1" + tags = ["zeta", "alpha"] + status, body = self._register(agent_id, tags) + self.assertEqual(201, status) + self.assertTrue(body["created"]) + token = body["token"] + pepper = self.broker.store.get_config("BROKER_INSTANCE_PEPPER") + self.assertEqual(pepper, body["pepper"]) + broker_side = _derive(pepper, self.app_uuid, self.user_id, agent_id, tags) + tool_side = _derive(body["pepper"], self.app_uuid, self.user_id, agent_id, tags) + self.assertEqual(broker_side, token) + self.assertEqual(tool_side, token) + self.assertEqual(44, len(token)) + self.assertTrue(token.isascii()) + row = self.broker.store.get_agent(agent_id, self.app_uuid) + stored = row[4] + self.assertNotEqual(token, stored) + if coreci_broker.bcrypt is None: + self.assertTrue(stored.startswith("scrypt$")) + else: + self.assertTrue(stored.startswith(coreci_broker.BCRYPT_PREFIXES)) + self.assertTrue(coreci_broker.verify_token(token, stored)) + self.assertFalse(coreci_broker.verify_token(token, token)) + self.assertEqual(self.user_id, row[2]) + self.assertEqual(json.dumps(sorted(tags)), row[3]) + self.assertEqual(body["token_expires_at"], row[5]) + + def test_ac7_revoked_or_unknown_uuid_denied_existing_token_honored(self): + status, first = self._register("tool-1", ["coreci"]) + self.assertEqual(201, status) + token = first["token"] + status, body = self._request( + "DELETE", APPS_PATH + "/" + self.app_uuid, headers=self.admin + ) + self.assertEqual(200, status) + self.assertEqual({"uuid": self.app_uuid, "status": "revoked"}, body) + status, body = self._register("tool-2", ["coreci"]) + self.assertEqual(403, status) + self.assertEqual(REGISTER_403_BODY, body) + self.assertIsNone(self.broker.store.get_agent("tool-2", self.app_uuid)) + status, body = self._register("tool-3", ["coreci"], app_uuid=str(uuidlib.uuid4())) + self.assertEqual(403, status) + self.assertEqual(REGISTER_403_BODY, body) + status, body = self._request( + "GET", AGENT_PATH % "tool-1", headers=self._app_header(token) + ) + self.assertEqual(200, status) + self.assertEqual("tool-1", body["agent_id"]) + self.assertEqual(self.user_id, body["user_id"]) + self.assertEqual(self.app_uuid, body["app_registration_uuid"]) + self.assertEqual(["coreci"], body["tags"]) + + def test_ac8_unknown_or_revoked_user_denied(self): + status, body = self._register("tool-1", ["coreci"], user_id="intern") + self.assertEqual(403, status) + self.assertEqual(REGISTER_403_BODY, body) + status, body = self._request( + "POST", USERS_PATH, {"user_id": "intern"}, headers=self.admin + ) + self.assertEqual(201, status) + status, body = self._register("tool-1", ["coreci"], user_id="intern") + self.assertEqual(201, status) + self.assertTrue(body["created"]) + status, body = self._request( + "DELETE", USERS_PATH + "/intern", headers=self.admin + ) + self.assertEqual(200, status) + self.assertEqual({"user_id": "intern", "status": "revoked"}, body) + status, body = self._register("tool-1", ["coreci"], user_id="intern") + self.assertEqual(403, status) + self.assertEqual(REGISTER_403_BODY, body) + + def test_ac10_expiry_401_reregister_refreshes_expiry_and_audits(self): + broker, base = self._boot({ + "BROKER_DB_PATH": os.path.join(self.tmp, "ttl.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "ttl.audit.log"), + "BROKER_TOKEN_TTL": "1", + }) + self._seed(base, self.app_uuid, self.user_id) + agent_id = "ttl-tool" + status, first = self._register(agent_id, ["coreci"], base=base) + self.assertEqual(201, status) + delta = (datetime.datetime.fromisoformat(first["token_expires_at"]) + - datetime.datetime.now(datetime.timezone.utc)).total_seconds() + self.assertGreater(delta, 0) + self.assertLess(delta, 3) + status, body = self._request( + "GET", AGENT_PATH % agent_id, + headers=self._app_header(first["token"]), base=base, + ) + self.assertEqual(200, status) + time.sleep(1.2) + status, body = self._request( + "GET", AGENT_PATH % agent_id, + headers=self._app_header(first["token"]), base=base, + ) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + status, body = self._register(agent_id, ["coreci"], base=base, user_id="ghost-user") + self.assertEqual(403, status) + status, second = self._register(agent_id, ["coreci"], base=base) + self.assertEqual(200, status) + self.assertFalse(second["created"]) + self.assertEqual(first["token"], second["token"]) + self.assertIn("pepper", second) + self.assertGreater( + datetime.datetime.fromisoformat(second["token_expires_at"]), + datetime.datetime.fromisoformat(first["token_expires_at"]), + ) + status, body = self._request( + "GET", AGENT_PATH % agent_id, + headers=self._app_header(second["token"]), base=base, + ) + self.assertEqual(200, status) + row = broker.store.get_agent(agent_id, self.app_uuid) + self.assertEqual(second["token_expires_at"], row[5]) + self.assertTrue(coreci_broker.verify_token(second["token"], row[4])) + entries = self._audit_entries(base, "?action=register") + self.assertEqual(["ok", "denied", "ok"], [entry["result"] for entry in entries]) + for entry in entries: + self.assertEqual(agent_id, entry["agent_id"]) + self.assertEqual(self.app_uuid, entry["app_registration_uuid"]) + self.assertEqual(REGISTER_PATH, entry["path"]) + self.assertEqual(self.user_id, entries[0]["user_id"]) + self.assertEqual("ghost-user", entries[1]["user_id"]) + self.assertEqual(self.user_id, entries[2]["user_id"]) + + def test_ac12_state_roundtrip_with_identity_enforcement(self): + status, a = self._register("tool-a", ["coreci", "alpha"]) + self.assertEqual(201, status) + status, b = self._register("tool-b", ["coreci"]) + self.assertEqual(201, status) + state = { + "agent_id": "tool-a", + "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, + "tags": ["alpha", "coreci"], + "pepper": a["pepper"], + "current_token": a["token"], + "token_expires_at": a["token_expires_at"], + "first_registered_at": "t0", + "last_seen_at": "t1", + "payload": {"counter": 7, "notes": ["x", "y"], "nested": {"k": None}}, + } + status, body = self._request( + "PUT", STATE_PATH % "tool-a", state, headers=self._app_header(a["token"]) + ) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + status, body = self._request( + "GET", STATE_PATH % "tool-a", headers=self._app_header(a["token"]) + ) + self.assertEqual(200, status) + self.assertEqual(state, body) + self.assertEqual(list(state), list(body)) + status, body = self._request( + "GET", STATE_PATH % "tool-a", headers=self._app_header(b["token"]) + ) + self.assertEqual(403, status) + self.assertEqual({"error": 403, "message": "agent identity mismatch"}, body) + status, body = self._request( + "PUT", STATE_PATH % "tool-a", {"intruder": True}, + headers=self._app_header(b["token"]), + ) + self.assertEqual(403, status) + self.assertEqual({"error": 403, "message": "agent identity mismatch"}, body) + status, body = self._request( + "PUT", STATE_PATH % "tool-b", {"mine": True}, + headers=self._app_header(b["token"]), + ) + self.assertEqual(200, status) + status, body = self._request( + "GET", STATE_PATH % "tool-b", headers=self._app_header(b["token"]) + ) + self.assertEqual(200, status) + self.assertEqual({"mine": True}, body) + status, body = self._request( + "GET", STATE_PATH % "ghost-agent", headers=self._app_header(a["token"]) + ) + self.assertEqual(404, status) + self.assertEqual({"error": 404, "message": "not found"}, body) + status, body = self._request( + "GET", STATE_PATH % "tool-a", headers=self._app_header(a["token"]) + ) + self.assertEqual(200, status) + self.assertEqual(state, body) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_acceptance_p4.py b/tests/test_acceptance_p4.py new file mode 100644 index 0000000..2463034 --- /dev/null +++ b/tests/test_acceptance_p4.py @@ -0,0 +1,290 @@ +import ast +import base64 +import contextlib +import importlib.util +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.parse +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +REGISTER_PATH = "/api/broker/agents/register" +AUDIT_PATH = "/api/broker/audit" +SECRETS_PREFIX = "/api/broker/secrets/" +CLIENT_STATIC_PATH = "/static/coreci_broker_client.py" +PROXMOX_PATH = "coreci/proxmox" +GITEA_PATH = "coreci/gitea" +PROXMOX_SECRET = {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"} +GITEA_SECRET = {"url": "https://git.coreci.dev", "token": "git-tok"} +AUTH_401_BODY = {"error": 401, "message": "invalid app credentials"} +GITEA_403_BODY = {"error": 403, "message": "forbidden"} +REFERENCE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, "reference_tools") +) +STANDALONE_CLIENT_IMPORTS = {"json", "time", "urllib.parse", "urllib.request"} + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +def _flip(token): + replacement = "x" if token[21] != "x" else "y" + return token[:21] + replacement + token[22:] + + +class AcceptanceP4Tests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.creds_path = os.path.join(self.tmp, "creds.json") + with open(self.creds_path, "w", encoding="utf-8") as handle: + handle.write(json.dumps( + {PROXMOX_PATH: PROXMOX_SECRET, GITEA_PATH: GITEA_SECRET} + )) + env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + "BROKER_CREDS_SOURCE": "file", + "BROKER_CREDS_FILE": self.creds_path, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(env) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed() + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8") + return error.code, json.loads(raw) if raw else None + + def _seed(self): + status, _ = self._request("POST", APPS_PATH, {"uuid": self.app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": self.user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self, tags): + payload = {"agent_id": "proxmox", "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": list(tags)} + status, body = self._request("POST", REGISTER_PATH, payload) + self.assertEqual(201, status) + return body + + def _app_header(self, token): + return {"Authorization": "App %s %s" % (self.app_uuid, token)} + + def _secret(self, suffix, headers): + return self._request("GET", SECRETS_PREFIX + suffix, headers=headers) + + def _audit(self, action): + query = urllib.parse.urlencode({"action": action, "limit": "1000"}) + status, body = self._request( + "GET", AUDIT_PATH + "?" + query, headers=self.admin + ) + self.assertEqual(200, status) + return body["entries"] + + def _fetch_client_source(self): + with urllib.request.urlopen( + self.base + CLIENT_STATIC_PATH, timeout=10 + ) as response: + self.assertEqual(200, response.status) + return response.read().decode("utf-8") + + def _import_module(self, name, source): + path = os.path.join(self.tmp, name + ".py") + with open(path, "w", encoding="utf-8") as handle: + handle.write(source) + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _assert_standalone_imports(self, source): + modules = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + modules.add(node.module or "") + self.assertTrue(modules) + for module in modules: + self.assertIn(module, STANDALONE_CLIENT_IMPORTS, module) + + def _bootstrapped_client(self): + module = self._import_module( + "p4_fetched_coreci_broker_client", self._fetch_client_source() + ) + return module.BrokerClient( + self.base, "proxmox", self.app_uuid, self.user_id, ["coreci", "bootstrap"] + ) + + def _reference_tools(self): + source_path = os.path.join(REFERENCE_DIR, "proxmox_readonly_reference.py") + spec = importlib.util.spec_from_file_location( + "p4_proxmox_readonly_reference", source_path + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + tools = module.Tools() + tools.valves.base_url = self.base + tools.valves.agent_id = "proxmox" + tools.valves.app_uuid = self.app_uuid + tools.valves.user_id = self.user_id + return tools + + def _persisted_state(self): + return self.broker.store.get_agent_state("proxmox", self.app_uuid) + + def _assert_tool_identity(self, entry): + self.assertEqual(self.user_id, entry["user_id"]) + self.assertIsNotNone(entry["user_id"]) + self.assertEqual(self.app_uuid, entry["app_registration_uuid"]) + self.assertIsNotNone(entry["app_registration_uuid"]) + self.assertEqual("proxmox", entry["agent_id"]) + self.assertIsNotNone(entry["agent_id"]) + + def test_ac5_valid_token_reads_secret_tampered_token_is_401(self): + registered = self._register(["coreci"]) + status, body = self._secret( + "coreci/proxmox", self._app_header(registered["token"]) + ) + self.assertEqual(200, status) + self.assertEqual({"path", "data", "version", "updated_at"}, set(body)) + self.assertEqual(PROXMOX_PATH, body["path"]) + self.assertEqual(PROXMOX_SECRET, body["data"]) + self.assertRegex(body["version"], r"^[0-9a-f]{8}$") + flipped = _flip(registered["token"]) + self.assertNotEqual(registered["token"], flipped) + status, body = self._secret("coreci/proxmox", self._app_header(flipped)) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + + def test_ac6_cross_path_read_is_403_with_denied_audit(self): + registered = self._register(["coreci"]) + status, body = self._secret("coreci/gitea", self._app_header(registered["token"])) + self.assertEqual(403, status) + self.assertEqual(GITEA_403_BODY, body) + denied = [ + entry for entry in self._audit("read") if entry["result"] == "denied" + ] + self.assertEqual(1, len(denied)) + self._assert_tool_identity(denied[0]) + self.assertEqual("read", denied[0]["action"]) + self.assertEqual(GITEA_PATH, denied[0]["path"]) + + def test_client_bootstrap_public_fetch_import_register_state_secret(self): + source = self._fetch_client_source() + self.assertIn("class BrokerClient", source) + self._assert_standalone_imports(source) + client = self._bootstrapped_client() + state = {} + client.ensure_registered(state) + self.assertEqual(44, len(state["current_token"])) + self.assertEqual("proxmox", state["agent_id"]) + self.assertEqual(self.app_uuid, state["app_registration_uuid"]) + self.assertEqual(self.user_id, state["user_id"]) + self.assertIn("pepper", state) + self.assertTrue(client.put_state(state)) + document = client.read_secret(PROXMOX_PATH, state) + self.assertEqual(PROXMOX_PATH, document["path"]) + self.assertEqual(PROXMOX_SECRET, document["data"]) + persisted = self._persisted_state() + self.assertEqual(state["current_token"], persisted["current_token"]) + + def test_reference_tool_full_flow_single_register_line(self): + tools = self._reference_tools() + before = len(self._audit("register")) + response = tools.list_nodes() + self.assertIs(True, response["dry_run"]) + self.assertIs(True, response["credential_present"]) + registers = self._audit("register") + self.assertEqual(1, len(registers) - before) + self.assertEqual("ok", registers[-1]["result"]) + self._assert_tool_identity(registers[-1]) + persisted = self._persisted_state() + self.assertEqual(tools._state["current_token"], persisted["current_token"]) + + def test_phase_audit_completeness_after_full_drive(self): + registered = self._register(["coreci"]) + token = registered["token"] + status, _ = self._secret("coreci/proxmox", self._app_header(token)) + self.assertEqual(200, status) + status, _ = self._secret("coreci/proxmox", self._app_header(_flip(token))) + self.assertEqual(401, status) + status, _ = self._secret("coreci/gitea", self._app_header(token)) + self.assertEqual(403, status) + client = self._bootstrapped_client() + state = {} + client.ensure_registered(state) + self.assertTrue(client.put_state(state)) + self.assertEqual(PROXMOX_SECRET, client.read_secret(PROXMOX_PATH, state)["data"]) + before = len(self._audit("register")) + tools = self._reference_tools() + response = tools.list_nodes() + self.assertIs(True, response["dry_run"]) + self.assertIs(True, response["credential_present"]) + registers = self._audit("register") + self.assertEqual(1, len(registers) - before) + self.assertEqual(3, len(registers)) + for entry in registers: + self.assertEqual("ok", entry["result"]) + self.assertEqual(REGISTER_PATH, entry["path"]) + self._assert_tool_identity(entry) + reads = self._audit("read") + ok_reads = [entry for entry in reads if entry["result"] == "ok"] + denied_reads = [entry for entry in reads if entry["result"] == "denied"] + self.assertEqual(3, len(ok_reads)) + for entry in ok_reads: + self.assertEqual(PROXMOX_PATH, entry["path"]) + self._assert_tool_identity(entry) + self.assertEqual(1, len(denied_reads)) + self.assertEqual(GITEA_PATH, denied_reads[0]["path"]) + self._assert_tool_identity(denied_reads[0]) + denied_actions = self._audit("denied") + self.assertEqual(1, len(denied_actions)) + self.assertEqual("denied", denied_actions[0]["action"]) + self.assertEqual("denied", denied_actions[0]["result"]) + self.assertEqual(SECRETS_PREFIX + "coreci/proxmox", denied_actions[0]["path"]) + self.assertEqual(self.app_uuid, denied_actions[0]["app_registration_uuid"]) + self.assertIsNone(denied_actions[0]["user_id"]) + self.assertIsNone(denied_actions[0]["agent_id"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_admin_auth.py b/tests/test_admin_auth.py new file mode 100644 index 0000000..fce08bb --- /dev/null +++ b/tests/test_admin_auth.py @@ -0,0 +1,251 @@ +import base64 +import contextlib +import datetime +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +LOGIN_PATH = "/api/broker/auth/login" +LOGOUT_PATH = "/api/broker/auth/logout" +WHOAMI_PATH = "/api/broker/auth/whoami" + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +def _iso_offset(seconds): + now = datetime.datetime.now(datetime.timezone.utc) + return (now + datetime.timedelta(seconds=seconds)).isoformat() + + +class AdminAuthTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + self.broker = self._boot_broker(self.env) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + + def _boot_broker(self, env): + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(dict(env)) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + return broker + + def _request(self, method, path, payload=None, headers=None, base=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + (base or self.base) + path, data=data, method=method + ) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, response.read(), response.headers + except urllib.error.HTTPError as error: + return error.code, error.read(), error.headers + + def _request_json(self, method, path, payload=None, headers=None, base=None): + status, body, response_headers = self._request( + method, path, payload, headers, base + ) + return status, json.loads(body.decode("utf-8")), response_headers + + def _login(self, username=ADMIN_USER, password=ADMIN_PASS, base=None): + return self._request_json( + "POST", + LOGIN_PATH, + {"username": username, "password": password}, + base=base, + ) + + def _login_cookie(self, base=None): + status, body, headers = self._login(base=base) + self.assertEqual(200, status) + self.assertEqual({"ok": True, "username": ADMIN_USER}, body) + raw = headers.get("Set-Cookie") + self.assertTrue(raw) + return raw + + def _cookie_id(self, raw): + return raw.split(";", 1)[0].split("=", 1)[1] + + def _audit_entries(self): + with open(self.env["BROKER_AUDIT_LOG_PATH"], "r", encoding="utf-8") as handle: + return [json.loads(raw) for raw in handle if raw.strip()] + + def _audit_find(self, action, result): + for entry in self._audit_entries(): + if entry["action"] == action and entry["result"] == result: + return entry + return None + + def _assert_null_identity(self, entry): + self.assertIsNotNone(entry) + self.assertIsNone(entry["user_id"]) + self.assertIsNone(entry["app_registration_uuid"]) + self.assertIsNone(entry["agent_id"]) + + def test_login_logout_whoami_flow_and_cookie_contract(self): + raw = self._login_cookie() + self.assertTrue(raw.startswith("broker_session=")) + self.assertIn("Path=/", raw) + self.assertIn("HttpOnly", raw) + self.assertIn("SameSite=Lax", raw) + cookie = {"Cookie": "broker_session=" + self._cookie_id(raw)} + status, body, _ = self._request_json("GET", WHOAMI_PATH, headers=cookie) + self.assertEqual(200, status) + self.assertEqual(ADMIN_USER, body["username"]) + created = datetime.datetime.fromisoformat(body["created_at"]) + expires = datetime.datetime.fromisoformat(body["expires_at"]) + self.assertEqual(datetime.timedelta(seconds=300), expires - created) + status, body, _ = self._request_json("POST", LOGOUT_PATH, headers=cookie) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + status, body, _ = self._request_json("GET", WHOAMI_PATH, headers=cookie) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + self.assertIsNone(self.broker.store.get_admin_session(self._cookie_id(raw))) + self._assert_null_identity(self._audit_find("login", "ok")) + self._assert_null_identity(self._audit_find("logout", "ok")) + + def test_wrong_password_is_401_with_denied_audit(self): + status, body, headers = self._login(password="not-the-pass") + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + self.assertTrue(body["message"]) + self.assertIsNone(headers.get("Set-Cookie")) + self._assert_null_identity(self._audit_find("login", "denied")) + + def test_wrong_username_is_401(self): + status, body, headers = self._login(username="mallory") + self.assertEqual(401, status) + self.assertIsNone(headers.get("Set-Cookie")) + self.assertIsNotNone(self._audit_find("login", "denied")) + + def test_login_with_missing_credentials_is_401(self): + status, body, _ = self._request_json("POST", LOGIN_PATH, payload={}) + self.assertEqual(401, status) + status, body, _ = self._request_json("POST", LOGIN_PATH) + self.assertEqual(401, status) + self.assertIsNotNone(self._audit_find("login", "denied")) + + def test_expired_session_is_rejected_lazily(self): + raw = self._login_cookie() + session_id = self._cookie_id(raw) + self.broker.store.conn.execute( + "UPDATE admin_sessions SET expires_at = ? WHERE session_id = ?", + (_iso_offset(-60), session_id), + ) + self.broker.store.conn.commit() + status, body, _ = self._request_json( + "GET", WHOAMI_PATH, headers={"Cookie": "broker_session=" + session_id} + ) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + + def test_boot_cleanup_deletes_stale_sessions_keeps_live(self): + self.broker.store.conn.executemany( + "INSERT INTO admin_sessions VALUES (?, ?, ?)", + [ + ("stale-session", _iso_offset(-600), _iso_offset(-60)), + ("live-session", _iso_offset(-600), _iso_offset(300)), + ], + ) + self.broker.store.conn.commit() + rebooted = self._boot_broker(self.env) + self.assertIsNone(rebooted.store.get_admin_session("stale-session")) + self.assertIsNotNone(rebooted.store.get_admin_session("live-session")) + + def test_basic_valid_on_whoami(self): + headers = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + status, body, _ = self._request_json("GET", WHOAMI_PATH, headers=headers) + self.assertEqual(200, status) + self.assertEqual(ADMIN_USER, body["username"]) + self.assertIsNone(body["created_at"]) + self.assertIsNone(body["expires_at"]) + + def test_basic_invalid_is_401(self): + headers = {"Authorization": _basic_header(ADMIN_USER, "wrong-pass")} + status, body, _ = self._request_json("GET", WHOAMI_PATH, headers=headers) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + self.assertTrue(body["message"]) + + def test_whoami_without_credentials_is_401(self): + status, body, _ = self._request_json("GET", WHOAMI_PATH) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + + def test_valid_session_cookie_wins_over_invalid_basic(self): + raw = self._login_cookie() + headers = { + "Cookie": "broker_session=" + self._cookie_id(raw), + "Authorization": _basic_header(ADMIN_USER, "wrong-pass"), + } + status, body, _ = self._request_json("GET", WHOAMI_PATH, headers=headers) + self.assertEqual(200, status) + self.assertEqual(ADMIN_USER, body["username"]) + + def test_logout_without_credentials_is_401(self): + status, body, _ = self._request_json("POST", LOGOUT_PATH) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + self.assertIsNone(self._audit_find("logout", "ok")) + + def test_logout_via_basic_without_session_is_ok(self): + headers = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + status, body, _ = self._request_json("POST", LOGOUT_PATH, headers=headers) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + self._assert_null_identity(self._audit_find("logout", "ok")) + + def test_session_ttl_is_configurable(self): + env = dict(self.env) + env["BROKER_DB_PATH"] = os.path.join(self.tmp, "ttl.db") + env["BROKER_AUDIT_LOG_PATH"] = os.path.join(self.tmp, "ttl.audit.log") + env["BROKER_SESSION_TTL"] = "60" + broker = self._boot_broker(env) + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(server.shutdown) + self.addCleanup(server.server_close) + base = "http://127.0.0.1:%d" % server.server_address[1] + raw = self._login_cookie(base=base) + status, body, _ = self._request_json( + "GET", + WHOAMI_PATH, + headers={"Cookie": "broker_session=" + self._cookie_id(raw)}, + base=base, + ) + self.assertEqual(200, status) + created = datetime.datetime.fromisoformat(body["created_at"]) + expires = datetime.datetime.fromisoformat(body["expires_at"]) + self.assertEqual(datetime.timedelta(seconds=60), expires - created) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_agent_meta.py b/tests/test_agent_meta.py new file mode 100644 index 0000000..d35a553 --- /dev/null +++ b/tests/test_agent_meta.py @@ -0,0 +1,134 @@ +import base64 +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +META_KEYS = {"agent_id", "app_registration_uuid", "user_id", "tags", + "token_expires_at", "first_registered_at", "last_seen_at"} +FORBIDDEN_RAW = ('"token"', '"token_hash"', '"pepper"') + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class AgentMetaTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed(self.app_uuid, self.user_id) + self.registered = self._register("agent-1", ["b", "a"]) + self.token = self.registered["token"] + self.pepper = self.registered["pepper"] + self.app = {"Authorization": "App %s %s" % (self.app_uuid, self.token)} + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw), raw + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8") + return error.code, json.loads(raw), raw + + def _seed(self, app_uuid, user_id): + status, _, _ = self._request("POST", APPS_PATH, {"uuid": app_uuid}, self.admin) + self.assertEqual(201, status) + status, _, _ = self._request("POST", USERS_PATH, {"user_id": user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self, agent_id, tags): + payload = {"agent_id": agent_id, "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": tags} + status, body, _ = self._request("POST", REGISTER_PATH, payload) + self.assertEqual(201, status) + return body + + def _meta(self, agent_id, headers=None): + return self._request( + "GET", "/api/broker/agents/" + agent_id, headers=headers if headers is not None else self.app + ) + + def test_identity_match_returns_metadata_only(self): + status, body, raw = self._meta("agent-1") + self.assertEqual(200, status) + self.assertEqual(META_KEYS, set(body)) + self.assertEqual("agent-1", body["agent_id"]) + self.assertEqual(self.app_uuid, body["app_registration_uuid"]) + self.assertEqual(self.user_id, body["user_id"]) + self.assertEqual(self.registered["token_expires_at"], body["token_expires_at"]) + row = self.broker.store.get_agent("agent-1", self.app_uuid) + self.assertEqual(row[6], body["first_registered_at"]) + self.assertEqual(row[7], body["last_seen_at"]) + for forbidden in FORBIDDEN_RAW: + self.assertNotIn(forbidden, raw) + self.assertNotIn(self.token, raw) + self.assertNotIn(self.pepper, raw) + + def test_tags_roundtrip_is_sorted_canonical_list(self): + status, body, _ = self._meta("agent-1") + self.assertEqual(200, status) + self.assertEqual(["a", "b"], body["tags"]) + + def test_url_agent_id_mismatch_is_403(self): + self._register("agent-2", ["coreci"]) + status, body, raw = self._meta("agent-2") + self.assertEqual(403, status) + self.assertEqual(403, body["error"]) + self.assertTrue(body["message"]) + for forbidden in FORBIDDEN_RAW: + self.assertNotIn(forbidden, raw) + + def test_unknown_agent_id_is_404(self): + status, body, _ = self._meta("ghost-agent") + self.assertEqual(404, status) + self.assertEqual({"error": 404, "message": "not found"}, body) + + def test_missing_app_header_is_401(self): + status, body, _ = self._meta("agent-1", headers={}) + self.assertEqual(401, status) + self.assertEqual({"error": 401, "message": "invalid app credentials"}, body) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py new file mode 100644 index 0000000..fb568b4 --- /dev/null +++ b/tests/test_allowlist.py @@ -0,0 +1,177 @@ +import base64 +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +BASE_PATH = "/api/broker/user_allowlist" + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class UserAllowlistTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _add(self, user_id, email=None, headers=None): + payload = {"user_id": user_id} + if email is not None: + payload["email"] = email + return self._request("POST", BASE_PATH, payload, headers or self.admin) + + def _row(self, user_id): + status, body = self._request("GET", BASE_PATH, headers=self.admin) + self.assertEqual(200, status) + for row in body["allowlist"]: + if row["user_id"] == user_id: + return row + return None + + def _audit(self): + with open(self.env["BROKER_AUDIT_LOG_PATH"], "r", encoding="utf-8") as handle: + return [json.loads(raw) for raw in handle if raw.strip()] + + def test_list_empty_then_add_then_list(self): + status, body = self._request("GET", BASE_PATH, headers=self.admin) + self.assertEqual(200, status) + self.assertEqual({"allowlist": []}, body) + status, body = self._add("alice", "alice@coreci.dev") + self.assertEqual(201, status) + self.assertEqual({"user_id": "alice", "status": "active"}, body) + row = self._row("alice") + self.assertEqual("alice@coreci.dev", row["email"]) + self.assertEqual("active", row["status"]) + self.assertTrue(row["created_at"]) + self.assertIsNone(row["revoked_at"]) + + def test_add_without_email_stores_null_email(self): + status, _ = self._add("bob") + self.assertEqual(201, status) + self.assertIsNone(self._row("bob")["email"]) + status, _ = self._request("POST", BASE_PATH, {"user_id": "bob2", "email": 42}, self.admin) + self.assertEqual(201, status) + self.assertIsNone(self._row("bob2")["email"]) + + def test_add_missing_or_empty_user_id_is_400(self): + for payload in ({"email": "x@coreci.dev"}, {}, {"user_id": ""}, + {"user_id": None}, {"user_id": 12345}, + {"user_id": ["alice"]}, [1, 2]): + status, body = self._request("POST", BASE_PATH, payload, self.admin) + self.assertEqual(400, status, repr(payload)) + self.assertEqual(400, body["error"]) + + def test_add_duplicate_active_or_revoked_is_409(self): + self._add("carol", "carol@coreci.dev") + status, body = self._add("carol") + self.assertEqual(409, status) + self.assertEqual(409, body["error"]) + self.assertTrue(body["message"]) + status, _ = self._request("DELETE", BASE_PATH + "/carol", headers=self.admin) + self.assertEqual(200, status) + status, body = self._add("carol") + self.assertEqual(409, status) + + def test_revoke_revoke_again_reinstate_lifecycle(self): + self._add("dave", "dave@coreci.dev") + status, body = self._request("DELETE", BASE_PATH + "/dave", headers=self.admin) + self.assertEqual(200, status) + self.assertEqual("revoked", body["status"]) + revoked_at = self._row("dave")["revoked_at"] + self.assertIsNotNone(revoked_at) + self.assertEqual("revoked", self._row("dave")["status"]) + status, body = self._request("DELETE", BASE_PATH + "/dave", headers=self.admin) + self.assertEqual(200, status) + self.assertEqual(revoked_at, self._row("dave")["revoked_at"]) + status, body = self._request( + "POST", BASE_PATH + "/dave/reinstate", headers=self.admin + ) + self.assertEqual(200, status) + self.assertEqual("active", body["status"]) + row = self._row("dave") + self.assertEqual("active", row["status"]) + self.assertIsNone(row["revoked_at"]) + + def test_revoke_missing_is_404(self): + status, body = self._request("DELETE", BASE_PATH + "/ghost", headers=self.admin) + self.assertEqual(404, status) + self.assertEqual(404, body["error"]) + + def test_reinstate_missing_is_404(self): + status, body = self._request( + "POST", BASE_PATH + "/ghost/reinstate", headers=self.admin + ) + self.assertEqual(404, status) + self.assertEqual(404, body["error"]) + + def test_admin_actions_audited_with_null_identity(self): + self._add("erin", "erin@coreci.dev") + self._request("DELETE", BASE_PATH + "/erin", headers=self.admin) + self._request("POST", BASE_PATH + "/erin/reinstate", headers=self.admin) + entries = self._audit() + self.assertEqual(["add_user", "remove_user", "add_user"], + [entry["action"] for entry in entries]) + for entry in entries: + self.assertEqual("ok", entry["result"]) + self.assertIsNone(entry["user_id"]) + self.assertIsNone(entry["app_registration_uuid"]) + self.assertIsNone(entry["agent_id"]) + self.assertEqual("/api/broker/user_allowlist/erin", entry["path"]) + + def test_unauthenticated_requests_are_401(self): + checks = ( + ("GET", BASE_PATH, None), + ("POST", BASE_PATH, {"user_id": "zoe"}), + ("DELETE", BASE_PATH + "/zoe", None), + ("POST", BASE_PATH + "/zoe/reinstate", None), + ) + for method, path, payload in checks: + status, body = self._request(method, path, payload) + self.assertEqual(401, status, method + " " + path) + self.assertEqual(401, body["error"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_app_auth.py b/tests/test_app_auth.py new file mode 100644 index 0000000..a2c8f0f --- /dev/null +++ b/tests/test_app_auth.py @@ -0,0 +1,263 @@ +import base64 +import contextlib +import datetime +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +PROTECTED_PATH = "/test/app-auth" +AUTH_401_BODY = {"error": 401, "message": "invalid app credentials"} + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +def _protected_echo(request): + request.send_json(200, dict(request.app_auth)) + + +class AppAuthTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed(self.app_uuid, self.user_id) + self.broker.router.add( + "GET", PROTECTED_PATH, coreci_broker._require_app(_protected_echo) + ) + self.token = self._register("agent-1")["token"] + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _seed(self, app_uuid, user_id): + status, _ = self._request("POST", APPS_PATH, {"uuid": app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self, agent_id): + payload = { + "agent_id": agent_id, + "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, + "tags": ["coreci"], + } + status, body = self._request("POST", REGISTER_PATH, payload) + self.assertEqual(201, status) + return body + + def _auth(self, authorization): + headers = {} if authorization is None else {"Authorization": authorization} + return self._request("GET", PROTECTED_PATH, headers=headers) + + def _app_header(self, token=None, app_uuid=None): + return "App %s %s" % (app_uuid or self.app_uuid, token or self.token) + + def _revoke_app(self): + status, _ = self._request( + "DELETE", APPS_PATH + "/" + self.app_uuid, headers=self.admin + ) + self.assertEqual(200, status) + + def _patch_column(self, agent_id, column, value): + self.broker.store.conn.execute( + "UPDATE agent_registrations SET %s = ? WHERE agent_id = ?" % column, + (value, agent_id), + ) + self.broker.store.conn.commit() + + def _count_verify_calls(self): + counter = {"calls": 0} + original = coreci_broker.verify_token + + def counted(token, stored): + counter["calls"] += 1 + return original(token, stored) + + coreci_broker.verify_token = counted + self.addCleanup(setattr, coreci_broker, "verify_token", original) + return counter + + def _past(self): + return (datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(seconds=60)).isoformat() + + def _future(self): + return (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=3600)).isoformat() + + def test_valid_header_returns_context_and_updates_last_seen(self): + past = self._past() + self._patch_column("agent-1", "last_seen_at", past) + status, body = self._auth(self._app_header()) + self.assertEqual(200, status) + self.assertEqual("agent-1", body["agent_id"]) + self.assertEqual(self.user_id, body["user_id"]) + self.assertEqual(self.app_uuid, body["app_registration_uuid"]) + row = self.broker.store.get_agent("agent-1", self.app_uuid) + self.assertEqual(row[5], body["token_expires_at"]) + self.assertGreater(row[7], past) + + def test_401_matrix_uniform_contract(self): + cases = ( + (None), + (_basic_header(ADMIN_USER, ADMIN_PASS)), + ("App"), + ("App %s" % self.app_uuid), + ("App %s %s extra" % (self.app_uuid, self.token)), + (self._app_header(app_uuid=str(uuidlib.uuid4()))), + (self._app_header(token="A" * 44)), + ("App not-a-uuid %s" % self.token), + ("App %s %s" % (self.app_uuid.upper(), self.token)), + ) + for header in cases: + with self.subTest(header=header): + status, body = self._auth(header) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + + def test_cache_first_success_scans_then_identical_call_hits_cache(self): + counter = self._count_verify_calls() + status, body = self._auth(self._app_header()) + self.assertEqual(200, status) + self.assertEqual(1, counter["calls"]) + status, body = self._auth(self._app_header()) + self.assertEqual(200, status) + self.assertEqual("agent-1", body["agent_id"]) + self.assertEqual(1, counter["calls"]) + self.assertIn((self.app_uuid, self.token), self.broker.token_cache) + + def test_clear_token_cache_forces_reverify(self): + counter = self._count_verify_calls() + self._auth(self._app_header()) + self.assertEqual(1, counter["calls"]) + self.broker.clear_token_cache() + self.assertEqual({}, self.broker.token_cache) + status, _ = self._auth(self._app_header()) + self.assertEqual(200, status) + self.assertEqual(2, counter["calls"]) + + def test_cache_hit_revoked_app_still_200_until_ttl(self): + counter = self._count_verify_calls() + self._auth(self._app_header()) + self._revoke_app() + status, body = self._auth(self._app_header()) + self.assertEqual(200, status) + self.assertEqual("agent-1", body["agent_id"]) + self.assertEqual(1, counter["calls"]) + row = self.broker.store.get_row("app_registrations", "uuid", self.app_uuid) + self.assertEqual("revoked", row[2]) + + def test_revoked_app_first_auth_still_200(self): + self._revoke_app() + counter = self._count_verify_calls() + status, body = self._auth(self._app_header()) + self.assertEqual(200, status) + self.assertEqual("agent-1", body["agent_id"]) + self.assertEqual(1, counter["calls"]) + + def test_revoked_app_expired_row_is_401(self): + self._revoke_app() + self._patch_column("agent-1", "token_expires_at", self._past()) + status, body = self._auth(self._app_header()) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + + def test_cache_hit_expired_row_401_then_evicted_and_reverifies(self): + counter = self._count_verify_calls() + self._auth(self._app_header()) + self.assertEqual(1, counter["calls"]) + self._patch_column("agent-1", "token_expires_at", self._past()) + status, body = self._auth(self._app_header()) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + self.assertEqual(1, counter["calls"]) + self.assertEqual({}, self.broker.token_cache) + self._patch_column("agent-1", "token_expires_at", self._future()) + status, body = self._auth(self._app_header()) + self.assertEqual(200, status) + self.assertEqual(2, counter["calls"]) + self.assertIn((self.app_uuid, self.token), self.broker.token_cache) + + def test_cache_hit_row_absent_is_401(self): + self._auth(self._app_header()) + self.broker.store.conn.execute( + "DELETE FROM agent_registrations WHERE agent_id = ?", ("agent-1",) + ) + self.broker.store.conn.commit() + status, body = self._auth(self._app_header()) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + self.assertEqual({}, self.broker.token_cache) + + def test_scrypt_hash_format_authenticates(self): + token2 = self._register("agent-2")["token"] + original_bcrypt = coreci_broker.bcrypt + coreci_broker.bcrypt = None + try: + scrypt_hash = coreci_broker.hash_token(token2) + finally: + coreci_broker.bcrypt = original_bcrypt + self.assertTrue(scrypt_hash.startswith("scrypt$")) + self._patch_column("agent-2", "token_hash", scrypt_hash) + status, body = self._auth(self._app_header(token=token2)) + self.assertEqual(200, status) + self.assertEqual("agent-2", body["agent_id"]) + + def test_bcrypt_hash_format_authenticates(self): + if coreci_broker.bcrypt is None: + self.skipTest("bcrypt not importable in this environment") + token3 = self._register("agent-3")["token"] + stored = self.broker.store.get_agent("agent-3", self.app_uuid)[4] + self.assertTrue(stored.startswith("$2")) + status, body = self._auth(self._app_header(token=token3)) + self.assertEqual(200, status) + self.assertEqual("agent-3", body["agent_id"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_apps.py b/tests/test_apps.py new file mode 100644 index 0000000..e886661 --- /dev/null +++ b/tests/test_apps.py @@ -0,0 +1,187 @@ +import base64 +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +BASE_PATH = "/api/broker/app_registrations" + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class AppRegistrationsTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _add(self, app_uuid, name=None, headers=None): + payload = {"uuid": app_uuid} + if name is not None: + payload["name"] = name + return self._request("POST", BASE_PATH, payload, headers or self.admin) + + def _row(self, app_uuid): + status, body = self._request("GET", BASE_PATH, headers=self.admin) + self.assertEqual(200, status) + for row in body["registrations"]: + if row["uuid"] == app_uuid: + return row + return None + + def _audit(self): + with open(self.env["BROKER_AUDIT_LOG_PATH"], "r", encoding="utf-8") as handle: + return [json.loads(raw) for raw in handle if raw.strip()] + + def test_list_empty_then_add_then_list(self): + status, body = self._request("GET", BASE_PATH, headers=self.admin) + self.assertEqual(200, status) + self.assertEqual({"registrations": []}, body) + app_uuid = str(uuidlib.uuid4()) + status, body = self._add(app_uuid, "proxmox-tool") + self.assertEqual(201, status) + self.assertEqual({"uuid": app_uuid, "status": "active"}, body) + row = self._row(app_uuid) + self.assertEqual("proxmox-tool", row["name"]) + self.assertEqual("active", row["status"]) + self.assertTrue(row["created_at"]) + self.assertIsNone(row["revoked_at"]) + + def test_add_without_name_stores_null_name(self): + app_uuid = str(uuidlib.uuid4()) + status, _ = self._add(app_uuid) + self.assertEqual(201, status) + self.assertIsNone(self._row(app_uuid)["name"]) + + def test_add_duplicate_active_or_revoked_is_409(self): + app_uuid = str(uuidlib.uuid4()) + self._add(app_uuid) + status, body = self._add(app_uuid) + self.assertEqual(409, status) + self.assertEqual(409, body["error"]) + self.assertTrue(body["message"]) + status, _ = self._request("DELETE", BASE_PATH + "/" + app_uuid, headers=self.admin) + self.assertEqual(200, status) + status, body = self._add(app_uuid) + self.assertEqual(409, status) + + def test_add_invalid_or_noncanonical_uuid_is_400(self): + canonical = str(uuidlib.uuid4()) + for bad in ("not-a-uuid", "", "{" + canonical.upper() + "}", canonical.upper(), + canonical.replace("-", ""), "urn:uuid:" + canonical): + status, body = self._add(bad) + self.assertEqual(400, status, bad) + self.assertEqual(400, body["error"]) + status, _ = self._request("POST", BASE_PATH, {}, self.admin) + self.assertEqual(400, status) + status, _ = self._request("POST", BASE_PATH, {"uuid": 12345}, self.admin) + self.assertEqual(400, status) + status, _ = self._request("POST", BASE_PATH, {"name": "no-uuid"}, self.admin) + self.assertEqual(400, status) + + def test_revoke_revoke_again_reinstate_lifecycle(self): + app_uuid = str(uuidlib.uuid4()) + self._add(app_uuid) + status, body = self._request("DELETE", BASE_PATH + "/" + app_uuid, headers=self.admin) + self.assertEqual(200, status) + self.assertEqual("revoked", body["status"]) + revoked_at = self._row(app_uuid)["revoked_at"] + self.assertIsNotNone(revoked_at) + self.assertEqual("revoked", self._row(app_uuid)["status"]) + status, body = self._request("DELETE", BASE_PATH + "/" + app_uuid, headers=self.admin) + self.assertEqual(200, status) + self.assertEqual(revoked_at, self._row(app_uuid)["revoked_at"]) + status, body = self._request( + "POST", BASE_PATH + "/" + app_uuid + "/reinstate", headers=self.admin + ) + self.assertEqual(200, status) + self.assertEqual("active", body["status"]) + row = self._row(app_uuid) + self.assertEqual("active", row["status"]) + self.assertIsNone(row["revoked_at"]) + + def test_revoke_missing_is_404(self): + status, body = self._request( + "DELETE", BASE_PATH + "/" + str(uuidlib.uuid4()), headers=self.admin + ) + self.assertEqual(404, status) + self.assertEqual(404, body["error"]) + + def test_reinstate_missing_is_404(self): + status, body = self._request( + "POST", BASE_PATH + "/" + str(uuidlib.uuid4()) + "/reinstate", headers=self.admin + ) + self.assertEqual(404, status) + self.assertEqual(404, body["error"]) + + def test_admin_actions_audited_with_null_identity(self): + app_uuid = str(uuidlib.uuid4()) + self._add(app_uuid) + self._request("DELETE", BASE_PATH + "/" + app_uuid, headers=self.admin) + self._request("POST", BASE_PATH + "/" + app_uuid + "/reinstate", headers=self.admin) + entries = self._audit() + self.assertEqual(["add_app", "remove_app", "add_app"], [e["action"] for e in entries]) + for entry in entries: + self.assertEqual("ok", entry["result"]) + self.assertIsNone(entry["user_id"]) + self.assertIsNone(entry["app_registration_uuid"]) + self.assertIsNone(entry["agent_id"]) + self.assertEqual("/api/broker/app_registrations/" + app_uuid, entry["path"]) + + def test_unauthenticated_requests_are_401(self): + checks = ( + ("GET", BASE_PATH, None), + ("POST", BASE_PATH, {"uuid": str(uuidlib.uuid4())}), + ("DELETE", BASE_PATH + "/" + str(uuidlib.uuid4()), None), + ("POST", BASE_PATH + "/" + str(uuidlib.uuid4()) + "/reinstate", None), + ) + for method, path, payload in checks: + status, body = self._request(method, path, payload) + self.assertEqual(401, status, method + " " + path) + self.assertEqual(401, body["error"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..ef73b19 --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,213 @@ +import contextlib +import datetime +import io +import json +import os +import sys +import tempfile +import threading +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +AUDIT_FIELDS = [ + "ts", + "user_id", + "app_registration_uuid", + "agent_id", + "action", + "path", + "result", +] + +ALL_ACTIONS = [ + "register", + "read", + "denied", + "login", + "logout", + "add_app", + "remove_app", + "add_user", + "remove_user", +] + +ADMIN_ACTIONS = [ + "login", + "logout", + "add_app", + "remove_app", + "add_user", + "remove_user", +] + + +class AuditTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.env_path = os.path.join(tmp.name, ".env") + self.db_path = os.path.join(tmp.name, "broker.db") + self.audit_path = os.path.join(tmp.name, "broker.audit.log") + + def _boot(self, extra_env=None): + env = { + "BROKER_ENV_PATH": self.env_path, + "BROKER_DB_PATH": self.db_path, + "BROKER_AUDIT_LOG_PATH": self.audit_path, + } + if extra_env: + env.update(extra_env) + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + broker = coreci_broker.boot(env) + self.addCleanup(broker.store.close) + self.addCleanup(broker.audit.close) + return broker + + def _raw_lines(self): + with open(self.audit_path, "r", encoding="utf-8") as handle: + return handle.read().splitlines() + + def _entries(self): + return [json.loads(raw) for raw in self._raw_lines()] + + def test_boot_wires_ready_audit_at_mode_0600(self): + broker = self._boot() + self.assertIsInstance(broker.audit, coreci_broker.Audit) + self.assertEqual(self.audit_path, broker.audit.path) + self.assertTrue(os.path.isfile(self.audit_path)) + self.assertEqual(0o600, os.stat(self.audit_path).st_mode & 0o777) + + def test_audit_mode_0600_independent_of_umask(self): + old_umask = os.umask(0o777) + try: + broker = self._boot() + finally: + os.umask(old_umask) + broker.audit.write("login", path="/ui/login") + self.assertEqual(0o600, os.stat(self.audit_path).st_mode & 0o777) + + def test_each_write_appends_exactly_one_valid_json_line(self): + broker = self._boot() + self.assertEqual(0, len(self._raw_lines())) + broker.audit.write("login", path="/api/broker/auth/login") + broker.audit.write( + "read", + path="coreci/proxmox", + user_id="staff-7", + app_registration_uuid="0b6c3370-6d38-4b92-b9a2-bc2f1c9c1b0a", + agent_id="proxmox", + ) + broker.audit.write("denied", path="coreci/gitea", result="denied") + lines = self._raw_lines() + self.assertEqual(3, len(lines)) + for line in lines: + self.assertTrue(line) + entries = self._entries() + for entry in entries: + self.assertEqual(AUDIT_FIELDS, list(entry)) + self.assertEqual(["login", "read", "denied"], [e["action"] for e in entries]) + + def test_all_nine_action_verbs_accepted_and_written(self): + broker = self._boot() + for action in ALL_ACTIONS: + broker.audit.write(action, path="/x") + entries = self._entries() + self.assertEqual(9, len(entries)) + self.assertEqual(sorted(ALL_ACTIONS), sorted(e["action"] for e in entries)) + + def test_unknown_action_verb_raises_and_appends_nothing(self): + broker = self._boot() + for bad in ("bogus", "", "LOGIN", None, "rotate"): + with self.assertRaises(ValueError): + broker.audit.write(bad, path="/x") + with self.assertRaises(ValueError): + broker.audit.write("read", path="/x", result="bogus") + self.assertEqual(0, len(self._raw_lines())) + + def test_admin_actions_carry_null_identity_fields(self): + broker = self._boot() + for action in ADMIN_ACTIONS: + broker.audit.write(action, path="/api/broker/app_registrations") + entries = self._entries() + self.assertEqual(len(ADMIN_ACTIONS), len(entries)) + for entry in entries: + self.assertIsNone(entry["user_id"]) + self.assertIsNone(entry["app_registration_uuid"]) + self.assertIsNone(entry["agent_id"]) + broker.audit.write( + "register", + path="/api/broker/agents/register", + user_id="staff-7", + app_registration_uuid="0b6c3370-6d38-4b92-b9a2-bc2f1c9c1b0a", + agent_id="proxmox", + ) + entry = self._entries()[-1] + self.assertEqual("staff-7", entry["user_id"]) + self.assertEqual("0b6c3370-6d38-4b92-b9a2-bc2f1c9c1b0a", entry["app_registration_uuid"]) + self.assertEqual("proxmox", entry["agent_id"]) + + def test_append_only_preserves_every_line_across_many_writes(self): + broker = self._boot() + for i in range(50): + broker.audit.write("read", path=f"coreci/agent-{i}") + entries = self._entries() + self.assertEqual(50, len(entries)) + self.assertEqual("coreci/agent-0", entries[0]["path"]) + self.assertEqual("coreci/agent-49", entries[-1]["path"]) + self.assertEqual( + [f"coreci/agent-{i}" for i in range(50)], + [e["path"] for e in entries], + ) + + def test_reboot_appends_without_truncating_history(self): + first = self._boot() + first.audit.write("login", path="/ui/login") + first.store.close() + first.audit.close() + second = self._boot() + second.audit.write("logout", path="/ui/logout") + entries = self._entries() + self.assertEqual(["login", "logout"], [e["action"] for e in entries]) + self.assertEqual(0o600, os.stat(self.audit_path).st_mode & 0o777) + + def test_ts_parses_as_iso8601_utc(self): + broker = self._boot() + broker.audit.write("read", path="coreci/proxmox", user_id="staff-7") + for entry in self._entries(): + parsed = datetime.datetime.fromisoformat(entry["ts"]) + self.assertEqual(datetime.timedelta(0), parsed.utcoffset()) + + def test_result_values_written(self): + broker = self._boot() + broker.audit.write("read", path="coreci/proxmox", result="ok") + broker.audit.write("read", path="coreci/gitea", result="denied") + broker.audit.write("read", path="coreci/boom", result="error") + self.assertEqual(["ok", "denied", "error"], [e["result"] for e in self._entries()]) + + def test_concurrent_writes_are_serialized(self): + broker = self._boot() + threads = [] + for t in range(4): + def worker(t=t): + for i in range(10): + broker.audit.write("read", path=f"coreci/t{t}-i{i}") + thread = threading.Thread(target=worker) + threads.append(thread) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + entries = self._entries() + self.assertEqual(40, len(entries)) + self.assertEqual( + sorted(f"coreci/t{t}-i{i}" for t in range(4) for i in range(10)), + sorted(e["path"] for e in entries), + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_audit_query.py b/tests/test_audit_query.py new file mode 100644 index 0000000..c11d853 --- /dev/null +++ b/tests/test_audit_query.py @@ -0,0 +1,305 @@ +import base64 +import contextlib +import io +import json +import os +import sys +import tempfile +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +AUDIT_PATH = "/api/broker/audit" +UUID_A = "0b6c3370-6d38-4b92-b9a2-bc2f1c9c1b0a" +UUID_B = "3f2d8a04-9c17-4b1e-8d2f-6a5b4c3d2e1f" + +SEED = [ + {"action": "read", "path": "coreci/proxmox", "user_id": "staff-1", + "app_registration_uuid": UUID_A, "agent_id": "proxmox"}, + {"action": "read", "path": "coreci/gitea", "user_id": "staff-2", + "app_registration_uuid": UUID_A, "agent_id": "gitea"}, + {"action": "denied", "path": "coreci/gitea", "result": "denied", + "user_id": "staff-2", "app_registration_uuid": UUID_B, "agent_id": "gitea"}, + {"action": "add_app", "path": "/api/broker/app_registrations"}, + {"action": "read", "path": "coreci/proxmox", "user_id": "staff-1", + "app_registration_uuid": UUID_B, "agent_id": "proxmox"}, + {"action": "login", "path": "/api/broker/auth/login"}, + {"action": "remove_user", "path": "/api/broker/user_allowlist/staff-2"}, +] + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class AuditQueryTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.audit_path = os.path.join(tmp.name, "broker.audit.log") + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot({ + "BROKER_ENV_PATH": os.path.join(tmp.name, ".env"), + "BROKER_DB_PATH": os.path.join(tmp.name, "broker.db"), + "BROKER_AUDIT_LOG_PATH": self.audit_path, + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + }) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.auth = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + + def _request_json(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _get(self, params=None, headers=None): + path = AUDIT_PATH + if params: + path += "?" + urllib.parse.urlencode(params) + return self._request_json( + "GET", path, headers=self.auth if headers is None else headers + ) + + def _seed(self, specs): + for spec in specs: + self.broker.audit.write(**spec) + time.sleep(0.005) + + def _ts(self): + values = [] + with open(self.audit_path, "r", encoding="utf-8") as handle: + for raw in handle: + try: + values.append(json.loads(raw)["ts"]) + except (ValueError, TypeError, KeyError): + continue + return values + + def _paths(self, body): + return [entry["path"] for entry in body["entries"]] + + def test_no_filters_returns_all_entries_with_null_cursor(self): + self._seed(SEED) + status, body = self._get() + self.assertEqual(200, status) + with open(self.audit_path, "r", encoding="utf-8") as handle: + expected = [json.loads(raw) for raw in handle] + self.assertEqual(expected, body["entries"]) + self.assertEqual(len(SEED), len(body["entries"])) + self.assertIsNone(body["next_since"]) + + def test_user_id_filter_exact_match(self): + self._seed(SEED) + status, body = self._get({"user_id": "staff-1"}) + self.assertEqual(200, status) + self.assertEqual(["coreci/proxmox", "coreci/proxmox"], self._paths(body)) + self.assertEqual([UUID_A, UUID_B], + [e["app_registration_uuid"] for e in body["entries"]]) + self.assertIsNone(body["next_since"]) + + def test_app_registration_uuid_filter_exact_match(self): + self._seed(SEED) + status, body = self._get({"app_registration_uuid": UUID_B}) + self.assertEqual(200, status) + self.assertEqual(["denied", "read"], [e["action"] for e in body["entries"]]) + self.assertEqual(["coreci/gitea", "coreci/proxmox"], self._paths(body)) + status, body = self._get({"app_registration_uuid": "no-such-uuid"}) + self.assertEqual(200, status) + self.assertEqual([], body["entries"]) + self.assertIsNone(body["next_since"]) + + def test_action_filter_exact_match(self): + self._seed(SEED) + status, body = self._get({"action": "denied"}) + self.assertEqual(200, status) + self.assertEqual(["coreci/gitea"], self._paths(body)) + status, body = self._get({"action": "read"}) + self.assertEqual(["coreci/proxmox", "coreci/gitea", "coreci/proxmox"], + self._paths(body)) + status, body = self._get({"action": "rea"}) + self.assertEqual(200, status) + self.assertEqual([], body["entries"]) + + def test_since_filter_is_exclusive(self): + self._seed(SEED) + ts = self._ts() + status, body = self._get({"since": ts[2]}) + self.assertEqual(200, status) + self.assertEqual(["add_app", "read", "login", "remove_user"], + [e["action"] for e in body["entries"]]) + self.assertEqual(4, len(body["entries"])) + self.assertIsNone(body["next_since"]) + status, body = self._get({"since": ts[-1]}) + self.assertEqual(200, status) + self.assertEqual([], body["entries"]) + self.assertIsNone(body["next_since"]) + + def test_until_filter_is_inclusive(self): + self._seed(SEED) + ts = self._ts() + status, body = self._get({"until": ts[2]}) + self.assertEqual(200, status) + self.assertEqual(["read", "read", "denied"], + [e["action"] for e in body["entries"]]) + self.assertEqual(3, len(body["entries"])) + self.assertIsNone(body["next_since"]) + + def test_since_and_until_range_together(self): + self._seed(SEED) + ts = self._ts() + status, body = self._get({"since": ts[1], "until": ts[4]}) + self.assertEqual(200, status) + self.assertEqual(["denied", "add_app", "read"], [e["action"] for e in body["entries"]]) + status, body = self._get({"since": ts[3], "until": ts[3]}) + self.assertEqual(200, status) + self.assertEqual([], body["entries"]) + + def test_combined_filters(self): + self._seed(SEED) + ts = self._ts() + status, body = self._get({"user_id": "staff-1", "action": "read"}) + self.assertEqual(200, status) + self.assertEqual(["coreci/proxmox", "coreci/proxmox"], self._paths(body)) + status, body = self._get({"user_id": "staff-2", "action": "read"}) + self.assertEqual(["coreci/gitea"], self._paths(body)) + status, body = self._get( + {"app_registration_uuid": UUID_B, "since": ts[2], "until": ts[6]} + ) + self.assertEqual(["coreci/proxmox"], self._paths(body)) + status, body = self._get( + {"user_id": "staff-1", "since": ts[0], "until": ts[4]} + ) + self.assertEqual(["coreci/proxmox"], self._paths(body)) + status, body = self._get( + {"user_id": "staff-1", "action": "denied", + "app_registration_uuid": UUID_A, "since": ts[0], "until": ts[6]} + ) + self.assertEqual([], body["entries"]) + + def test_limit_pagination_with_next_since_cursor(self): + specs = [{"action": "read", "path": "coreci/agent-%d" % i} for i in range(5)] + self._seed(specs) + ts = self._ts() + status, body = self._get({"limit": 2}) + self.assertEqual(200, status) + self.assertEqual(["coreci/agent-0", "coreci/agent-1"], self._paths(body)) + self.assertEqual(ts[1], body["next_since"]) + status, body = self._get({"since": body["next_since"], "limit": 2}) + self.assertEqual(200, status) + self.assertEqual(["coreci/agent-2", "coreci/agent-3"], self._paths(body)) + self.assertEqual(ts[3], body["next_since"]) + status, body = self._get({"since": ts[1]}) + self.assertEqual(200, status) + self.assertEqual(["coreci/agent-2", "coreci/agent-3", "coreci/agent-4"], + self._paths(body)) + self.assertIsNone(body["next_since"]) + status, body = self._get({"since": ts[4]}) + self.assertEqual(200, status) + self.assertEqual([], body["entries"]) + self.assertIsNone(body["next_since"]) + + def test_limit_zero_and_negative_clamp_to_one(self): + self._seed([{"action": "read", "path": "coreci/a"}, + {"action": "read", "path": "coreci/b"}, + {"action": "read", "path": "coreci/c"}]) + ts = self._ts() + for bad in (0, -3): + status, body = self._get({"limit": bad}) + self.assertEqual(200, status) + self.assertEqual(["coreci/a"], self._paths(body)) + self.assertEqual(ts[0], body["next_since"]) + + def test_limit_above_max_and_non_numeric_are_clamped(self): + self._seed([{"action": "read", "path": "coreci/a"}, + {"action": "read", "path": "coreci/b"}, + {"action": "read", "path": "coreci/c"}]) + for value in (5000, "abc"): + status, body = self._get({"limit": value}) + self.assertEqual(200, status) + self.assertEqual(3, len(body["entries"])) + self.assertIsNone(body["next_since"]) + + def test_default_limit_is_exactly_100(self): + specs = [{"action": "read", "path": "coreci/agent-%d" % i} for i in range(100)] + self._seed(specs) + ts = self._ts() + status, body = self._get() + self.assertEqual(200, status) + self.assertEqual(100, len(body["entries"])) + self.assertEqual("coreci/agent-99", body["entries"][-1]["path"]) + self.assertEqual(ts[99], body["next_since"]) + status, body = self._get({"since": ts[99]}) + self.assertEqual(200, status) + self.assertEqual([], body["entries"]) + self.assertIsNone(body["next_since"]) + + def test_invalid_since_or_until_is_400(self): + self._seed(SEED) + status, body = self._get({"since": "not-a-timestamp"}) + self.assertEqual(400, status) + self.assertEqual(400, body["error"]) + self.assertTrue(body["message"]) + status, body = self._get({"until": "2026-02-30T00:00:00Z"}) + self.assertEqual(400, status) + self.assertEqual(400, body["error"]) + self.assertTrue(body["message"]) + status, body = self._get({"since": ""}) + self.assertEqual(200, status) + self.assertEqual(len(SEED), len(body["entries"])) + + def test_malformed_lines_skipped(self): + self._seed([{"action": "read", "path": "coreci/a"}, + {"action": "read", "path": "coreci/b"}]) + with open(self.audit_path, "a", encoding="utf-8") as handle: + handle.write("{not json\n") + handle.write("[1, 2, 3]\n") + self._seed([{"action": "read", "path": "coreci/c"}]) + ts = self._ts() + self.assertEqual(3, len(ts)) + status, body = self._get() + self.assertEqual(200, status) + self.assertEqual(["coreci/a", "coreci/b", "coreci/c"], self._paths(body)) + self.assertIsNone(body["next_since"]) + status, body = self._get({"limit": 2}) + self.assertEqual(200, status) + self.assertEqual(["coreci/a", "coreci/b"], self._paths(body)) + self.assertEqual(ts[1], body["next_since"]) + status, body = self._get({"since": ts[1], "action": "read"}) + self.assertEqual(200, status) + self.assertEqual(["coreci/c"], self._paths(body)) + + def test_unauthenticated_is_401(self): + self._seed(SEED) + status, body = self._get(headers={}) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + status, body = self._get( + headers={"Authorization": _basic_header(ADMIN_USER, "wrong-pass")} + ) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_auth_audit.py b/tests/test_auth_audit.py new file mode 100644 index 0000000..7d96d6b --- /dev/null +++ b/tests/test_auth_audit.py @@ -0,0 +1,188 @@ +import base64 +import contextlib +import datetime +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.parse +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +AUDIT_PATH = "/api/broker/audit" +STATE_PATH = "/api/broker/agents/%s/state" +AUTH_401_BODY = {"error": 401, "message": "invalid app credentials"} +BAD_JSON_BODY = {"error": 400, "message": "invalid JSON body"} +NEST_DEPTH = 50000 + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class AuthAuditTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self.agent_id = "agent-1" + self._seed(self.app_uuid, self.user_id) + self.token = self._register()["token"] + self.app = {"Authorization": "App %s %s" % (self.app_uuid, self.token)} + + def _raw(self, method, path, data, headers): + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + return self._raw(method, path, data, headers) + + def _seed(self, app_uuid, user_id): + status, _ = self._request("POST", APPS_PATH, {"uuid": app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self): + payload = {"agent_id": self.agent_id, "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": ["coreci"]} + status, body = self._request("POST", REGISTER_PATH, payload) + self.assertEqual(201, status) + return body + + def _denied(self): + status, body = self._request("GET", AUDIT_PATH + "?action=denied", + headers=self.admin) + self.assertEqual(200, status) + return body["entries"] + + def _assert_denied_line(self, entry, app_uuid, path): + self.assertEqual("denied", entry["action"]) + self.assertEqual("denied", entry["result"]) + self.assertEqual(app_uuid, entry["app_registration_uuid"]) + self.assertEqual(path, entry["path"]) + self.assertIsNone(entry["user_id"]) + self.assertIsNone(entry["agent_id"]) + + def test_deep_nested_json_body_is_400_not_500(self): + data = ("[" * NEST_DEPTH).encode("utf-8") + self.assertLess(len(data), 1024 * 1024) + status, body = self._raw("PUT", STATE_PATH % self.agent_id, data, self.app) + self.assertEqual(400, status) + self.assertEqual(BAD_JSON_BODY, body) + status, body = self._request("GET", "/healthz") + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + + def test_wrong_token_denied_audit_carries_uuid_and_path(self): + path = STATE_PATH % self.agent_id + status, body = self._request( + "GET", path, headers={"Authorization": "App %s %s" % (self.app_uuid, "A" * 44)} + ) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + entries = self._denied() + self.assertEqual(1, len(entries)) + self._assert_denied_line(entries[0], self.app_uuid, path) + + def test_expired_or_absent_token_same_denied_audit(self): + path = STATE_PATH % self.agent_id + past = (datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(seconds=60)).isoformat() + self.broker.store.conn.execute( + "UPDATE agent_registrations SET token_expires_at = ? WHERE agent_id = ?", + (past, self.agent_id), + ) + self.broker.store.conn.commit() + status, body = self._request("GET", path, headers=self.app) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + status, body = self._request( + "GET", path, headers={"Authorization": "App %s" % self.app_uuid} + ) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + entries = self._denied() + self.assertEqual(2, len(entries)) + for entry in entries: + self._assert_denied_line(entry, self.app_uuid, path) + + def test_missing_app_header_401_writes_no_denied_audit(self): + self.assertEqual(0, len(self._denied())) + path = STATE_PATH % self.agent_id + status, body = self._request("GET", path, headers={}) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + status, body = self._request("GET", path, headers=self.admin) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + self.assertEqual(0, len(self._denied())) + + def test_malformed_uuid_denied_audit_carries_null_uuid(self): + path = STATE_PATH % self.agent_id + status, body = self._request( + "GET", path, headers={"Authorization": "App not-a-uuid %s" % self.token} + ) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + entries = self._denied() + self.assertEqual(1, len(entries)) + self._assert_denied_line(entries[0], None, path) + + def test_admin_audit_action_filter_returns_denied_lines(self): + path = STATE_PATH % self.agent_id + for header in ("App %s wrong" % self.app_uuid, + "App not-a-uuid %s" % self.token): + status, _ = self._request("GET", path, headers={"Authorization": header}) + self.assertEqual(401, status) + query = urllib.parse.urlencode({"action": "denied"}) + status, body = self._request("GET", AUDIT_PATH + "?" + query, + headers=self.admin) + self.assertEqual(200, status) + self.assertEqual(2, len(body["entries"])) + self.assertEqual([self.app_uuid, None], + [entry["app_registration_uuid"] for entry in body["entries"]]) + for entry in body["entries"]: + self.assertEqual(path, entry["path"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..5be0081 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,308 @@ +import ast +import base64 +import contextlib +import datetime +import io +import json +import os +import sys +import tempfile +import time +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker +import coreci_broker_client + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +PROXMOX_SECRET = {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"} +GITEA_SECRET = {"url": "https://git.coreci.dev", "token": "git-tok"} +CLIENT_SOURCE_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, "coreci_broker_client.py") +) +ALLOWED_CLIENT_IMPORTS = {"urllib.request", "urllib.parse", "json", "time"} +SECRETS_PROXMOX_PATH = "/api/broker/secrets/coreci/proxmox" + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class ClientHygieneTests(unittest.TestCase): + def setUp(self): + with open(CLIENT_SOURCE_PATH, "r", encoding="utf-8") as handle: + self.source = handle.read() + self.tree = ast.parse(self.source) + + def test_imports_are_stdlib_only(self): + modules = set() + for node in ast.walk(self.tree): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + modules.add(node.module or "") + self.assertIn("urllib.request", modules) + for module in modules: + self.assertIn(module, ALLOWED_CLIENT_IMPORTS, module) + + def test_single_module_docstring_and_no_comments(self): + docstring = ast.get_docstring(self.tree) + self.assertIsNotNone(docstring) + self.assertNotIn("\n", docstring) + for node in ast.walk(self.tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + self.assertIsNone(ast.get_docstring(node)) + for number, line in enumerate(self.source.splitlines(), 1): + self.assertFalse(line.lstrip().startswith("#"), "comment on line %d" % number) + + def test_broker_error_carries_status_and_message(self): + error = coreci_broker_client.BrokerError(403, "forbidden") + self.assertEqual(403, error.status) + self.assertEqual("forbidden", error.message) + self.assertEqual("forbidden", str(error)) + + +class _ClientCase(unittest.TestCase): + extra_env = {} + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.creds_path = os.path.join(self.tmp, "creds.json") + with open(self.creds_path, "w", encoding="utf-8") as handle: + handle.write(json.dumps( + {"coreci/proxmox": PROXMOX_SECRET, "coreci/gitea": GITEA_SECRET} + )) + env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + "BROKER_CREDS_SOURCE": "file", + "BROKER_CREDS_FILE": self.creds_path, + } + env.update(self.extra_env) + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(env) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed() + + def _raw_request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8") + return error.code, json.loads(raw) if raw else None + + def _seed(self): + status, _ = self._raw_request("POST", APPS_PATH, {"uuid": self.app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._raw_request("POST", USERS_PATH, {"user_id": self.user_id}, self.admin) + self.assertEqual(201, status) + + def _client(self, agent_id="proxmox", tags=("coreci",)): + return coreci_broker_client.BrokerClient( + self.base, agent_id, self.app_uuid, self.user_id, list(tags) + ) + + def _count_requests(self, client): + calls = [] + original = client._request + + def counting(method, path, payload=None, headers=None): + calls.append((method, path)) + return original(method, path, payload, headers) + + def restore(): + client._request = original + + client._request = counting + return calls, restore + + +class ClientFlowTests(_ClientCase): + def test_ensure_registered_from_empty_state_populates_token_fields(self): + client = self._client() + state = {} + returned = client.ensure_registered(state) + self.assertIs(state, returned) + token = state["current_token"] + self.assertEqual(44, len(token)) + self.assertTrue(token.isascii()) + expires = datetime.datetime.fromisoformat(state["token_expires_at"]) + self.assertGreater(expires, datetime.datetime.now(datetime.timezone.utc)) + self.assertEqual("proxmox", state["agent_id"]) + self.assertEqual(self.app_uuid, state["app_registration_uuid"]) + self.assertEqual(self.user_id, state["user_id"]) + self.assertEqual(["coreci"], state["tags"]) + status, body = self._raw_request( + "GET", SECRETS_PROXMOX_PATH, + headers={"Authorization": "App %s %s" % (self.app_uuid, token)}, + ) + self.assertEqual(200, status) + self.assertEqual(PROXMOX_SECRET, body["data"]) + + def test_first_register_merges_pepper_into_state(self): + client = self._client() + state = client.ensure_registered({}) + self.assertIn("pepper", state) + self.assertEqual( + self.broker.store.get_config("BROKER_INSTANCE_PEPPER"), state["pepper"] + ) + + def test_ensure_registered_with_live_token_makes_no_network_call(self): + client = self._client() + state = client.ensure_registered({}) + snapshot = dict(state) + calls, restore = self._count_requests(client) + try: + returned = client.ensure_registered(state) + finally: + restore() + self.assertIs(state, returned) + self.assertEqual([], calls) + self.assertEqual(snapshot, state) + + def test_reregister_with_valid_token_pepperless_response_does_not_crash(self): + client = self._client() + state = client.ensure_registered({}) + pepper, token, expires = state["pepper"], state["current_token"], state["token_expires_at"] + status, body = self._raw_request( + "POST", coreci_broker_client.REGISTER_PATH, + {"agent_id": "proxmox", "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": ["coreci"]}, + {"Authorization": "App %s %s" % (self.app_uuid, token)}, + ) + self.assertEqual(200, status) + self.assertNotIn("pepper", body) + returned = client.register(state) + self.assertIs(state, returned) + self.assertEqual(pepper, state["pepper"]) + self.assertEqual(token, state["current_token"]) + self.assertEqual(expires, state["token_expires_at"]) + + def test_put_then_get_state_roundtrip_through_broker(self): + client = self._client() + state = client.ensure_registered({}) + state["scratchpad"] = {"calls": 3, "notes": ["x"]} + self.assertTrue(client.put_state(state)) + document = client.get_state(state) + self.assertEqual(state, document) + + def test_get_state_returns_none_before_first_put(self): + client = self._client() + state = client.ensure_registered({}) + self.assertIsNone(client.get_state(state)) + + def test_read_secret_returns_secret_document(self): + client = self._client() + state = client.ensure_registered({}) + document = client.read_secret("coreci/proxmox", state) + self.assertEqual("coreci/proxmox", document["path"]) + self.assertEqual(PROXMOX_SECRET, document["data"]) + self.assertRegex(document["version"], r"^[0-9a-f]{8}$") + self.assertIn("updated_at", document) + + def test_cross_path_read_secret_raises_403_broker_error(self): + client = self._client() + state = client.ensure_registered({}) + with self.assertRaises(coreci_broker_client.BrokerError) as caught: + client.read_secret("coreci/gitea", state) + self.assertEqual(403, caught.exception.status) + self.assertTrue(caught.exception.message) + + def test_missing_secret_read_raises_404_broker_error(self): + client = self._client("nonexistent") + state = client.ensure_registered({}) + with self.assertRaises(coreci_broker_client.BrokerError) as caught: + client.read_secret("coreci/nonexistent", state) + self.assertEqual(404, caught.exception.status) + + def test_read_secret_from_cold_state_registers_then_retries_once(self): + client = self._client() + state = {} + calls, restore = self._count_requests(client) + try: + document = client.read_secret("coreci/proxmox", state) + finally: + restore() + self.assertEqual(PROXMOX_SECRET, document["data"]) + self.assertEqual( + [("GET", SECRETS_PROXMOX_PATH), + ("POST", coreci_broker_client.REGISTER_PATH), + ("GET", SECRETS_PROXMOX_PATH)], + calls, + ) + self.assertIn("current_token", state) + + def test_get_state_with_foreign_agent_token_is_403_identity_enforced(self): + intruder = self._client("proxmox") + other = self._client("gitea-agent") + other_state = other.ensure_registered({}) + intruder.ensure_registered({}) + with self.assertRaises(coreci_broker_client.BrokerError) as caught: + intruder.get_state({"current_token": other_state["current_token"]}) + self.assertEqual(403, caught.exception.status) + + +class TtlAutoReregisterTests(_ClientCase): + extra_env = {"BROKER_TOKEN_TTL": "1"} + + def test_expired_token_secret_read_auto_reregisters_and_retries(self): + client = self._client() + state = {} + first = client.read_secret("coreci/proxmox", state) + self.assertEqual(PROXMOX_SECRET, first["data"]) + first_expires = datetime.datetime.fromisoformat(state["token_expires_at"]) + time.sleep(1.2) + second = client.read_secret("coreci/proxmox", state) + self.assertEqual(PROXMOX_SECRET, second["data"]) + second_expires = datetime.datetime.fromisoformat(state["token_expires_at"]) + self.assertGreater(second_expires, first_expires) + self.assertIn("pepper", state) + + +class StaticDeliveryTests(_ClientCase): + def test_fetch_client_module_returns_own_source(self): + with open(CLIENT_SOURCE_PATH, "r", encoding="utf-8") as handle: + source = handle.read() + fetched = coreci_broker_client.fetch_client_module(self.base) + self.assertEqual(source, fetched) + self.assertIn("class BrokerClient", fetched) + + def test_client_module_served_unauthenticated(self): + with open(CLIENT_SOURCE_PATH, "r", encoding="utf-8") as handle: + source = handle.read() + with urllib.request.urlopen( + self.base + "/static/coreci_broker_client.py", timeout=10 + ) as response: + self.assertEqual(200, response.status) + self.assertEqual(source, response.read().decode("utf-8")) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..250e625 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,105 @@ +import base64 +import contextlib +import io +import os +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + + +class LoadConfigTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.env_path = os.path.join(tmp.name, ".env") + + def _boot(self, env): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + config = coreci_broker.load_config(env) + return config, buffer.getvalue() + + def _read_env_file(self): + values = {} + with open(self.env_path, "r", encoding="utf-8") as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values[key.strip()] = value.strip() + return values + + def test_first_boot_writes_env_and_prints_pass_once(self): + config, stdout = self._boot({"BROKER_ENV_PATH": self.env_path}) + self.assertTrue(os.path.isfile(self.env_path)) + stored = self._read_env_file() + admin_pass = stored["BROKER_ADMIN_PASS"] + pepper = stored["BROKER_INSTANCE_PEPPER"] + self.assertGreaterEqual(len(admin_pass), 32) + self.assertEqual(32, len(base64.b64decode(pepper))) + self.assertEqual(admin_pass, config["BROKER_ADMIN_PASS"]) + self.assertEqual(pepper, config["BROKER_INSTANCE_PEPPER"]) + self.assertEqual(1, stdout.count(admin_pass)) + self.assertNotIn(pepper, stdout) + self.assertEqual(0o600, os.stat(self.env_path).st_mode & 0o777) + + def test_second_boot_prints_nothing(self): + first, first_stdout = self._boot({"BROKER_ENV_PATH": self.env_path}) + second, second_stdout = self._boot({"BROKER_ENV_PATH": self.env_path}) + self.assertEqual(1, first_stdout.count(first["BROKER_ADMIN_PASS"])) + self.assertEqual("", second_stdout) + self.assertEqual(first["BROKER_ADMIN_PASS"], second["BROKER_ADMIN_PASS"]) + self.assertEqual(first["BROKER_INSTANCE_PEPPER"], second["BROKER_INSTANCE_PEPPER"]) + + def test_env_overrides_dotenv(self): + with open(self.env_path, "w", encoding="utf-8") as handle: + handle.write("BROKER_BIND=10.1.2.3:9999\n") + handle.write('BROKER_ADMIN_USER="ceo"\n') + handle.write("BROKER_ADMIN_PASS=oldpass\n") + handle.write("BROKER_TOKEN_TTL=100\n") + env = { + "BROKER_ENV_PATH": self.env_path, + "BROKER_ADMIN_PASS": "envpass", + "BROKER_TOKEN_TTL": "55", + "BROKER_DB_PATH": "/tmp/elsewhere/broker.db", + } + config, stdout = self._boot(env) + self.assertEqual("envpass", config["BROKER_ADMIN_PASS"]) + self.assertEqual("55", config["BROKER_TOKEN_TTL"]) + self.assertEqual("/tmp/elsewhere/broker.db", config["BROKER_DB_PATH"]) + self.assertEqual("10.1.2.3:9999", config["BROKER_BIND"]) + self.assertEqual("ceo", config["BROKER_ADMIN_USER"]) + self.assertEqual("300", config["BROKER_SESSION_TTL"]) + self.assertEqual("", stdout) + stored = self._read_env_file() + self.assertEqual("oldpass", stored["BROKER_ADMIN_PASS"]) + self.assertIn("BROKER_INSTANCE_PEPPER", stored) + + def test_defaults_applied(self): + config, _ = self._boot({"BROKER_ENV_PATH": self.env_path}) + self.assertEqual(self.env_path, config["BROKER_ENV_PATH"]) + self.assertEqual("127.0.0.1:8765", config["BROKER_BIND"]) + self.assertEqual("admin", config["BROKER_ADMIN_USER"]) + self.assertEqual("86400", config["BROKER_TOKEN_TTL"]) + self.assertEqual("300", config["BROKER_SESSION_TTL"]) + self.assertEqual("broker.db", config["BROKER_DB_PATH"]) + self.assertEqual("broker.audit.log", config["BROKER_AUDIT_LOG_PATH"]) + + def test_reads_process_environ_by_default(self): + env = {"BROKER_ENV_PATH": self.env_path, "BROKER_BIND": "127.0.0.1:9000"} + with mock.patch.dict(os.environ, env, clear=True): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + config = coreci_broker.load_config() + self.assertEqual("127.0.0.1:9000", config["BROKER_BIND"]) + self.assertEqual(self.env_path, config["BROKER_ENV_PATH"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_creds_source.py b/tests/test_creds_source.py new file mode 100644 index 0000000..3998784 --- /dev/null +++ b/tests/test_creds_source.py @@ -0,0 +1,312 @@ +import contextlib +import hashlib +import io +import json +import os +import sys +import tempfile +import threading +import unittest +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +API_KEY = "sk-test-key" +KB_NAME = "creds-coreci-coo" +PROXMOX_SECRET = {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"} +PROXMOX_UPDATED = "2026-10-01T12:00:00Z" + + +def _standard_fixtures(): + kbs = [{"id": "kb-1", "name": KB_NAME}] + files = {"kb-1": [{"name": "coreci/proxmox", "id": "f1", "updated_at": PROXMOX_UPDATED}]} + contents = {"f1": json.dumps(PROXMOX_SECRET)} + return kbs, files, contents + + +class _StubOwuiHandler(BaseHTTPRequestHandler): + kbs = None + files = None + contents = None + failures = None + hits = None + + def log_message(self, format, *args): + pass + + def _send(self, status, payload): + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + parsed = urllib.parse.urlsplit(self.path) + path = parsed.path + self.hits.append((path, self.headers.get("Authorization"))) + if path in self.failures: + self._send(self.failures[path], {"error": self.failures[path]}) + elif path == "/api/v1/knowledge/": + page = max(1, int((urllib.parse.parse_qs(parsed.query).get("page") or ["1"])[0])) + self._send(200, {"items": self.kbs[(page - 1) * 30:page * 30], "total": len(self.kbs)}) + elif path.startswith("/api/v1/knowledge/"): + kb_id = path[len("/api/v1/knowledge/"):] + kb = next((entry for entry in self.kbs if entry["id"] == kb_id), None) + if kb is None: + self._send(404, {"error": 404}) + else: + self._send(200, dict(kb, files=self.files.get(kb_id, []))) + elif path.startswith("/api/v1/files/") and path.endswith("/data/content"): + file_id = path[len("/api/v1/files/"):-len("/data/content")] + if file_id in self.contents: + self._send(200, {"content": self.contents[file_id]}) + else: + self._send(404, {"error": 404}) + else: + self._send(404, {"error": 404}) + + +class StubOwuiServer: + def __init__(self, kbs, files, contents): + self.kbs, self.files, self.contents, self.failures = kbs, files, contents, {} + self.hits = [] + handler = type("BoundStubHandler", (_StubOwuiHandler,), { + "kbs": kbs, "files": files, "contents": contents, + "failures": self.failures, "hits": self.hits, + }) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + @property + def base_url(self): + return "http://127.0.0.1:%d" % self.server.server_address[1] + + def hits_for(self, path): + return [hit for hit in self.hits if hit[0] == path] + + def close(self): + self.server.shutdown() + self.server.server_close() + + +class FileCredsSourceTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.creds_path = os.path.join(tmp.name, "creds.json") + self.document = { + "coreci/proxmox": {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"}, + "coreci/gitea": {"url": "https://git.coreci.dev", "token": "git-tok"}, + } + self._write(self.document) + self.source = coreci_broker.FileCredsSource(self.creds_path) + + def _write(self, document): + with open(self.creds_path, "w", encoding="utf-8") as handle: + json.dump(document, handle) + + def test_fetches_both_paths_with_response_shape(self): + for path, secret in self.document.items(): + result = self.source.get_secret(path) + self.assertEqual(path, result["path"]) + self.assertEqual(secret, result["data"]) + self.assertIsNone(result["updated_at"]) + self.assertRegex(result["version"], r"^[0-9a-f]{8}$") + + def test_version_is_short_sha256_of_serialized_data(self): + result = self.source.get_secret("coreci/proxmox") + serialized = json.dumps(result["data"], sort_keys=True, separators=(",", ":")) + expected = hashlib.sha256(serialized.encode("utf-8")).hexdigest()[:8] + self.assertEqual(expected, result["version"]) + + def test_equal_data_yields_equal_versions(self): + first = self.source.get_secret("coreci/proxmox") + second = self.source.get_secret("coreci/proxmox") + self.assertEqual(first["version"], second["version"]) + + def test_missing_path_is_not_found(self): + with self.assertRaises(coreci_broker._SecretNotFound): + self.source.get_secret("coreci/ghost") + + def test_missing_file_means_all_paths_not_found(self): + os.remove(self.creds_path) + for path in self.document: + with self.assertRaises(coreci_broker._SecretNotFound): + self.source.get_secret(path) + + def test_invalid_json_is_source_error(self): + with open(self.creds_path, "w", encoding="utf-8") as handle: + handle.write("{not-json") + with self.assertRaises(coreci_broker._CredsSourceError): + self.source.get_secret("coreci/proxmox") + + def test_non_object_document_is_source_error(self): + self._write(["not", "a", "mapping"]) + with self.assertRaises(coreci_broker._CredsSourceError): + self.source.get_secret("coreci/proxmox") + + def test_fresh_read_sees_out_of_band_edits(self): + first = self.source.get_secret("coreci/proxmox") + self.document["coreci/proxmox"]["token"] = "rotated" + self._write(self.document) + second = self.source.get_secret("coreci/proxmox") + self.assertEqual("pve-tok", first["data"]["token"]) + self.assertEqual("rotated", second["data"]["token"]) + self.assertNotEqual(first["version"], second["version"]) + + +class OwuiApiCredsSourceTests(unittest.TestCase): + def setUp(self): + kbs, files, contents = _standard_fixtures() + self.stub = StubOwuiServer(kbs, files, contents) + self.addCleanup(self.stub.close) + self.source = coreci_broker.OwuiApiCredsSource(self.stub.base_url, API_KEY) + + def test_roundtrip_returns_response_shape(self): + result = self.source.get_secret("coreci/proxmox") + self.assertEqual("coreci/proxmox", result["path"]) + self.assertEqual(PROXMOX_SECRET, result["data"]) + self.assertEqual(PROXMOX_UPDATED, result["updated_at"]) + self.assertRegex(result["version"], r"^[0-9a-f]{8}$") + + def test_kb_and_file_map_cached_across_calls(self): + self.source.get_secret("coreci/proxmox") + self.source.get_secret("coreci/proxmox") + self.assertEqual(1, len(self.stub.hits_for("/api/v1/knowledge/"))) + self.assertEqual(1, len(self.stub.hits_for("/api/v1/knowledge/kb-1"))) + self.assertEqual(2, len(self.stub.hits_for("/api/v1/files/f1/data/content"))) + + def test_bearer_header_present_on_every_upstream_call(self): + self.source.get_secret("coreci/proxmox") + self.assertTrue(self.stub.hits) + for _, auth in self.stub.hits: + self.assertEqual("Bearer " + API_KEY, auth) + + def test_refresh_on_miss_picks_up_newly_added_files(self): + self.source.get_secret("coreci/proxmox") + self.stub.files["kb-1"].append( + {"name": "coreci/gitea", "id": "f2", "updated_at": "2026-10-02T00:00:00Z"}) + self.stub.contents["f2"] = json.dumps({"user": "ceo"}) + result = self.source.get_secret("coreci/gitea") + self.assertEqual({"user": "ceo"}, result["data"]) + self.assertEqual("2026-10-02T00:00:00Z", result["updated_at"]) + self.assertEqual(2, len(self.stub.hits_for("/api/v1/knowledge/"))) + + def test_unknown_path_is_not_found(self): + with self.assertRaises(coreci_broker._SecretNotFound): + self.source.get_secret("coreci/ghost") + + def test_missing_kb_is_not_found(self): + self.stub.kbs.clear() + with self.assertRaises(coreci_broker._SecretNotFound): + self.source.get_secret("coreci/proxmox") + + def test_kb_list_404_is_not_found(self): + self.stub.failures["/api/v1/knowledge/"] = 404 + with self.assertRaises(coreci_broker._SecretNotFound): + self.source.get_secret("coreci/proxmox") + + def test_kb_detail_404_is_not_found(self): + self.stub.failures["/api/v1/knowledge/kb-1"] = 404 + with self.assertRaises(coreci_broker._SecretNotFound): + self.source.get_secret("coreci/proxmox") + + def test_file_content_404_is_not_found(self): + del self.stub.contents["f1"] + with self.assertRaises(coreci_broker._SecretNotFound): + self.source.get_secret("coreci/proxmox") + + def test_upstream_500_is_source_error_not_crash(self): + self.stub.failures["/api/v1/files/f1/data/content"] = 500 + with self.assertRaises(coreci_broker._CredsSourceError): + self.source.get_secret("coreci/proxmox") + + def test_kb_list_500_is_source_error(self): + self.stub.failures["/api/v1/knowledge/"] = 500 + with self.assertRaises(coreci_broker._CredsSourceError): + self.source.get_secret("coreci/proxmox") + + def test_unavailable_upstream_maps_to_distinct_source_error(self): + source = coreci_broker.OwuiApiCredsSource("http://127.0.0.1:9", API_KEY) + with self.assertRaises(coreci_broker._CredsSourceError) as caught: + source.get_secret("coreci/proxmox") + self.assertIn("unavailable", str(caught.exception)) + + def test_invalid_content_json_is_source_error(self): + self.stub.contents["f1"] = "not-json" + with self.assertRaises(coreci_broker._CredsSourceError): + self.source.get_secret("coreci/proxmox") + + def test_pagination_finds_kb_beyond_first_page(self): + self.stub.kbs[:] = [{"id": "kb-%d" % i, "name": "kb-%d" % i} for i in range(30)] + self.stub.kbs.append({"id": "kb-creds", "name": KB_NAME}) + self.stub.files["kb-creds"] = [ + {"name": "coreci/proxmox", "id": "f1", "updated_at": PROXMOX_UPDATED}] + result = self.source.get_secret("coreci/proxmox") + self.assertEqual(PROXMOX_SECRET, result["data"]) + self.assertEqual(2, len(self.stub.hits_for("/api/v1/knowledge/"))) + + +class CredsSourceFactoryTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + + def _boot(self, extra): + env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + } + env.update(extra) + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(env) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + return broker + + def test_default_source_is_file_with_default_path(self): + broker = self._boot({}) + self.assertIsInstance(broker.creds_source, coreci_broker.FileCredsSource) + self.assertEqual("data/creds.json", broker.creds_source.path) + + def test_explicit_file_source_with_custom_path(self): + creds_path = os.path.join(self.tmp, "custom-creds.json") + with open(creds_path, "w", encoding="utf-8") as handle: + json.dump({"coreci/proxmox": PROXMOX_SECRET}, handle) + broker = self._boot({ + "BROKER_CREDS_SOURCE": "file", + "BROKER_CREDS_FILE": creds_path, + }) + self.assertIsInstance(broker.creds_source, coreci_broker.FileCredsSource) + self.assertEqual(creds_path, broker.creds_source.path) + self.assertEqual(PROXMOX_SECRET, broker.creds_source.get_secret("coreci/proxmox")["data"]) + + def test_owui_source_selected_by_config(self): + kbs, files, contents = _standard_fixtures() + stub = StubOwuiServer(kbs, files, contents) + self.addCleanup(stub.close) + broker = self._boot({ + "BROKER_CREDS_SOURCE": "owui", + "BROKER_OWUI_BASE_URL": stub.base_url, + "BROKER_OWUI_API_KEY": API_KEY, + }) + source = broker.creds_source + self.assertIsInstance(source, coreci_broker.OwuiApiCredsSource) + self.assertEqual(stub.base_url, source.base_url) + self.assertEqual(API_KEY, source.api_key) + self.assertEqual(PROXMOX_SECRET, source.get_secret("coreci/proxmox")["data"]) + + def test_unknown_source_value_raises_at_boot(self): + with self.assertRaises(ValueError): + self._boot({"BROKER_CREDS_SOURCE": "banana"}) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_hardening.py b/tests/test_hardening.py new file mode 100644 index 0000000..29da28a --- /dev/null +++ b/tests/test_hardening.py @@ -0,0 +1,162 @@ +import contextlib +import io +import json +import os +import sqlite3 +import sys +import tempfile +import threading +import unittest +import urllib.error +import urllib.request +from unittest import mock + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +THREAD_COUNT = 8 +WRITES_PER_THREAD = 16 +MAX_BODY_BYTES = 1024 * 1024 + + +class HardeningTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + } + + def _boot_and_serve(self): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + broker = coreci_broker.boot(dict(self.env)) + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return broker, server + + def test_oversized_declared_content_length_post_rejected(self): + broker, server = self._boot_and_serve() + port = server.server_address[1] + request = urllib.request.Request( + "http://127.0.0.1:%d/healthz" % port, + data=b"", + headers={"Content-Length": str(MAX_BODY_BYTES + 1)}, + method="POST", + ) + with self.assertRaises(urllib.error.HTTPError) as raised: + urllib.request.urlopen(request, timeout=5) + error = raised.exception + self.assertEqual(400, error.code) + payload = json.loads(error.read().decode("utf-8")) + self.assertEqual({"error": 400, "message": "body too large"}, payload) + + def test_handler_timeout_attribute_is_30(self): + self.assertEqual(30, coreci_broker.BrokerRequestHandler.timeout) + broker, server = self._boot_and_serve() + self.assertEqual(30, server.RequestHandlerClass.timeout) + + def test_concurrent_store_writes_from_eight_threads(self): + db_path = os.path.join(self.tmp, "concurrent.db") + store = coreci_broker.Store(db_path) + self.addCleanup(store.close) + errors = [] + barrier = threading.Barrier(THREAD_COUNT) + + def worker(index): + try: + barrier.wait(timeout=5) + for slot in range(WRITES_PER_THREAD): + store.set_config( + "key-%d-%d" % (index, slot), "value-%d-%d" % (index, slot) + ) + store.get_config("key-%d-0" % index) + except BaseException as error: + errors.append(error) + + threads = [ + threading.Thread(target=worker, args=(index,)) + for index in range(THREAD_COUNT) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + self.assertEqual([], errors) + for index in range(THREAD_COUNT): + for slot in range(WRITES_PER_THREAD): + key = "key-%d-%d" % (index, slot) + expected = "value-%d-%d" % (index, slot) + self.assertEqual(expected, store.get_config(key)) + conn = sqlite3.connect(db_path) + self.addCleanup(conn.close) + count = conn.execute("SELECT COUNT(*) FROM broker_config").fetchone()[0] + self.assertEqual(THREAD_COUNT * WRITES_PER_THREAD, count) + + def test_first_boot_env_write_path_still_works(self): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + config = coreci_broker.load_config(dict(self.env)) + env_path = self.env["BROKER_ENV_PATH"] + self.assertTrue(os.path.isfile(env_path)) + self.assertEqual(0o600, os.stat(env_path).st_mode & 0o777) + stored = {} + with open(env_path, "r", encoding="utf-8") as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + stored[key.strip()] = value.strip() + self.assertEqual(config["BROKER_ADMIN_PASS"], stored["BROKER_ADMIN_PASS"]) + self.assertEqual( + config["BROKER_INSTANCE_PEPPER"], stored["BROKER_INSTANCE_PEPPER"] + ) + self.assertEqual(1, buffer.getvalue().count(config["BROKER_ADMIN_PASS"])) + second = io.StringIO() + with contextlib.redirect_stdout(second): + coreci_broker.load_config(dict(self.env)) + self.assertEqual("", second.getvalue()) + + def test_append_env_entries_write_error_propagates_with_single_close(self): + env_path = os.path.join(self.tmp, ".env") + with open(env_path, "w", encoding="utf-8") as handle: + handle.write("seed=1\n") + closes = [] + + class FailingHandle: + def __init__(self, fd): + self.fd = fd + + def write(self, data): + raise OSError("simulated write failure") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + os.close(self.fd) + closes.append(self.fd) + return False + + def fake_fdopen(fd, *args, **kwargs): + return FailingHandle(fd) + + with mock.patch.object(os, "fdopen", fake_fdopen): + with self.assertRaises(OSError) as raised: + coreci_broker._append_env_entries(env_path, [("NEW", "value")]) + self.assertEqual(1, len(closes)) + self.assertIn("simulated write failure", str(raised.exception)) + with open(env_path, "r", encoding="utf-8") as handle: + self.assertEqual("seed=1\n", handle.read()) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_hasher.py b/tests/test_hasher.py new file mode 100644 index 0000000..b2bd1b9 --- /dev/null +++ b/tests/test_hasher.py @@ -0,0 +1,207 @@ +import base64 +import hashlib +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +SCRYPT_N = 2 ** 14 +SCRYPT_R = 8 +SCRYPT_P = 1 + +BCRYPT_SHAPED = "$2b$12$K7L9O1P3QRSTUVWXYZabcdefghijk012345678901234567890123456789" + + +def _scrypt_stored(token, salt): + digest = hashlib.scrypt( + token.encode("utf-8"), salt=salt, n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P + ) + return "{}${}${}".format( + "scrypt", + base64.urlsafe_b64encode(salt).decode("ascii"), + base64.urlsafe_b64encode(digest).decode("ascii"), + ) + + +class _FakeBcryptModule: + def __init__(self): + self.gensalt_calls = [] + self.hashpw_calls = [] + self.checkpw_calls = [] + + def gensalt(self): + self.gensalt_calls.append(True) + return b"$2b$12$fakesaltfakesalt" + + def hashpw(self, password, salt): + self.hashpw_calls.append((password, salt)) + return b"$2b$12$storedhashvalue" + + def checkpw(self, password, hashed): + self.checkpw_calls.append((password, hashed)) + return password == b"mock-token" + + +class HashFormatTests(unittest.TestCase): + def test_bcrypt_is_not_importable_on_this_host(self): + self.assertIsNone(coreci_broker.bcrypt) + with self.assertRaises(ImportError): + import bcrypt + + def test_hash_output_starts_with_scrypt_prefix(self): + stored = coreci_broker.hash_token("format-token") + self.assertTrue(stored.startswith("scrypt$")) + + def test_hash_output_carries_salt_and_digest_b64url_fields(self): + stored = coreci_broker.hash_token("format-token") + fields = stored.split("$") + self.assertEqual(3, len(fields)) + self.assertEqual("scrypt", fields[0]) + salt = base64.urlsafe_b64decode(fields[1].encode("ascii")) + digest = base64.urlsafe_b64decode(fields[2].encode("ascii")) + self.assertEqual(16, len(salt)) + self.assertEqual(64, len(digest)) + + +class RoundtripTests(unittest.TestCase): + def test_hash_then_verify_roundtrip(self): + stored = coreci_broker.hash_token("roundtrip-token") + self.assertTrue(coreci_broker.verify_token("roundtrip-token", stored)) + + def test_wrong_token_fails(self): + stored = coreci_broker.hash_token("roundtrip-token") + self.assertFalse(coreci_broker.verify_token("wrong-token", stored)) + + def test_two_hashes_of_same_token_differ(self): + first = coreci_broker.hash_token("same-token") + second = coreci_broker.hash_token("same-token") + self.assertNotEqual(first, second) + self.assertTrue(coreci_broker.verify_token("same-token", first)) + self.assertTrue(coreci_broker.verify_token("same-token", second)) + + +class ScryptParseTests(unittest.TestCase): + def test_hand_built_stored_value_verifies(self): + stored = _scrypt_stored("parse-token", bytes(range(16))) + self.assertTrue(stored.startswith("scrypt$")) + self.assertTrue(coreci_broker.verify_token("parse-token", stored)) + + def test_hand_built_stored_value_rejects_other_token(self): + stored = _scrypt_stored("parse-token", bytes(range(16))) + self.assertFalse(coreci_broker.verify_token("parse-token-x", stored)) + + def test_tampered_digest_field_fails(self): + salt = bytes(range(16)) + digest = hashlib.scrypt( + b"parse-token", salt=salt, n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P + ) + tampered = digest[:-1] + bytes([digest[-1] ^ 0x01]) + stored = "{}${}${}".format( + "scrypt", + base64.urlsafe_b64encode(salt).decode("ascii"), + base64.urlsafe_b64encode(tampered).decode("ascii"), + ) + self.assertFalse(coreci_broker.verify_token("parse-token", stored)) + + def test_malformed_scrypt_values_fail_cleanly(self): + for stored in ( + "scrypt$", + "scrypt$abc", + "scrypt$abc$def$ghi", + "scrypt$$", + "scrypt$!@#$%", + ): + with self.subTest(stored=stored): + self.assertFalse(coreci_broker.verify_token("x", stored)) + + +class UnknownPrefixTests(unittest.TestCase): + def test_unknown_prefixes_return_false_without_raising(self): + for stored in ( + "", + "plaintext", + "$2c$12$abc", + "$2x$12$abc", + "argon2$abc$def", + "SCRYPT$abc$def", + "scrypt2$abc$def", + "0123456789", + ): + with self.subTest(stored=stored): + self.assertFalse(coreci_broker.verify_token("x", stored)) + + def test_non_string_stored_returns_false_cleanly(self): + self.assertFalse(coreci_broker.verify_token("x", None)) + self.assertFalse(coreci_broker.verify_token("x", 5)) + self.assertFalse(coreci_broker.verify_token("x", ["scrypt$abc$def"])) + + +class BcryptDispatchContractTests(unittest.TestCase): + def test_bcrypt_prefix_returns_false_cleanly_with_bcrypt_absent(self): + self.assertIsNone(coreci_broker.bcrypt) + with self.assertRaises(ImportError): + import bcrypt + self.assertFalse(coreci_broker.verify_token("x", BCRYPT_SHAPED)) + + def test_all_bcrypt_prefixes_return_false_cleanly_with_bcrypt_absent(self): + self.assertIsNone(coreci_broker.bcrypt) + for prefix in ("$2a$", "$2b$", "$2y$"): + with self.subTest(prefix=prefix): + self.assertFalse(coreci_broker.verify_token("x", prefix + "12$storedhashvalue")) + + +class BcryptBranchTests(unittest.TestCase): + def test_hash_token_uses_bcrypt_branch_when_importable(self): + fake = _FakeBcryptModule() + with mock.patch.object(coreci_broker, "bcrypt", fake): + stored = coreci_broker.hash_token("mock-token") + self.assertEqual("$2b$12$storedhashvalue", stored) + self.assertEqual([True], fake.gensalt_calls) + self.assertEqual([(b"mock-token", b"$2b$12$fakesaltfakesalt")], fake.hashpw_calls) + + def test_verify_dispatches_bcrypt_prefixes_to_bcrypt_branch(self): + fake = _FakeBcryptModule() + expected = [ + (b"mock-token", (prefix + "12$storedhashvalue").encode("ascii")) + for prefix in ("$2a$", "$2b$", "$2y$") + ] + with mock.patch.object(coreci_broker, "bcrypt", fake): + for password, hashed in expected: + with self.subTest(hashed=hashed): + self.assertTrue(coreci_broker.verify_token(password.decode("utf-8"), hashed.decode("ascii"))) + self.assertEqual(expected, fake.checkpw_calls) + + def test_bcrypt_checkpw_rejection_returns_false(self): + fake = _FakeBcryptModule() + with mock.patch.object(coreci_broker, "bcrypt", fake): + self.assertFalse(coreci_broker.verify_token("other-token", "$2b$12$storedhashvalue")) + self.assertEqual([(b"other-token", b"$2b$12$storedhashvalue")], fake.checkpw_calls) + + def test_bcrypt_checkpw_error_returns_false_cleanly(self): + fake = _FakeBcryptModule() + + def broken(password, hashed): + raise ValueError("invalid salt") + + fake.checkpw = broken + with mock.patch.object(coreci_broker, "bcrypt", fake): + self.assertFalse(coreci_broker.verify_token("mock-token", "$2b$garbage")) + + def test_scrypt_and_unknown_stored_never_dispatch_to_bcrypt(self): + fake = _FakeBcryptModule() + stored = coreci_broker.hash_token("mock-token") + self.assertTrue(stored.startswith("scrypt$")) + with mock.patch.object(coreci_broker, "bcrypt", fake): + self.assertTrue(coreci_broker.verify_token("mock-token", stored)) + self.assertFalse(coreci_broker.verify_token("other", stored)) + self.assertFalse(coreci_broker.verify_token("mock-token", "unknown$prefix")) + self.assertEqual([], fake.checkpw_calls) + self.assertEqual([], fake.hashpw_calls) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_mounts.py b/tests/test_mounts.py new file mode 100644 index 0000000..7b49aa3 --- /dev/null +++ b/tests/test_mounts.py @@ -0,0 +1,148 @@ +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.request +from http.server import ThreadingHTTPServer +from unittest import mock + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + + +class MountsTests(unittest.TestCase): + def setUp(self): + coreci_broker._reset_server_guard() + tmp = tempfile.TemporaryDirectory() + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_BIND": "127.0.0.1:0", + } + self.addCleanup(tmp.cleanup) + self.addCleanup(coreci_broker._reset_server_guard) + + def _patched_environ(self): + return mock.patch.dict(os.environ, self.env, clear=True) + + def _read_env_values(self): + values = {} + with open(self.env["BROKER_ENV_PATH"], "r", encoding="utf-8") as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values[key.strip()] = value.strip() + return values + + def _healthz(self, port): + url = "http://127.0.0.1:%d/healthz" % port + with urllib.request.urlopen(url, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + + def test_owui_docstring_frontmatter(self): + doc = coreci_broker.__doc__ or "" + self.assertTrue(doc.startswith("title: CoreCI Broker")) + for line in ( + "author: coreci", + "description: ", + "version: 0.1.0", + "requirements: []", + "licence: MIT", + ): + self.assertIn(line, doc) + + def test_source_never_references_dunder_file(self): + with open(coreci_broker.__file__, "r", encoding="utf-8") as handle: + source = handle.read() + self.assertNotIn("__file__", source) + + def test_tools_double_instantiation_starts_exactly_one_server(self): + serve_calls = [] + real_serve = coreci_broker.serve + + def counting_serve(host, port, broker): + serve_calls.append((host, port)) + return real_serve(host, port, broker) + + with self._patched_environ(): + with mock.patch.object(coreci_broker, "serve", counting_serve): + with contextlib.redirect_stdout(io.StringIO()): + first = coreci_broker.Tools() + port_after_first = coreci_broker._started_server.server_address[1] + second = coreci_broker.Tools() + guard = coreci_broker._started_server + self.assertEqual(1, len(serve_calls)) + self.assertIsInstance(guard, ThreadingHTTPServer) + self.assertIs(coreci_broker._started_server, first._server) + self.assertIs(first._server, second._server) + self.assertEqual(port_after_first, guard.server_address[1]) + self.assertNotEqual(0, guard.server_address[1]) + self.assertIsInstance(first.valves, coreci_broker.Tools.Valves) + self.assertIsInstance(second.valves, coreci_broker.Tools.Valves) + status, payload = self._healthz(port_after_first) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, payload) + + def test_tools_status_returns_non_empty_string(self): + with self._patched_environ(): + with contextlib.redirect_stdout(io.StringIO()): + tools = coreci_broker.Tools() + status = tools.status() + port = tools._server.server_address[1] + self.assertIsInstance(status, str) + self.assertTrue(status.strip()) + self.assertIn(str(port), status) + + def test_first_boot_print_discipline_through_tools_boot(self): + with self._patched_environ(): + first = io.StringIO() + with contextlib.redirect_stdout(first): + coreci_broker.Tools() + second = io.StringIO() + with contextlib.redirect_stdout(second): + coreci_broker.Tools() + stored = self._read_env_values() + admin_pass = stored["BROKER_ADMIN_PASS"] + pepper = stored["BROKER_INSTANCE_PEPPER"] + self.assertEqual(1, first.getvalue().count(admin_pass)) + self.assertNotIn(pepper, first.getvalue()) + self.assertEqual("", second.getvalue()) + + def test_run_standalone_boots_serves_and_shuts_down_cleanly(self): + captured = {} + real_serve = coreci_broker.serve + + def capturing_serve(host, port, broker): + server = real_serve(host, port, broker) + captured["server"] = server + return server + + def fake_sleep(seconds): + server = captured["server"] + status, payload = self._healthz(server.server_address[1]) + captured["healthz"] = (status, payload) + raise KeyboardInterrupt + + stdout = io.StringIO() + with mock.patch.object(coreci_broker, "serve", capturing_serve): + with mock.patch.object(coreci_broker.time, "sleep", fake_sleep): + with contextlib.redirect_stdout(stdout): + coreci_broker.run_standalone(dict(self.env)) + out = stdout.getvalue() + self.assertEqual((200, {"ok": True}), captured["healthz"]) + self.assertEqual(1, out.count("BROKER_ADMIN_PASS=")) + self.assertIn("listening on http://127.0.0.1:", out) + self.assertIn("shut down", out) + self.assertEqual(-1, captured["server"].socket.fileno()) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_p3_remediation.py b/tests/test_p3_remediation.py new file mode 100644 index 0000000..1223b10 --- /dev/null +++ b/tests/test_p3_remediation.py @@ -0,0 +1,249 @@ +import base64 +import contextlib +import datetime +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +PROTECTED_PATH = "/test/p3-remediation" +AUTH_401_BODY = {"error": 401, "message": "invalid app credentials"} +CAP_BODY = {"error": 400, "message": "too many agents for this app"} +AGENT_ID_BODY = {"error": 400, "message": "invalid agent_id"} + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +def _protected_echo(request): + request.send_json(200, dict(request.app_auth)) + + +class P3RemediationTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed(self.app_uuid, self.user_id) + self.broker.router.add( + "GET", PROTECTED_PATH, coreci_broker._require_app(_protected_echo) + ) + + def _request(self, method, path, payload=None, headers=None, base=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + (base or self.base) + path, data=data, method=method + ) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _seed(self, app_uuid, user_id, base=None): + status, _ = self._request("POST", APPS_PATH, {"uuid": app_uuid}, self.admin, base) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": user_id}, self.admin, base) + self.assertEqual(201, status) + + def _register(self, agent_id="agent-1", tags=("coreci",), app_uuid=None, + user_id=None, base=None): + payload = { + "agent_id": agent_id, + "app_registration_uuid": app_uuid or self.app_uuid, + "user_id": user_id or self.user_id, + "tags": list(tags), + } + return self._request("POST", REGISTER_PATH, payload, base=base) + + def _auth(self, token, app_uuid=None): + header = "App %s %s" % (app_uuid or self.app_uuid, token) + return self._request("GET", PROTECTED_PATH, headers={"Authorization": header}) + + def _audit(self, action=None, result=None): + with open(self.env["BROKER_AUDIT_LOG_PATH"], "r", encoding="utf-8") as handle: + entries = [json.loads(raw) for raw in handle if raw.strip()] + return [ + entry for entry in entries + if (action is None or entry["action"] == action) + and (result is None or entry["result"] == result) + ] + + def _future(self, seconds=3600): + return (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=seconds)).isoformat() + + def test_agent_id_colon_slash_backslash_rejected(self): + for bad in ("a:b", "a/b", "a\\b"): + status, body = self._register(agent_id=bad) + self.assertEqual(400, status, repr(bad)) + self.assertEqual(AGENT_ID_BODY, body) + status, body = self._register(agent_id="a:b", app_uuid=str(uuidlib.uuid4())) + self.assertEqual(400, status) + self.assertEqual(AGENT_ID_BODY, body) + self.assertEqual([], self._audit("register")) + self.assertEqual([], self.broker.store.agents_by_uuid(self.app_uuid)) + + def test_tag_containing_colon_rejected(self): + status, body = self._register(tags=("ok", "ba:d")) + self.assertEqual(400, status) + self.assertEqual({"error": 400, "message": "invalid tags"}, body) + self.assertEqual([], self.broker.store.agents_by_uuid(self.app_uuid)) + status, body = self._register(tags=("ok",)) + self.assertEqual(201, status) + self.assertTrue(body["created"]) + + def test_user_id_colon_rejected_at_allowlist_add(self): + status, body = self._request("POST", USERS_PATH, {"user_id": "co:o"}, self.admin) + self.assertEqual(400, status) + self.assertEqual({"error": 400, "message": "invalid user_id"}, body) + status, body = self._register(user_id="co:o") + self.assertEqual(403, status) + self.assertEqual({"error": 403, "message": "registration not allowed"}, body) + self.assertEqual(1, len(self._audit("register", "denied"))) + status, body = self._request("POST", USERS_PATH, {"user_id": "coo"}, self.admin) + self.assertEqual(201, status) + status, body = self._register(agent_id="coo-agent", user_id="coo") + self.assertEqual(201, status) + + def test_default_fifty_agent_cap_blocks_register(self): + self.assertEqual("50", self.broker.config["BROKER_MAX_AGENTS_PER_APP"]) + now = datetime.datetime.now(datetime.timezone.utc).isoformat() + seed_hash = coreci_broker.hash_token("seed") + for index in range(50): + self.broker.store.save_agent( + "seeded-%d" % index, self.app_uuid, self.user_id, "[]", + seed_hash, self._future(), now, + ) + status, body = self._register(agent_id="overflow") + self.assertEqual(400, status) + self.assertEqual(CAP_BODY, body) + status, body = self._register(agent_id="seeded-0") + self.assertEqual(400, status) + self.assertEqual(CAP_BODY, body) + self.assertIsNone(self.broker.store.get_agent("overflow", self.app_uuid)) + self.assertEqual(50, len(self.broker.store.agents_by_uuid(self.app_uuid))) + self.assertEqual([], self._audit("register")) + + def test_max_agents_per_app_env_override_honored(self): + env = dict(self.env) + env["BROKER_DB_PATH"] = os.path.join(self.tmp, "cap.db") + env["BROKER_AUDIT_LOG_PATH"] = os.path.join(self.tmp, "cap.audit.log") + env["BROKER_MAX_AGENTS_PER_APP"] = "2" + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(dict(env)) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + self.assertEqual("2", broker.config["BROKER_MAX_AGENTS_PER_APP"]) + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(server.shutdown) + self.addCleanup(server.server_close) + base = "http://127.0.0.1:%d" % server.server_address[1] + self._seed(self.app_uuid, self.user_id, base=base) + for agent_id in ("agent-a", "agent-b"): + status, body = self._register(agent_id=agent_id, base=base) + self.assertEqual(201, status, agent_id) + self.assertTrue(body["created"]) + status, body = self._register(agent_id="agent-c", base=base) + self.assertEqual(400, status) + self.assertEqual(CAP_BODY, body) + self.assertIsNone(broker.store.get_agent("agent-c", self.app_uuid)) + self.assertEqual(2, len(broker.store.agents_by_uuid(self.app_uuid))) + + def test_overwrite_evicts_uuid_cache_entries(self): + _, first = self._register(tags=("red",)) + old_token = first["token"] + status, _ = self._auth(old_token) + self.assertEqual(200, status) + self.assertIn((self.app_uuid, old_token), self.broker.token_cache) + status, second = self._register(tags=("blue",)) + self.assertEqual(200, status) + self.assertFalse(second["created"]) + new_token = second["token"] + self.assertNotEqual(old_token, new_token) + self.assertEqual([], [k for k in self.broker.token_cache if k[0] == self.app_uuid]) + status, body = self._auth(old_token) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + self.assertNotIn((self.app_uuid, old_token), self.broker.token_cache) + status, body = self._auth(new_token) + self.assertEqual(200, status) + self.assertEqual("agent-1", body["agent_id"]) + self.assertIn((self.app_uuid, new_token), self.broker.token_cache) + row = self.broker.store.get_agent("agent-1", self.app_uuid) + self.assertFalse(coreci_broker.verify_token(old_token, row[4])) + self.assertTrue(coreci_broker.verify_token(new_token, row[4])) + + def test_integrity_error_race_takes_idempotent_path(self): + agent_id = "race-agent" + tags = ["coreci"] + pepper = self.broker.store.get_config("BROKER_INSTANCE_PEPPER") + token = coreci_broker.derive_token(pepper, self.app_uuid, self.user_id, agent_id, tags) + expires = self._future() + now = datetime.datetime.now(datetime.timezone.utc).isoformat() + self.broker.store.save_agent( + agent_id, self.app_uuid, self.user_id, json.dumps(sorted(tags)), + coreci_broker.hash_token(token), expires, now, + ) + real_get_agent = self.broker.store.get_agent + calls = [] + + def racing_get_agent(agent_id, app_uuid): + calls.append(agent_id) + if len(calls) == 1: + return None + return real_get_agent(agent_id, app_uuid) + + self.broker.store.get_agent = racing_get_agent + self.addCleanup(setattr, self.broker.store, "get_agent", real_get_agent) + status, body = self._register(agent_id=agent_id, tags=tags) + self.assertEqual(200, status) + self.assertFalse(body["created"]) + self.assertEqual(token, body["token"]) + self.assertEqual(expires, body["token_expires_at"]) + self.assertIn("pepper", body) + self.assertEqual(2, len(calls)) + row = real_get_agent(agent_id, self.app_uuid) + self.assertTrue(coreci_broker.verify_token(token, row[4])) + self.assertGreaterEqual(row[7], now) + self.assertEqual(1, len(self._audit("register", "ok"))) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_p5_remediation.py b/tests/test_p5_remediation.py new file mode 100644 index 0000000..2866e50 --- /dev/null +++ b/tests/test_p5_remediation.py @@ -0,0 +1,207 @@ +import base64 +import contextlib +import errno +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib +from unittest import mock + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +AGENT_ID_BODY = {"error": 400, "message": "invalid agent_id"} +TAGS_BODY = {"error": 400, "message": "invalid tags"} +USER_ID_BODY = {"error": 400, "message": "invalid user_id"} + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class P5RegisterValidationTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed() + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _seed(self): + status, _ = self._request("POST", APPS_PATH, {"uuid": self.app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": self.user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self, agent_id="agent-1", tags=("coreci",)): + return self._request("POST", REGISTER_PATH, { + "agent_id": agent_id, + "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, + "tags": list(tags), + }) + + def _audit(self, action=None, result=None): + with open(self.env["BROKER_AUDIT_LOG_PATH"], "r", encoding="utf-8") as handle: + entries = [json.loads(raw) for raw in handle if raw.strip()] + return [ + entry for entry in entries + if (action is None or entry["action"] == action) + and (result is None or entry["result"] == result) + ] + + def test_agent_id_bare_dot_and_dotdot_rejected(self): + for agent_id in (".", ".."): + status, body = self._register(agent_id=agent_id) + self.assertEqual(400, status, repr(agent_id)) + self.assertEqual(AGENT_ID_BODY, body) + self.assertEqual([], self.broker.store.agents_by_uuid(self.app_uuid)) + self.assertEqual([], self._audit("register")) + + def test_agent_id_control_chars_rejected(self): + for agent_id in ("a\x00b", "a\nb", "a\x1fb", "a\x7fb"): + status, body = self._register(agent_id=agent_id) + self.assertEqual(400, status, repr(agent_id)) + self.assertEqual(AGENT_ID_BODY, body) + self.assertEqual([], self.broker.store.agents_by_uuid(self.app_uuid)) + self.assertEqual([], self._audit("register")) + + def test_agent_id_embedded_dots_still_accepted(self): + status, body = self._register(agent_id="a.b") + self.assertEqual(201, status) + self.assertTrue(body["created"]) + status, body = self._register(agent_id="tool.v2") + self.assertEqual(201, status) + self.assertTrue(body["created"]) + self.assertIsNotNone(self.broker.store.get_agent("a.b", self.app_uuid)) + self.assertIsNotNone(self.broker.store.get_agent("tool.v2", self.app_uuid)) + self.assertEqual(2, len(self._audit("register", "ok"))) + + def test_tag_control_chars_rejected(self): + for tags in (("ok", "ba\x00d"), ("ok", "ba\x7fd"), ("ok", "ba:d")): + status, body = self._register(tags=tags) + self.assertEqual(400, status, repr(tags)) + self.assertEqual(TAGS_BODY, body) + self.assertEqual([], self.broker.store.agents_by_uuid(self.app_uuid)) + status, body = self._register(tags=("a.b", "..")) + self.assertEqual(201, status) + self.assertTrue(body["created"]) + row = self.broker.store.get_agent("agent-1", self.app_uuid) + self.assertEqual(["..", "a.b"], json.loads(row[3])) + + def test_agent_id_path_separators_still_rejected(self): + for agent_id in ("a:b", "a/b", "a\\b"): + status, body = self._register(agent_id=agent_id) + self.assertEqual(400, status, repr(agent_id)) + self.assertEqual(AGENT_ID_BODY, body) + self.assertEqual([], self._audit("register")) + status, body = self._register(agent_id="plain-agent") + self.assertEqual(201, status) + self.assertTrue(body["created"]) + + def test_user_allowlist_add_shares_id_rules(self): + for user_id in (".", "..", "a\x00b", "a:b", "a/b"): + status, body = self._request("POST", USERS_PATH, {"user_id": user_id}, self.admin) + self.assertEqual(400, status, repr(user_id)) + self.assertEqual(USER_ID_BODY, body) + status, body = self._request("POST", USERS_PATH, {"user_id": "ceo.two"}, self.admin) + self.assertEqual(201, status) + + +class P5ReexecGuardTests(unittest.TestCase): + def setUp(self): + coreci_broker._reset_server_guard() + self.addCleanup(coreci_broker._reset_server_guard) + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + + def _env(self, bind): + return { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_BIND": bind, + } + + def _exec_broker(self): + with open(coreci_broker.__file__, "r", encoding="utf-8") as handle: + source = handle.read() + namespace = {"__name__": "coreci_broker_reexec"} + exec(compile(source, "coreci_broker.py", "exec"), namespace) + return namespace + + def _healthz(self, port): + url = "http://127.0.0.1:%d/healthz" % port + with urllib.request.urlopen(url, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + + def test_fresh_namespace_tools_fails_loud_when_old_module_serves(self): + first = self._exec_broker() + with mock.patch.dict(os.environ, self._env("127.0.0.1:0"), clear=True): + with contextlib.redirect_stdout(io.StringIO()): + first_tools = first["Tools"]() + first_again = first["Tools"]() + server = first["_started_server"] + self.addCleanup(first["_reset_server_guard"]) + self.assertIs(server, first_tools._server) + self.assertIs(first_tools._server, first_again._server) + port = server.server_address[1] + self.assertNotEqual(0, port) + second = self._exec_broker() + self.assertIsNone(second["_started_server"]) + with mock.patch.dict(os.environ, self._env("127.0.0.1:%d" % port), clear=True): + with contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(RuntimeError) as caught: + second["Tools"]() + message = str(caught.exception).lower() + self.assertIn("restart", message) + self.assertIn("open webui", message) + self.assertIsInstance(caught.exception.__cause__, OSError) + self.assertEqual(errno.EADDRINUSE, caught.exception.__cause__.errno) + self.assertIsNone(second["_started_server"]) + self.assertEqual((200, {"ok": True}), self._healthz(port)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_reference_tools.py b/tests/test_reference_tools.py new file mode 100644 index 0000000..76cb203 --- /dev/null +++ b/tests/test_reference_tools.py @@ -0,0 +1,271 @@ +import ast +import base64 +import contextlib +import importlib.util +import io +import json +import os +import sys +import tempfile +import time +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +PROXMOX_SECRET = {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"} +REFERENCE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, "reference_tools") +) +REFERENCE_TITLES = { + "proxmox_readonly_reference": "Proxmox Read-Only Reference [CoreCI]", + "minimal_auth_showcase": "Minimal Auth Showcase [CoreCI]", +} +ALLOWED_REFERENCE_IMPORTS = {"coreci_broker_client", "datetime"} +FORBIDDEN_SOURCE_MARKERS = ("urllib", "httpx", "socket", "proxmoxer") +BROKER_ONLY_AUDIT_ACTIONS = {"add_app", "add_user", "register", "read"} + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +def _load_reference(name): + spec = importlib.util.spec_from_file_location( + name, os.path.join(REFERENCE_DIR, name + ".py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _reference_source(name): + with open(os.path.join(REFERENCE_DIR, name + ".py"), "r", encoding="utf-8") as handle: + return handle.read() + + +PROXMOX_REFERENCE = _load_reference("proxmox_readonly_reference") +MINIMAL_SHOWCASE = _load_reference("minimal_auth_showcase") + + +class _ReferenceCase(unittest.TestCase): + extra_env = {} + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.creds_path = os.path.join(self.tmp, "creds.json") + with open(self.creds_path, "w", encoding="utf-8") as handle: + handle.write(json.dumps({"coreci/proxmox": PROXMOX_SECRET})) + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + "BROKER_CREDS_SOURCE": "file", + "BROKER_CREDS_FILE": self.creds_path, + } + self.env.update(self.extra_env) + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(self.env) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed() + + def _seed(self): + status, _ = self._request("POST", APPS_PATH, {"uuid": self.app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": self.user_id}, self.admin) + self.assertEqual(201, status) + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8") + return error.code, json.loads(raw) if raw else None + + def _audit(self, action=None, result=None): + with open(self.env["BROKER_AUDIT_LOG_PATH"], "r", encoding="utf-8") as handle: + entries = [json.loads(raw) for raw in handle if raw.strip()] + return [ + entry for entry in entries + if (action is None or entry["action"] == action) + and (result is None or entry["result"] == result) + ] + + def _tools(self, module, agent_id="proxmox"): + tools = module.Tools() + tools.valves.base_url = self.base + tools.valves.agent_id = agent_id + tools.valves.app_uuid = self.app_uuid + tools.valves.user_id = self.user_id + return tools + + def _persisted_state(self, agent_id="proxmox"): + return self.broker.store.get_agent_state(agent_id, self.app_uuid) + + +class ProxmoxReferenceFlowTests(_ReferenceCase): + def test_first_call_registers_persists_state_reads_secret_dry_runs(self): + tools = self._tools(PROXMOX_REFERENCE) + response = tools.list_nodes() + self.assertIs(True, response["dry_run"]) + self.assertEqual("list_nodes", response["action"]) + self.assertIs(True, response["credential_present"]) + self.assertTrue(str(response["note"]).strip()) + self.assertNotIn("data", response) + self.assertNotIn("pve-tok", json.dumps(response)) + self.assertEqual(1, len(self._audit(action="register"))) + document = self._persisted_state() + self.assertIsNotNone(document) + self.assertIn("current_token", document) + self.assertEqual(tools._state["current_token"], document["current_token"]) + self.assertEqual([], self._audit(result="denied")) + self.assertTrue( + {entry["action"] for entry in self._audit()} <= BROKER_ONLY_AUDIT_ACTIONS + ) + + def test_second_call_reuses_persisted_state_without_new_register(self): + tools = self._tools(PROXMOX_REFERENCE) + tools.list_nodes() + token = tools._state["current_token"] + second = tools.list_nodes() + self.assertIs(True, second["dry_run"]) + self.assertEqual("list_nodes", second["action"]) + self.assertIs(True, second["credential_present"]) + self.assertEqual(1, len(self._audit(action="register"))) + self.assertEqual(token, tools._state["current_token"]) + document = self._persisted_state() + self.assertIn("current_token", document) + self.assertEqual(token, document["current_token"]) + + def test_vm_status_is_dry_run_with_credential_present(self): + tools = self._tools(PROXMOX_REFERENCE) + response = tools.vm_status("101") + self.assertIs(True, response["dry_run"]) + self.assertEqual("vm_status", response["action"]) + self.assertIs(True, response["credential_present"]) + self.assertNotIn("pve-tok", json.dumps(response)) + self.assertEqual(1, len(self._audit(action="register"))) + + +class ProxmoxReferenceTtlTests(_ReferenceCase): + extra_env = {"BROKER_TOKEN_TTL": "1"} + + def test_expired_subsequent_call_reregisters_and_persists_new_expiry(self): + tools = self._tools(PROXMOX_REFERENCE) + self.assertTrue(tools.list_nodes()["credential_present"]) + first_expires = tools._state["token_expires_at"] + time.sleep(1.2) + response = tools.list_nodes() + self.assertIs(True, response["dry_run"]) + self.assertIs(True, response["credential_present"]) + self.assertEqual(2, len(self._audit(action="register"))) + document = self._persisted_state() + self.assertEqual(tools._state["current_token"], document["current_token"]) + self.assertEqual(tools._state["token_expires_at"], document["token_expires_at"]) + self.assertNotEqual(first_expires, document["token_expires_at"]) + + +class MinimalShowcaseTests(_ReferenceCase): + def test_demo_authenticates_then_reads_secret(self): + tools = self._tools(MINIMAL_SHOWCASE) + response = tools.demo() + self.assertEqual( + {"registered": True, "secret_read": True, + "path": "coreci/proxmox", "note": "auth first, then use"}, + response, + ) + self.assertNotIn("pve-tok", json.dumps(response)) + self.assertEqual(1, len(self._audit(action="register"))) + self.assertEqual([], self._audit(result="denied")) + + +class ReferenceHygieneTests(unittest.TestCase): + def test_sources_carry_no_secret_literals(self): + for name in REFERENCE_TITLES: + source = _reference_source(name) + self.assertNotRegex(source, r"(?im)^\s*(password|pepper)\s*=\s*[\"']", name) + for value in PROXMOX_SECRET.values(): + self.assertNotIn(value, source, name) + + def test_sources_use_no_direct_network_or_infra_clients(self): + for name in REFERENCE_TITLES: + source = _reference_source(name) + for marker in FORBIDDEN_SOURCE_MARKERS: + self.assertNotIn(marker, source, "%s: %s" % (name, marker)) + + def test_imports_limited_to_client_module_and_stdlib(self): + for name in REFERENCE_TITLES: + tree = ast.parse(_reference_source(name)) + modules = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + modules.add(node.module or "") + for module in modules: + self.assertIn(module, ALLOWED_REFERENCE_IMPORTS, "%s: %s" % (name, module)) + + def test_owui_frontmatter_docstrings(self): + for name, title in REFERENCE_TITLES.items(): + doc = ast.get_docstring(ast.parse(_reference_source(name))) + self.assertTrue(doc.startswith("title: " + title), name) + for line in ("author: coreci", "description: ", "version: 0.1.0", + "requirements: []", "licence: MIT"): + self.assertIn(line, doc, "%s: %s" % (name, line)) + + def test_tools_surface_valves_state_and_docstring_discipline(self): + for name in REFERENCE_TITLES: + module = _load_reference(name) + tools = module.Tools() + self.assertIsInstance(tools.valves, module.Tools.Valves) + for attr in ("base_url", "agent_id", "app_uuid", "user_id"): + self.assertEqual("", getattr(module.Tools.Valves, attr), attr) + self.assertEqual("", getattr(tools.valves, attr), attr) + self.assertEqual({}, tools._state) + public = [] + for node in ast.walk(ast.parse(_reference_source(name))): + if not isinstance(node, ast.FunctionDef): + continue + doc = ast.get_docstring(node) + if node.name.startswith("_"): + self.assertIsNone(doc, "%s.%s" % (name, node.name)) + else: + self.assertIsNotNone(doc, "%s.%s" % (name, node.name)) + public.append(node.name) + self.assertTrue(public, name) + + def test_no_comments_in_sources(self): + for name in REFERENCE_TITLES: + for number, line in enumerate(_reference_source(name).splitlines(), 1): + self.assertFalse(line.lstrip().startswith("#"), "%s:%d" % (name, number)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_register.py b/tests/test_register.py new file mode 100644 index 0000000..7c35194 --- /dev/null +++ b/tests/test_register.py @@ -0,0 +1,347 @@ +import base64 +import contextlib +import datetime +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +WHOAMI_PATH = "/api/broker/auth/whoami" +DEFAULT_TTL = 86400 + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +def _seed_direct(broker, app_uuid, user_id): + broker.store.add_row("app_registrations", app_uuid, None) + broker.store.add_row("user_allowlist", user_id, None) + + +class AgentRegisterTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed(self.app_uuid, self.user_id) + + def _request(self, method, path, payload=None, headers=None, base=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + (base or self.base) + path, data=data, method=method + ) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + def _seed(self, app_uuid, user_id): + status, _ = self._request("POST", APPS_PATH, {"uuid": app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self, agent_id="agent-1", tags=("coreci",), app_uuid=None, + user_id=None, headers=None, payload=None): + if payload is None: + payload = { + "agent_id": agent_id, + "app_registration_uuid": app_uuid or self.app_uuid, + "user_id": user_id or self.user_id, + "tags": list(tags), + } + return self._request("POST", REGISTER_PATH, payload, headers) + + def _agent_row(self, agent_id="agent-1", app_uuid=None): + return self.broker.store.get_agent(agent_id, app_uuid or self.app_uuid) + + def _audit(self, action=None, result=None): + with open(self.env["BROKER_AUDIT_LOG_PATH"], "r", encoding="utf-8") as handle: + entries = [json.loads(raw) for raw in handle if raw.strip()] + return [ + entry for entry in entries + if (action is None or entry["action"] == action) + and (result is None or entry["result"] == result) + ] + + def _pepper(self): + return self.broker.store.get_config("BROKER_INSTANCE_PEPPER") + + def _app_header(self, token, app_uuid=None): + return {"Authorization": "App %s %s" % (app_uuid or self.app_uuid, token)} + + def _ttl_delta_seconds(self, body): + expires = datetime.datetime.fromisoformat(body["token_expires_at"]) + now = datetime.datetime.now(datetime.timezone.utc) + return (expires - now).total_seconds() + + def test_gate_matrix_unknown_and_revoked_uuid_and_user(self): + unknown_uuid = str(uuidlib.uuid4()) + status, body = self._register(app_uuid=unknown_uuid) + self.assertEqual(403, status) + self.assertEqual(403, body["error"]) + self.assertTrue(body["message"]) + self._request("DELETE", APPS_PATH + "/" + self.app_uuid, headers=self.admin) + status, body = self._register() + self.assertEqual(403, status) + self._request("POST", APPS_PATH + "/" + self.app_uuid + "/reinstate", headers=self.admin) + status, body = self._register(user_id="stranger") + self.assertEqual(403, status) + self._request("DELETE", USERS_PATH + "/" + self.user_id, headers=self.admin) + status, body = self._register() + self.assertEqual(403, status) + denied = self._audit("register", "denied") + self.assertEqual(4, len(denied)) + for entry in denied: + self.assertEqual(REGISTER_PATH, entry["path"]) + self.assertEqual("agent-1", entry["agent_id"]) + self.assertEqual(unknown_uuid, denied[0]["app_registration_uuid"]) + self.assertEqual(self.user_id, denied[0]["user_id"]) + self.assertEqual(self.app_uuid, denied[1]["app_registration_uuid"]) + self.assertIsNone(self._agent_row()) + self._request("POST", USERS_PATH + "/" + self.user_id + "/reinstate", headers=self.admin) + status, body = self._register() + self.assertEqual(201, status) + self.assertTrue(body["created"]) + self.assertIsNotNone(self._agent_row()) + + def test_malformed_bodies_are_400(self): + base = {"agent_id": "a", "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": ["x"]} + variants = ( + {}, {"agent_id": "a"}, {"agent_id": "a", "app_registration_uuid": self.app_uuid}, + dict(base, agent_id=None), dict(base, agent_id=123), dict(base, agent_id=""), + dict(base, app_registration_uuid=None), dict(base, app_registration_uuid=[]), + dict(base, user_id=7), dict(base, user_id=""), dict(base, tags=None), + dict(base, tags="x"), dict(base, tags=["a", 1]), dict(base, tags=[["nested"]]), + dict(base, tags={"a": 1}), [1, 2, 3], "nope", + ) + for payload in variants: + status, body = self._request("POST", REGISTER_PATH, payload) + self.assertEqual(400, status, repr(payload)) + self.assertEqual(400, body["error"]) + self.assertTrue(body["message"]) + self.assertIsNone(self._agent_row()) + self.assertEqual([], self._audit("register")) + + def test_first_register_creates_row_pepper_and_default_ttl(self): + status, body = self._register(agent_id="px-1", tags=("proxmox", "readonly")) + self.assertEqual(201, status) + self.assertTrue(body["created"]) + self.assertEqual(self._pepper(), body["pepper"]) + expected = coreci_broker.derive_token( + self._pepper(), self.app_uuid, self.user_id, "px-1", ["proxmox", "readonly"] + ) + self.assertEqual(expected, body["token"]) + self.assertEqual(44, len(body["token"])) + self.assertTrue(body["token"].isascii()) + delta = self._ttl_delta_seconds(body) + self.assertGreater(delta, DEFAULT_TTL - 300) + self.assertLess(delta, DEFAULT_TTL + 300) + row = self._agent_row("px-1") + self.assertNotEqual(body["token"], row[4]) + self.assertTrue(coreci_broker.verify_token(body["token"], row[4])) + self.assertEqual(self.user_id, row[2]) + self.assertEqual(["proxmox", "readonly"], json.loads(row[3])) + self.assertEqual(body["token_expires_at"], row[5]) + self.assertTrue(row[6]) + self.assertTrue(row[7]) + entries = self._audit("register", "ok") + self.assertEqual(1, len(entries)) + entry = entries[0] + self.assertEqual(self.user_id, entry["user_id"]) + self.assertEqual(self.app_uuid, entry["app_registration_uuid"]) + self.assertEqual("px-1", entry["agent_id"]) + self.assertEqual(REGISTER_PATH, entry["path"]) + + def test_reregister_unchanged_reissues_same_token_without_rehash(self): + _, first = self._register() + stored_hash = self._agent_row()[4] + seen_at = self._agent_row()[7] + status, second = self._register() + self.assertEqual(200, status) + self.assertFalse(second["created"]) + self.assertEqual(first["token"], second["token"]) + self.assertEqual(first["token_expires_at"], second["token_expires_at"]) + row = self._agent_row() + self.assertEqual(stored_hash, row[4]) + self.assertGreaterEqual(row[7], seen_at) + self.assertTrue(coreci_broker.verify_token(first["token"], row[4])) + self.assertEqual(2, len(self._audit("register", "ok"))) + + def test_reregister_pepper_rule_tokenless_vs_valid_app_token(self): + _, first = self._register() + status, tokenless = self._register() + self.assertEqual(200, status) + self.assertEqual(first["token"], tokenless["token"]) + self.assertIn("pepper", tokenless) + status, with_token = self._register(headers=self._app_header(first["token"])) + self.assertEqual(200, status) + self.assertNotIn("pepper", with_token) + self.assertEqual(first["token"], with_token["token"]) + status, bad_token = self._register( + headers={"Authorization": "App %s wrong-token" % self.app_uuid} + ) + self.assertEqual(200, status) + self.assertIn("pepper", bad_token) + + def test_changed_tags_overwrite_hash_token_and_expiry(self): + _, first = self._register(tags=("red",)) + status, second = self._register(tags=("red", "blue")) + self.assertEqual(200, status) + self.assertFalse(second["created"]) + self.assertNotEqual(first["token"], second["token"]) + row = self._agent_row() + self.assertFalse(coreci_broker.verify_token(first["token"], row[4])) + self.assertTrue(coreci_broker.verify_token(second["token"], row[4])) + self.assertEqual(["blue", "red"], json.loads(row[3])) + self.assertEqual(second["token_expires_at"], row[5]) + + def test_changed_user_overwrites_row_identity(self): + self._request("POST", USERS_PATH, {"user_id": "coo"}, self.admin) + _, first = self._register() + status, second = self._register(user_id="coo") + self.assertEqual(200, status) + self.assertNotEqual(first["token"], second["token"]) + row = self._agent_row() + self.assertEqual("coo", row[2]) + self.assertFalse(coreci_broker.verify_token(first["token"], row[4])) + self.assertTrue(coreci_broker.verify_token(second["token"], row[4])) + + def test_expired_row_reregister_rehashes_and_extends(self): + _, first = self._register() + row = self._agent_row() + old_hash, first_registered = row[4], row[6] + past = (datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(seconds=60)).isoformat() + self.broker.store.conn.execute( + "UPDATE agent_registrations SET token_expires_at = ? WHERE agent_id = ?", + (past, "agent-1"), + ) + self.broker.store.conn.commit() + status, second = self._register() + self.assertEqual(200, status) + self.assertFalse(second["created"]) + self.assertEqual(first["token"], second["token"]) + delta = self._ttl_delta_seconds(second) + self.assertGreater(delta, DEFAULT_TTL - 300) + row = self._agent_row() + self.assertNotEqual(old_hash, row[4]) + self.assertEqual(first_registered, row[6]) + self.assertTrue(coreci_broker.verify_token(first["token"], row[4])) + self.assertEqual(second["token_expires_at"], row[5]) + self.assertIn("pepper", second) + + def test_revoked_app_blocks_new_registrations_rows_untouched(self): + _, first = self._register() + row_before = self._agent_row() + self._request("DELETE", APPS_PATH + "/" + self.app_uuid, headers=self.admin) + status, body = self._register(agent_id="fresh-agent") + self.assertEqual(403, status) + status, body = self._register() + self.assertEqual(403, status) + self.assertEqual(2, len(self._audit("register", "denied"))) + row_after = self._agent_row() + self.assertEqual(row_before[4], row_after[4]) + self.assertEqual(row_before[5], row_after[5]) + self.assertTrue(coreci_broker.verify_token(first["token"], row_after[4])) + + def test_token_ttl_override_honored(self): + env = dict(self.env) + env["BROKER_DB_PATH"] = os.path.join(self.tmp, "ttl.db") + env["BROKER_AUDIT_LOG_PATH"] = os.path.join(self.tmp, "ttl.audit.log") + env["BROKER_TOKEN_TTL"] = "60" + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(dict(env)) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + _seed_direct(broker, self.app_uuid, self.user_id) + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(server.shutdown) + self.addCleanup(server.server_close) + base = "http://127.0.0.1:%d" % server.server_address[1] + payload = {"agent_id": "ttl-agent", "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": []} + status, body = self._request("POST", REGISTER_PATH, payload, base=base) + self.assertEqual(201, status) + delta = self._ttl_delta_seconds(body) + self.assertGreater(delta, 30) + self.assertLess(delta, 90) + status, body = self._request("POST", REGISTER_PATH, payload, base=base) + self.assertEqual(200, status) + delta = self._ttl_delta_seconds(body) + self.assertGreater(delta, 30) + self.assertLess(delta, 90) + + def test_user_allowlist_rejects_slash_backslash_and_control_chars(self): + for bad in ("a/b", "a\\b", "a\nb", "a\x00b", "a\x1fb", "a\x7fb"): + status, body = self._request("POST", USERS_PATH, {"user_id": bad}, self.admin) + self.assertEqual(400, status, repr(bad)) + self.assertEqual({"error": 400, "message": "invalid user_id"}, body) + status, body = self._request("GET", USERS_PATH, headers=self.admin) + self.assertEqual([self.user_id], [row["user_id"] for row in body["allowlist"]]) + + def test_bearer_scheme_with_valid_creds_is_401(self): + header = "Bearer " + base64.b64encode( + ("%s:%s" % (ADMIN_USER, ADMIN_PASS)).encode("utf-8") + ).decode("ascii") + status, body = self._request("GET", WHOAMI_PATH, headers={"Authorization": header}) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + + def test_basic_without_colon_is_401(self): + header = "Basic " + base64.b64encode(b"just-a-username").decode("ascii") + status, body = self._request("GET", WHOAMI_PATH, headers={"Authorization": header}) + self.assertEqual(401, status) + self.assertEqual(401, body["error"]) + + def test_basic_garbage_rejected_lowercase_scheme_accepted(self): + status, body = self._request( + "GET", WHOAMI_PATH, headers={"Authorization": "Basic !!!not-base64!!!"} + ) + self.assertEqual(401, status) + header = _basic_header(ADMIN_USER, ADMIN_PASS).replace("Basic", "basic") + status, body = self._request("GET", WHOAMI_PATH, headers={"Authorization": header}) + self.assertEqual(200, status) + self.assertEqual(ADMIN_USER, body["username"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_router.py b/tests/test_router.py new file mode 100644 index 0000000..5571c04 --- /dev/null +++ b/tests/test_router.py @@ -0,0 +1,242 @@ +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +from http.server import ThreadingHTTPServer + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + + +def _echo(request): + request.send_json( + 200, + {"body": request.body, "query": request.query, "params": request.path_params}, + ) + + +def _unauthorized(request): + request.send_error_json(401, "unauthorized") + + +def _forbidden(request): + request.send_error_json(403, "forbidden") + + +def _conflict(request): + request.send_error_json(409, "conflict") + + +def _explode(request): + raise RuntimeError("boom") + + +class RouterTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.ui_root = os.path.join(self.tmp, "ui") + self.static_root = os.path.join(self.tmp, "static") + os.makedirs(self.ui_root) + os.makedirs(self.static_root) + env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_UI_ROOT": self.ui_root, + "BROKER_STATIC_ROOT": self.static_root, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(env) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.broker.router.add("POST", "/test/echo/", _echo) + self.broker.router.add("GET", "/test/401", _unauthorized) + self.broker.router.add("GET", "/test/403", _forbidden) + self.broker.router.add("GET", "/test/409", _conflict) + self.broker.router.add("GET", "/test/500", _explode) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + + def _request(self, method, path, data=None): + request = urllib.request.Request(self.base + path, data=data, method=method) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, response.read(), response.headers + except urllib.error.HTTPError as error: + return error.code, error.read(), error.headers + + def _request_json(self, method, path, payload=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + status, body, headers = self._request(method, path, data=data) + return status, json.loads(body.decode("utf-8")), headers + + def _write(self, directory, name, content): + with open(os.path.join(directory, name), "w", encoding="utf-8") as handle: + handle.write(content) + + def test_serve_returns_started_threading_daemon_server(self): + self.assertIsInstance(self.server, ThreadingHTTPServer) + self.assertTrue(self.server.daemon_threads) + self.assertEqual(self.broker, self.server.broker) + + def test_healthz(self): + status, payload, headers = self._request_json("GET", "/healthz") + self.assertEqual(200, status) + self.assertEqual({"ok": True}, payload) + self.assertEqual("application/json; charset=utf-8", headers.get("Content-Type")) + + def test_error_contract_for_every_code(self): + cases = [ + ("GET", "/test/401", None, 401), + ("GET", "/test/403", None, 403), + ("GET", "/test/409", None, 409), + ("GET", "/test/500", None, 500), + ("GET", "/no/such/route", None, 404), + ("POST", "/test/echo/alice", b"{not json", 400), + ] + for method, path, data, code in cases: + with self.subTest(code=code): + status, body, headers = self._request(method, path, data=data) + self.assertEqual(code, status) + self.assertEqual( + "application/json; charset=utf-8", headers.get("Content-Type") + ) + payload = json.loads(body.decode("utf-8")) + self.assertEqual({"error", "message"}, set(payload)) + self.assertEqual(code, payload["error"]) + self.assertIsInstance(payload["message"], str) + self.assertTrue(payload["message"]) + + def test_json_body_query_and_params_roundtrip(self): + payload = {"k": [1, 2, {"z": True}], "s": "værdi", "n": None} + status, body, _ = self._request_json( + "POST", "/test/echo/alice?x=1&tag=a&tag=b", payload + ) + self.assertEqual(200, status) + self.assertEqual( + { + "body": payload, + "query": {"x": "1", "tag": "a"}, + "params": {"name": "alice"}, + }, + body, + ) + + def test_post_with_empty_body_yields_null_body(self): + status, body, _ = self._request_json("POST", "/test/echo/alice") + self.assertEqual(200, status) + self.assertIsNone(body["body"]) + + def test_unknown_route_is_404_even_with_valid_json_body(self): + status, body, _ = self._request_json("POST", "/api/broker/nope", {"a": 1}) + self.assertEqual(404, status) + self.assertEqual({"error", "message"}, set(body)) + self.assertEqual(404, body["error"]) + + def test_method_mismatch_is_404(self): + status, body, _ = self._request_json("DELETE", "/healthz") + self.assertEqual(404, status) + self.assertEqual(404, body["error"]) + + def test_static_content_types(self): + fixtures = { + "index.html": ("ok", "text/html; charset=utf-8"), + "app.css": ("body{margin:0}", "text/css; charset=utf-8"), + "app.js": ("console.log(1)", "application/javascript; charset=utf-8"), + "mod.py": ("import os\n", "text/x-python; charset=utf-8"), + "notes.txt": ("hello world", "text/plain; charset=utf-8"), + "data.json": ("{\"k\": 1}", "application/json; charset=utf-8"), + } + for name, (content, _) in fixtures.items(): + self._write(self.ui_root, name, content) + for name, (content, content_type) in fixtures.items(): + with self.subTest(name=name): + status, body, headers = self._request("GET", "/ui/" + name) + self.assertEqual(200, status) + self.assertEqual(content.encode("utf-8"), body) + self.assertEqual(content_type, headers.get("Content-Type")) + + def test_static_serves_client_module(self): + self._write( + self.static_root, "coreci_broker_client.py", "import urllib.request\n" + ) + status, body, headers = self._request("GET", "/static/coreci_broker_client.py") + self.assertEqual(200, status) + self.assertEqual(b"import urllib.request\n", body) + self.assertEqual("text/x-python; charset=utf-8", headers.get("Content-Type")) + + def test_ui_index_and_extensionless_page(self): + self._write(self.ui_root, "index.html", "index") + self._write(self.ui_root, "login.html", "login") + status, body, _ = self._request("GET", "/ui/") + self.assertEqual(200, status) + self.assertEqual(b"index", body) + status, body, _ = self._request("GET", "/ui/login") + self.assertEqual(200, status) + self.assertEqual(b"login", body) + + def test_static_missing_file_is_404_contract(self): + status, body, headers = self._request("GET", "/ui/absent.html") + self.assertEqual(404, status) + self.assertEqual("application/json; charset=utf-8", headers.get("Content-Type")) + self.assertEqual( + {"error": 404, "message": "not found"}, json.loads(body.decode("utf-8")) + ) + + def test_static_prefix_rejects_non_get(self): + status, body, _ = self._request("POST", "/ui/index.html", data=b"{}") + self.assertEqual(404, status) + self.assertEqual(404, json.loads(body.decode("utf-8"))["error"]) + + def test_ui_root_attribute_is_configurable(self): + alt = os.path.join(self.tmp, "ui_alt") + os.makedirs(alt) + self._write(alt, "x.html", "alt root") + self.broker.ui_root = alt + status, body, _ = self._request("GET", "/ui/x.html") + self.assertEqual(200, status) + self.assertEqual(b"alt root", body) + + def test_traversal_attempts_rejected(self): + attempts = [ + "/ui/../etc/passwd", + "/static/%2e%2e/etc/passwd", + "/static/..%2fetc%2fpasswd", + "/static/..%5cetc%5cpasswd", + "/ui/..%5c..%5cetc%5cpasswd", + "/static/%00etc/passwd", + "/static//etc/passwd", + "/ui/./.././../etc/passwd", + ] + for path in attempts: + with self.subTest(path=path): + status, body, _ = self._request("GET", path) + self.assertIn(status, (400, 404)) + payload = json.loads(body.decode("utf-8")) + self.assertEqual({"error", "message"}, set(payload)) + self.assertEqual(status, payload["error"]) + + def test_symlink_escape_rejected(self): + outside = os.path.join(self.tmp, "outside") + os.makedirs(outside) + self._write(outside, "secret.txt", "top secret") + os.symlink(outside, os.path.join(self.ui_root, "alias")) + status, body, _ = self._request("GET", "/ui/alias/secret.txt") + self.assertEqual(404, status) + self.assertEqual( + {"error": 404, "message": "not found"}, json.loads(body.decode("utf-8")) + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_secrets.py b/tests/test_secrets.py new file mode 100644 index 0000000..02b3351 --- /dev/null +++ b/tests/test_secrets.py @@ -0,0 +1,233 @@ +import base64 +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.parse +import urllib.request +import uuid as uuidlib + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +AUDIT_PATH = "/api/broker/audit" +SECRETS_PREFIX = "/api/broker/secrets/" +AUTH_401_BODY = {"error": 401, "message": "invalid app credentials"} +PROXMOX_SECRET = {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"} +GITEA_SECRET = {"url": "https://git.coreci.dev", "token": "git-tok"} + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class _SecretsCase(unittest.TestCase): + creds_json = json.dumps( + {"coreci/proxmox": PROXMOX_SECRET, "coreci/gitea": GITEA_SECRET} + ) + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.creds_path = os.path.join(self.tmp, "creds.json") + with open(self.creds_path, "w", encoding="utf-8") as handle: + handle.write(self.creds_json) + env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + "BROKER_CREDS_SOURCE": "file", + "BROKER_CREDS_FILE": self.creds_path, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(env) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed() + self.registered = self._register("proxmox", ["coreci"]) + self.app = {"Authorization": "App %s %s" % (self.app_uuid, self.registered["token"])} + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8") + return error.code, json.loads(raw) if raw else None + + def _seed(self): + status, _ = self._request("POST", APPS_PATH, {"uuid": self.app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": self.user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self, agent_id, tags): + payload = {"agent_id": agent_id, "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": list(tags)} + status, body = self._request("POST", REGISTER_PATH, payload) + self.assertEqual(201, status) + return body + + def _secret(self, suffix, headers=None): + return self._request("GET", SECRETS_PREFIX + suffix, + headers=self.app if headers is None else headers) + + def _audit(self, action): + query = urllib.parse.urlencode({"action": action, "limit": "1000"}) + status, body = self._request("GET", AUDIT_PATH + "?" + query, headers=self.admin) + self.assertEqual(200, status) + return body["entries"] + + def _read_entries(self, result): + return [entry for entry in self._audit("read") if entry["result"] == result] + + def _assert_identity(self, entry, path, agent_id="proxmox"): + self.assertEqual("read", entry["action"]) + self.assertEqual(path, entry["path"]) + self.assertEqual(self.user_id, entry["user_id"]) + self.assertEqual(self.app_uuid, entry["app_registration_uuid"]) + self.assertEqual(agent_id, entry["agent_id"]) + + def _ghost_headers(self, agent_id): + ghost = self._register(agent_id, ["coreci"]) + return {"Authorization": "App %s %s" % (self.app_uuid, ghost["token"])} + + +class SecretsReadTests(_SecretsCase): + def test_valid_read_returns_document_and_ok_audit(self): + status, body = self._secret("coreci/proxmox") + self.assertEqual(200, status) + self.assertEqual("coreci/proxmox", body["path"]) + self.assertEqual(PROXMOX_SECRET, body["data"]) + self.assertRegex(body["version"], r"^[0-9a-f]{8}$") + self.assertIn("updated_at", body) + entries = self._read_entries("ok") + self.assertEqual(1, len(entries)) + self._assert_identity(entries[0], "coreci/proxmox") + + def test_encoded_slash_path_resolves_to_same_secret(self): + status, body = self._secret("coreci%2Fproxmox") + self.assertEqual(200, status) + self.assertEqual("coreci/proxmox", body["path"]) + self.assertEqual(PROXMOX_SECRET, body["data"]) + self.assertEqual(1, len(self._read_entries("ok"))) + + def test_missing_app_header_is_401(self): + status, body = self._secret("coreci/proxmox", headers={}) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + self.assertEqual([], self._read_entries("ok")) + self.assertEqual([], self._read_entries("denied")) + + def test_tampered_token_is_401_with_denied_audit(self): + token = self.registered["token"] + twisted = ("x" if token[0] != "x" else "y") + token[1:] + status, body = self._secret("coreci/proxmox", headers={ + "Authorization": "App %s %s" % (self.app_uuid, twisted)}) + self.assertEqual(401, status) + self.assertEqual(AUTH_401_BODY, body) + entries = self._audit("denied") + self.assertEqual(1, len(entries)) + self.assertEqual("denied", entries[0]["action"]) + self.assertEqual("denied", entries[0]["result"]) + self.assertEqual(self.app_uuid, entries[0]["app_registration_uuid"]) + self.assertIsNone(entries[0]["user_id"]) + self.assertIsNone(entries[0]["agent_id"]) + + def test_cross_path_read_is_403_with_denied_audit(self): + status, body = self._secret("coreci/gitea") + self.assertEqual(403, status) + self.assertEqual(403, body["error"]) + self.assertTrue(body["message"]) + entries = self._read_entries("denied") + self.assertEqual(1, len(entries)) + self._assert_identity(entries[0], "coreci/gitea") + + def test_out_of_scope_paths_are_403(self): + suffixes = ("coreci/proxmox/extra", "other/proxmox", "coreci", "CORECI/proxmox") + for suffix in suffixes: + with self.subTest(suffix=suffix): + status, body = self._secret(suffix) + self.assertEqual(403, status) + self.assertEqual(403, body["error"]) + self.assertEqual(len(suffixes), len(self._read_entries("denied"))) + + def test_absent_secret_is_404_with_error_audit(self): + status, body = self._secret("coreci/nonexistent", + headers=self._ghost_headers("nonexistent")) + self.assertEqual(404, status) + self.assertEqual({"error": 404, "message": "not found"}, body) + entries = self._read_entries("error") + self.assertEqual(1, len(entries)) + self._assert_identity(entries[0], "coreci/nonexistent", "nonexistent") + + def test_traversal_payloads_never_reach_source(self): + original = self.broker.creds_source.get_secret + self.addCleanup(setattr, self.broker.creds_source, "get_secret", original) + + def detonate(path): + raise AssertionError("creds source reached for %r" % path) + + self.broker.creds_source.get_secret = detonate + attempts = ["%2e%2e/", "..%2f", "coreci%5cproxmox", "coreci%00proxmox", + "//coreci/proxmox", "%2e%2e/coreci/proxmox", "coreci/..%2fproxmox"] + for suffix in attempts: + with self.subTest(suffix=suffix): + status, body = self._secret(suffix) + self.assertEqual(403, status) + self.assertEqual(403, body["error"]) + self.assertEqual(len(attempts), len(self._read_entries("denied"))) + + def test_audit_completeness_action_read_covers_all_results(self): + self._secret("coreci/proxmox") + self._secret("coreci/gitea") + self._secret("coreci/nonexistent", headers=self._ghost_headers("nonexistent")) + entries = self._audit("read") + self.assertEqual(3, len(entries)) + by_result = {entry["result"]: entry for entry in entries} + self.assertEqual({"ok", "denied", "error"}, set(by_result)) + self._assert_identity(by_result["ok"], "coreci/proxmox") + self._assert_identity(by_result["denied"], "coreci/gitea") + self._assert_identity(by_result["error"], "coreci/nonexistent", "nonexistent") + + +class SecretsSourceErrorTests(_SecretsCase): + creds_json = "{not-json" + + def test_source_failure_is_500_contract_with_error_audit(self): + status, body = self._secret("coreci/proxmox") + self.assertEqual(500, status) + self.assertEqual({"error": 500, "message": "creds source unavailable"}, body) + entries = self._read_entries("error") + self.assertEqual(1, len(entries)) + self._assert_identity(entries[0], "coreci/proxmox") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_state.py b/tests/test_state.py new file mode 100644 index 0000000..36cdb9c --- /dev/null +++ b/tests/test_state.py @@ -0,0 +1,318 @@ +import base64 +import contextlib +import io +import json +import os +import sys +import tempfile +import threading +import unittest +import urllib.error +import urllib.request +import uuid as uuidlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +ADMIN_USER = "admin" +ADMIN_PASS = "correct-horse-battery-staple" +REGISTER_PATH = "/api/broker/agents/register" +APPS_PATH = "/api/broker/app_registrations" +USERS_PATH = "/api/broker/user_allowlist" +STATE_PATH = "/api/broker/agents/%s/state" +STUB_PREFIX = "/api/v1/broker-agent-state/" + + +def _basic_header(username, password): + raw = "%s:%s" % (username, password) + return "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") + + +class _StubOwuiHandler(BaseHTTPRequestHandler): + documents = None + records = None + + def log_message(self, format, *args): + pass + + def do_GET(self): + agent_id = self.path[len(STUB_PREFIX):] + self.records.append(("GET", self.path, self.headers.get("Authorization"), None)) + if agent_id not in self.documents: + self._send(404, {"error": 404}) + return + self._send(200, self.documents[agent_id]) + + def do_PUT(self): + agent_id = self.path[len(STUB_PREFIX):] + length = int(self.headers.get("Content-Length") or "0") + raw = self.rfile.read(length) + self.records.append(("PUT", self.path, self.headers.get("Authorization"), raw)) + self.documents[agent_id] = json.loads(raw.decode("utf-8")) + self._send(200, {"ok": True}) + + def _send(self, status, payload): + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +class StubOwuiServer: + def __init__(self): + self.documents, self.records = {}, [] + handler = type("BoundStubHandler", (_StubOwuiHandler,), + {"documents": self.documents, "records": self.records}) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + @property + def base_url(self): + return "http://127.0.0.1:%d" % self.server.server_address[1] + + def close(self): + self.server.shutdown() + self.server.server_close() + + +class _StateCase(unittest.TestCase): + backend = "internal" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_ADMIN_USER": ADMIN_USER, + "BROKER_ADMIN_PASS": ADMIN_PASS, + "BROKER_AGENT_STATE_BACKEND": self.backend, + } + self.env.update(getattr(self, "extra_env", {})) + self._start_broker() + self.admin = {"Authorization": _basic_header(ADMIN_USER, ADMIN_PASS)} + self.app_uuid = str(uuidlib.uuid4()) + self.user_id = "ceo" + self._seed(self.app_uuid, self.user_id) + self.registered = self._register("agent-1", ["b", "a"]) + self.app = {"Authorization": "App %s %s" % (self.app_uuid, self.registered["token"])} + + def _start_broker(self): + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + + def _request(self, method, path, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.base + path, data=data, method=method) + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8") + return error.code, json.loads(raw) if raw else None + + def _state(self, method, agent_id, payload=None, headers=None): + return self._request(method, STATE_PATH % agent_id, payload, + self.app if headers is None else headers) + + def _seed(self, app_uuid, user_id): + status, _ = self._request("POST", APPS_PATH, {"uuid": app_uuid}, self.admin) + self.assertEqual(201, status) + status, _ = self._request("POST", USERS_PATH, {"user_id": user_id}, self.admin) + self.assertEqual(201, status) + + def _register(self, agent_id, tags): + payload = {"agent_id": agent_id, "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": list(tags)} + status, body = self._request("POST", REGISTER_PATH, payload) + self.assertEqual(201, status) + return body + + +class InternalStateTests(_StateCase): + def test_put_then_get_roundtrip_is_document_faithful(self): + state = {"agent_id": "agent-1", "app_registration_uuid": self.app_uuid, + "user_id": self.user_id, "tags": ["b", "a"], + "pepper": self.registered["pepper"], + "current_token": self.registered["token"], + "token_expires_at": self.registered["token_expires_at"], + "first_registered_at": "t0", "last_seen_at": "t1", + "nested": {"flags": [1, 2.5, None, True], "deep": {"k": None}}, "z": 1} + status, body = self._state("PUT", "agent-1", state) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + status, body = self._state("GET", "agent-1") + self.assertEqual(200, status) + self.assertEqual(state, body) + self.assertEqual(list(state), list(body)) + self.assertEqual(state, self.broker.store.get_agent_state("agent-1", self.app_uuid)) + + def test_state_survives_broker_restart(self): + state = {"counter": 7, "notes": ["x"], "pepper": "keep-me"} + status, _ = self._state("PUT", "agent-1", state) + self.assertEqual(200, status) + self.server.shutdown() + self.server.server_close() + self.broker.store.close() + with contextlib.redirect_stdout(io.StringIO()): + second = coreci_broker.boot(dict(self.env)) + self.addCleanup(second.audit.close) + self.addCleanup(second.store.close) + server = coreci_broker.serve("127.0.0.1", 0, second) + self.addCleanup(server.shutdown) + self.addCleanup(server.server_close) + self.base = "http://127.0.0.1:%d" % server.server_address[1] + status, body = self._state("GET", "agent-1") + self.assertEqual(200, status) + self.assertEqual(state, body) + row = second.store.get_agent("agent-1", self.app_uuid) + self.assertTrue(coreci_broker.verify_token(self.registered["token"], row[4])) + + def test_identity_mismatch_is_403_on_get_and_put(self): + second = self._register("agent-2", ["coreci"]) + for method, payload in (("GET", None), ("PUT", {"who": "agent-2"})): + status, body = self._state(method, "agent-2", payload) + self.assertEqual(403, status, method) + self.assertEqual(403, body["error"]) + self.assertTrue(body["message"]) + status, body = self._state("GET", "agent-2", headers={ + "Authorization": "App %s %s" % (self.app_uuid, second["token"])}) + self.assertEqual(404, status) + + def test_unregistered_agent_id_is_404_on_get(self): + status, body = self._state("GET", "ghost-agent") + self.assertEqual(404, status) + self.assertEqual({"error": 404, "message": "not found"}, body) + + def test_get_without_state_is_404(self): + status, body = self._state("GET", "agent-1") + self.assertEqual(404, status) + self.assertEqual({"error": 404, "message": "not found"}, body) + + def test_put_non_dict_is_400(self): + for payload in ([1, 2], "nope", 5, None): + status, body = self._state("PUT", "agent-1", payload) + self.assertEqual(400, status, repr(payload)) + self.assertEqual(400, body["error"]) + status, _ = self._state("GET", "agent-1") + self.assertEqual(404, status) + + def test_put_above_256kib_is_400(self): + status, body = self._state("PUT", "agent-1", {"pad": "x" * (300 * 1024)}) + self.assertEqual(400, status) + self.assertEqual(400, body["error"]) + status, _ = self._state("GET", "agent-1") + self.assertEqual(404, status) + + def test_put_at_exactly_256kib_is_stored(self): + pad = "x" * (256 * 1024 - len(json.dumps({"pad": ""}))) + status, body = self._state("PUT", "agent-1", {"pad": pad}) + self.assertEqual(200, status) + status, body = self._state("GET", "agent-1") + self.assertEqual(200, status) + self.assertEqual({"pad": pad}, body) + + def test_missing_app_header_is_401(self): + status, body = self._state("GET", "agent-1", headers={}) + self.assertEqual(401, status) + self.assertEqual({"error": 401, "message": "invalid app credentials"}, body) + + +class OwuiStateTests(_StateCase): + backend = "owui" + + def setUp(self): + self.stub = StubOwuiServer() + self.addCleanup(self.stub.close) + self.extra_env = { + "BROKER_OWUI_BASE_URL": self.stub.base_url, + "BROKER_OWUI_API_KEY": "test-key", + } + super().setUp() + + def test_put_then_get_roundtrip_flows_through_stub(self): + state = {"agent_id": "agent-1", "current_token": "tok", "n": 3} + status, body = self._state("PUT", "agent-1", state) + self.assertEqual(200, status) + self.assertEqual({"ok": True}, body) + self.assertEqual(state, self.stub.documents["agent-1"]) + status, body = self._state("GET", "agent-1") + self.assertEqual(200, status) + self.assertEqual(state, body) + puts = [record for record in self.stub.records if record[0] == "PUT"] + gets = [record for record in self.stub.records if record[0] == "GET"] + self.assertEqual(1, len(puts)) + self.assertEqual(1, len(gets)) + for _, path, auth, _ in self.stub.records: + self.assertEqual("Bearer test-key", auth) + self.assertEqual(STUB_PREFIX + "agent-1", puts[0][1]) + self.assertEqual(STUB_PREFIX + "agent-1", gets[0][1]) + self.assertEqual(state, json.loads(puts[0][3].decode("utf-8"))) + + def test_stub_miss_maps_to_broker_404(self): + status, body = self._state("GET", "agent-1") + self.assertEqual(404, status) + self.assertEqual({"error": 404, "message": "not found"}, body) + + def test_identity_enforced_before_backend_call(self): + self._register("agent-2", ["coreci"]) + status, body = self._state("GET", "agent-2") + self.assertEqual(403, status) + status, body = self._state("PUT", "agent-2", {"x": 1}) + self.assertEqual(403, status) + self.assertEqual([], self.stub.records) + + +class BackendToggleTests(unittest.TestCase): + def _boot(self, extra): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + env = { + "BROKER_ENV_PATH": os.path.join(tmp.name, ".env"), + "BROKER_DB_PATH": os.path.join(tmp.name, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(tmp.name, "broker.audit.log"), + } + env.update(extra) + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(env) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + return broker + + def test_default_backend_is_internal(self): + broker = self._boot({}) + self.assertIsInstance(broker.agent_state_backend, coreci_broker.InternalAgentStateBackend) + + def test_owui_backend_selected_by_config(self): + broker = self._boot({ + "BROKER_AGENT_STATE_BACKEND": "owui", + "BROKER_OWUI_BASE_URL": "http://127.0.0.1:9999/", + "BROKER_OWUI_API_KEY": "k", + }) + backend = broker.agent_state_backend + self.assertIsInstance(backend, coreci_broker.OwuiAgentStateBackend) + self.assertEqual("http://127.0.0.1:9999", backend.base_url) + self.assertEqual("k", backend.api_key) + + def test_unknown_backend_value_falls_back_to_internal(self): + broker = self._boot({"BROKER_AGENT_STATE_BACKEND": "banana"}) + self.assertIsInstance(broker.agent_state_backend, coreci_broker.InternalAgentStateBackend) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_static_disclosure.py b/tests/test_static_disclosure.py new file mode 100644 index 0000000..dcb2eea --- /dev/null +++ b/tests/test_static_disclosure.py @@ -0,0 +1,181 @@ +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + + +class StaticDisclosureGuardTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.env = { + "BROKER_ENV_PATH": os.path.join(self.tmp, ".env"), + "BROKER_DB_PATH": os.path.join(self.tmp, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(self.tmp, "broker.audit.log"), + "BROKER_UI_ROOT": self.tmp, + "BROKER_STATIC_ROOT": self.tmp, + } + with contextlib.redirect_stdout(io.StringIO()): + self.broker = coreci_broker.boot(dict(self.env)) + self.addCleanup(self.broker.audit.close) + self.addCleanup(self.broker.store.close) + with open( + os.path.join(self.tmp, "coreci_broker_client.py"), "w", encoding="utf-8" + ) as handle: + handle.write("import urllib.request\n") + with open( + os.path.join(self.tmp, ".hidden.css"), "w", encoding="utf-8" + ) as handle: + handle.write("body{display:none}") + os.symlink(self.env["BROKER_ENV_PATH"], os.path.join(self.tmp, "env_alias")) + self.server = coreci_broker.serve("127.0.0.1", 0, self.broker) + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + + def _get(self, path): + try: + with urllib.request.urlopen(self.base + path, timeout=5) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + + def test_state_files_never_served_when_roots_overlap_state_dir(self): + for path in ( + "/static/.env", + "/static/broker.db", + "/static/broker.db-journal", + "/static/broker.db-wal", + "/static/broker.audit.log", + "/ui/.env", + "/ui/broker.db", + "/ui/broker.audit.log", + ): + with self.subTest(path=path): + status, body = self._get(path) + self.assertEqual(404, status) + self.assertEqual( + {"error": 404, "message": "not found"}, json.loads(body.decode("utf-8")) + ) + + def test_symlink_alias_to_env_rejected(self): + status, _ = self._get("/static/env_alias") + self.assertEqual(404, status) + + def test_dotfiles_rejected_even_with_known_extension(self): + for path in ("/static/.hidden.css", "/ui/.hidden.css"): + with self.subTest(path=path): + status, _ = self._get(path) + self.assertEqual(404, status) + + def test_public_files_still_served(self): + status, body = self._get("/static/coreci_broker_client.py") + self.assertEqual(200, status) + self.assertEqual(b"import urllib.request\n", body) + + def test_default_config_relative_paths_do_not_disclose_state(self): + cwd = os.getcwd() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + os.chdir(tmp.name) + try: + env = {"BROKER_ENV_PATH": os.path.join(tmp.name, ".env")} + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(dict(env)) + server = coreci_broker.serve("127.0.0.1", 0, broker) + base = "http://127.0.0.1:%d" % server.server_address[1] + for path in ("/static/.env", "/static/broker.db", "/static/broker.audit.log"): + with self.subTest(path=path): + try: + with urllib.request.urlopen(base + path, timeout=5) as response: + status = response.status + except urllib.error.HTTPError as error: + status = error.code + self.assertEqual(404, status) + server.shutdown() + server.server_close() + broker.audit.close() + broker.store.close() + finally: + os.chdir(cwd) + + +class CredsFileDisclosureGuardTests(unittest.TestCase): + SECRET_JSON = json.dumps({"coreci/proxmox": {"token": "pve-tok"}}) + + def _get(self, base, path): + try: + with urllib.request.urlopen(base + path, timeout=5) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + + def _write_creds(self, path): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + handle.write(self.SECRET_JSON) + + def test_default_layout_never_serves_creds_file(self): + cwd = os.getcwd() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + creds_path = os.path.join(tmp.name, "data", "creds.json") + self._write_creds(creds_path) + os.chdir(tmp.name) + try: + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot({"BROKER_ENV_PATH": os.path.join(tmp.name, ".env")}) + server = coreci_broker.serve("127.0.0.1", 0, broker) + base = "http://127.0.0.1:%d" % server.server_address[1] + for path in ("/static/data/creds.json", "/static/data%2Fcreds.json"): + with self.subTest(path=path): + status, body = self._get(base, path) + self.assertEqual(404, status) + self.assertNotIn(b"pve-tok", body) + server.shutdown() + server.server_close() + broker.audit.close() + broker.store.close() + finally: + os.chdir(cwd) + + def test_creds_file_blocked_when_roots_overlap_creds_dir(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + creds_path = os.path.join(tmp.name, "creds.json") + self._write_creds(creds_path) + env = { + "BROKER_ENV_PATH": os.path.join(tmp.name, ".env"), + "BROKER_DB_PATH": os.path.join(tmp.name, "broker.db"), + "BROKER_AUDIT_LOG_PATH": os.path.join(tmp.name, "broker.audit.log"), + "BROKER_UI_ROOT": tmp.name, + "BROKER_STATIC_ROOT": tmp.name, + "BROKER_CREDS_FILE": creds_path, + } + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(dict(env)) + self.addCleanup(broker.audit.close) + self.addCleanup(broker.store.close) + server = coreci_broker.serve("127.0.0.1", 0, broker) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + base = "http://127.0.0.1:%d" % server.server_address[1] + for path in ("/static/creds.json", "/ui/creds.json"): + with self.subTest(path=path): + status, body = self._get(base, path) + self.assertEqual(404, status) + self.assertNotIn(b"pve-tok", body) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..dfddea2 --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,155 @@ +import base64 +import contextlib +import io +import os +import sqlite3 +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +TABLE_COLUMNS = { + "broker_config": ["key", "value"], + "app_registrations": ["uuid", "name", "status", "created_at", "revoked_at"], + "user_allowlist": ["user_id", "email", "status", "created_at", "revoked_at"], + "agent_registrations": [ + "agent_id", + "app_registration_uuid", + "user_id", + "tags", + "token_hash", + "token_expires_at", + "first_registered_at", + "last_seen_at", + ], + "admin_sessions": ["session_id", "created_at", "expires_at"], +} + + +class StoreTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.env_path = os.path.join(tmp.name, ".env") + self.db_path = os.path.join(tmp.name, "broker.db") + + def _boot(self, extra_env=None): + env = {"BROKER_ENV_PATH": self.env_path, "BROKER_DB_PATH": self.db_path} + if extra_env: + env.update(extra_env) + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + broker = coreci_broker.boot(env) + self.addCleanup(broker.store.close) + return broker, buffer.getvalue() + + def _connect(self): + conn = sqlite3.connect(self.db_path) + self.addCleanup(conn.close) + return conn + + def test_boot_creates_db_with_mode_0600(self): + broker, stdout = self._boot() + self.assertTrue(os.path.isfile(self.db_path)) + self.assertEqual(0o600, os.stat(self.db_path).st_mode & 0o777) + self.assertEqual(1, stdout.count(broker.config["BROKER_ADMIN_PASS"])) + + def test_db_mode_0600_independent_of_umask(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + env = { + "BROKER_ENV_PATH": os.path.join(tmp.name, ".env"), + "BROKER_DB_PATH": os.path.join(tmp.name, "broker.db"), + } + old_umask = os.umask(0o777) + try: + with contextlib.redirect_stdout(io.StringIO()): + broker = coreci_broker.boot(env) + self.addCleanup(broker.store.close) + self.assertEqual(0o600, os.stat(env["BROKER_DB_PATH"]).st_mode & 0o777) + self.assertEqual(0o600, os.stat(env["BROKER_ENV_PATH"]).st_mode & 0o777) + finally: + os.umask(old_umask) + + def test_all_five_tables_present_with_columns(self): + self._boot() + conn = self._connect() + tables = { + row[0] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'") + } + self.assertTrue(set(TABLE_COLUMNS).issubset(tables)) + for table, expected in TABLE_COLUMNS.items(): + actual = [row[1] for row in conn.execute("PRAGMA table_info(%s)" % table)] + self.assertEqual(expected, actual) + + def test_index_on_agent_registrations_app_registration_uuid(self): + self._boot() + conn = self._connect() + names = [ + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_registrations'" + ) + ] + matching = [] + for name in names: + columns = [row[2] for row in conn.execute("PRAGMA index_info(%s)" % name)] + if columns and columns[0] == "app_registration_uuid": + matching.append(name) + self.assertTrue(matching) + + def test_pepper_synced_into_broker_config(self): + broker, _ = self._boot() + conn = self._connect() + row = conn.execute( + "SELECT value FROM broker_config WHERE key = 'BROKER_INSTANCE_PEPPER'" + ).fetchone() + self.assertIsNotNone(row) + self.assertEqual(broker.config["BROKER_INSTANCE_PEPPER"], row[0]) + self.assertEqual(32, len(base64.b64decode(row[0]))) + + def test_pepper_env_override_synced(self): + self._boot({"BROKER_INSTANCE_PEPPER": "Zm9vYmFy"}) + conn = self._connect() + row = conn.execute( + "SELECT value FROM broker_config WHERE key = 'BROKER_INSTANCE_PEPPER'" + ).fetchone() + self.assertEqual("Zm9vYmFy", row[0]) + + def test_second_boot_silent_and_idempotent(self): + first, first_stdout = self._boot() + second, second_stdout = self._boot() + self.assertEqual(1, first_stdout.count(first.config["BROKER_ADMIN_PASS"])) + self.assertEqual("", second_stdout) + self.assertEqual( + first.config["BROKER_INSTANCE_PEPPER"], + second.config["BROKER_INSTANCE_PEPPER"], + ) + self.assertEqual( + first.config["BROKER_INSTANCE_PEPPER"], + second.store.get_config("BROKER_INSTANCE_PEPPER"), + ) + self.assertEqual(0o600, os.stat(self.db_path).st_mode & 0o777) + + def test_restart_preserves_rows(self): + broker, _ = self._boot() + broker.store.conn.execute( + "INSERT INTO app_registrations (uuid, name) VALUES (?, ?)", + ("0b6c3370-6d38-4b92-b9a2-bc2f1c9c1b0a", "proxmox-tool"), + ) + broker.store.conn.commit() + broker.store.close() + conn = self._connect() + row = conn.execute( + "SELECT name FROM app_registrations WHERE uuid = ?", + ("0b6c3370-6d38-4b92-b9a2-bc2f1c9c1b0a",), + ).fetchone() + self.assertEqual("proxmox-tool", row[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_token.py b/tests/test_token.py new file mode 100644 index 0000000..dde9c96 --- /dev/null +++ b/tests/test_token.py @@ -0,0 +1,182 @@ +import base64 +import hashlib +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) + +import coreci_broker + +PEPPER = "ZpQ1hP3rLmX7wYc2vB4nT6kS9dF5gR8j" +APP_UUID = "3f2a9c1e-77d0-4f0a-9b2e-6c1d5a8e4b3c" +USER_ID = "user-42" +AGENT_ID = "proxmox" + +B64_ALPHABET = set( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" +) + + +def _independent_token(pepper, app_uuid, user_id, agent_id, tags): + payload = "{}:{}:{}:{}:{}".format( + pepper, + app_uuid, + user_id, + agent_id, + ",".join(sorted(tags)), + ) + digest = hashlib.sha256(payload.encode("utf-8")).digest() + return base64.b64encode(digest).decode("ascii") + + +class FormulaTests(unittest.TestCase): + def test_matches_independent_computation(self): + token = coreci_broker.derive_token( + PEPPER, APP_UUID, USER_ID, AGENT_ID, ["infra", "proxmox"] + ) + expected = _independent_token( + PEPPER, APP_UUID, USER_ID, AGENT_ID, ["infra", "proxmox"] + ) + self.assertEqual(expected, token) + + def test_exact_spec_formula_byte_for_byte(self): + expected = base64.b64encode( + hashlib.sha256( + "{}:{}:{}:{}:{}".format( + PEPPER, APP_UUID, USER_ID, AGENT_ID, "a,b" + ).encode("utf-8") + ).digest() + ).decode("ascii") + actual = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a", "b"]) + self.assertEqual(expected, actual) + + def test_empty_tags_match_independent_computation(self): + token = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, []) + self.assertEqual( + _independent_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, []), + token, + ) + + def test_deterministic_for_same_inputs(self): + first = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + second = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertEqual(first, second) + + +class ShapeTests(unittest.TestCase): + def test_token_is_exactly_44_chars(self): + token = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertEqual(44, len(token)) + + def test_token_is_pure_ascii(self): + token = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertTrue(token.isascii()) + token.encode("ascii") + + def test_token_carries_base64_padding_char(self): + token = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertTrue(token.endswith("=")) + self.assertEqual(1, token.count("=")) + + def test_token_uses_only_base64_alphabet(self): + token = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertTrue(set(token) <= B64_ALPHABET) + + def test_token_decodes_to_32_byte_sha256_digest(self): + token = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertEqual(32, len(base64.b64decode(token))) + + def test_token_has_no_prefix(self): + token = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + for prefix in ("App", "Bearer", "scrypt$", "$2", "token", "tk_", "coreci"): + with self.subTest(prefix=prefix): + self.assertFalse(token.startswith(prefix)) + self.assertNotIn(" ", token) + self.assertNotIn(":", token) + self.assertNotIn("$", token) + + +class TagNormalizationTests(unittest.TestCase): + def test_tag_order_does_not_change_token(self): + first = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["b", "a"]) + second = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a", "b"]) + self.assertEqual(first, second) + + def test_accepts_tuple_and_set_iterables(self): + expected = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a", "b"]) + self.assertEqual( + expected, + coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ("a", "b")), + ) + self.assertEqual( + expected, + coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, {"a", "b"}), + ) + + +class InputSensitivityTests(unittest.TestCase): + def test_pepper_changes_token(self): + base = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + other = coreci_broker.derive_token("other-pepper", APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertNotEqual(base, other) + + def test_app_uuid_changes_token(self): + base = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + other = coreci_broker.derive_token( + PEPPER, "00000000-0000-0000-0000-000000000000", USER_ID, AGENT_ID, ["a"] + ) + self.assertNotEqual(base, other) + + def test_user_id_changes_token(self): + base = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + other = coreci_broker.derive_token(PEPPER, APP_UUID, "user-99", AGENT_ID, ["a"]) + self.assertNotEqual(base, other) + + def test_agent_id_changes_token(self): + base = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + other = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, "gitea", ["a"]) + self.assertNotEqual(base, other) + + def test_tags_change_token(self): + base = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + other = coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a", "c"]) + self.assertNotEqual(base, other) + + +class PurityTests(unittest.TestCase): + def test_derivation_writes_nothing_to_disk(self): + cwd = os.getcwd() + with tempfile.TemporaryDirectory() as tmp: + os.chdir(tmp) + try: + coreci_broker.derive_token(PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"]) + self.assertEqual([], os.listdir(tmp)) + finally: + os.chdir(cwd) + + def test_derivation_isolated_from_store_and_audit(self): + with tempfile.TemporaryDirectory() as tmp: + db_path = os.path.join(tmp, "broker.db") + audit_path = os.path.join(tmp, "broker.audit.log") + store = coreci_broker.Store(db_path) + audit = coreci_broker.Audit(audit_path) + try: + token = coreci_broker.derive_token( + PEPPER, APP_UUID, USER_ID, AGENT_ID, ["a"] + ) + rows = store.conn.execute( + "SELECT * FROM agent_registrations" + ).fetchall() + self.assertEqual([], rows) + finally: + store.close() + audit.close() + with open(audit_path, "r", encoding="utf-8") as handle: + self.assertEqual([], handle.readlines()) + self.assertNotIn(token, open(db_path, "rb").read().decode("latin-1")) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file