broker/tests/test_cli_packaging.py

"""End-to-end packaged-boot acceptance for the coreci-broker CLI zipapp (REQ-21).

Builds the P1-01 artifact once per class via the scripts/build_cli.py build
API into a ``TemporaryDirectory``, then boots it as a real subprocess child —
always ``[sys.executable, "<pyz>"]``, never a bare "python3" — and drives the
live server with stdlib urllib, reusing the v0.1 acceptance request patterns
(tests/test_acceptance_full.py, tests/test_acceptance_p4.py).

G-010 harness hardening, all bound here:
- explicit deadline loop (30s) polling the merged stdout/stderr pipe for the
  ``coreci-broker listening on http://…`` line; loud failure with the captured
  child output when it never arrives;
- pass-once and silent-second-boot assertions are made only after
  ``terminate()`` plus a full ``communicate()`` drain of the child;
- child env pins ``BROKER_CREDS_SOURCE=file`` + a seeded ``BROKER_CREDS_FILE``
  (v0.1 precedent: tests/test_acceptance_full.py);
- one un-overridden default-anchoring boot — state lands in the child's CWD;
- every child is terminated + drained exactly once (idempotent ``stop``
  registered with ``addCleanup``): no leaked processes, and every wait is
  deadline-bounded (boot wait, drain timeout with kill fallback, HTTP and
  subprocess timeouts).
"""

import importlib.util
import json
import os
import select
import subprocess
import sys
import tempfile
import time
import unittest
import urllib.error
import urllib.request
import uuid as uuidlib

REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
BUILD_CLI_PATH = os.path.join(REPO, "scripts", "build_cli.py")
REPO_CLIENT_PATH = os.path.join(REPO, "coreci_broker_client.py")
REPO_LOGIN_PAGE_PATH = os.path.join(REPO, "coreci_broker_ui", "login.html")
REPO_DASHBOARD_PAGE_PATH = os.path.join(REPO, "coreci_broker_ui", "dashboard.html")
VERSION = "v0.2.0"

ADMIN_USER = "admin"
ADMIN_PASS_PREFIX = "BROKER_ADMIN_PASS="
LISTENING_PREFIX = "coreci-broker listening on http://"
ASSET_DIR_NAME = ".coreci_broker_assets"
STATE_FILE_NAMES = (".env", "broker.db", "broker.audit.log")

LOGIN_PATH = "/api/broker/auth/login"
APPS_PATH = "/api/broker/app_registrations"
USERS_PATH = "/api/broker/user_allowlist"
REGISTER_PATH = "/api/broker/agents/register"
AGENT_STATE_PATH = "/api/broker/agents/proxmox/state"
SECRETS_PATH = "/api/broker/secrets/coreci/proxmox"
AUDIT_PATH = "/api/broker/audit"
CLIENT_STATIC_PATH = "/static/coreci_broker_client.py"
UI_LOGIN_PATH = "/ui/login"
UI_DASHBOARD_PATH = "/ui/dashboard"

PROXMOX_PATH = "coreci/proxmox"
PROXMOX_SECRET = {"url": "https://pve.coreci.dev:8006", "token": "pve-tok"}
GITEA_PATH = "coreci/gitea"
GITEA_SECRET = {"url": "https://git.coreci.dev", "token": "git-tok"}

BOOT_DEADLINE_SECONDS = 30.0
POLL_SECONDS = 0.1
DRAIN_TIMEOUT_SECONDS = 20.0
HTTP_TIMEOUT_SECONDS = 10
SUBPROCESS_TIMEOUT_SECONDS = 60


def _load_build_cli():
    spec = importlib.util.spec_from_file_location("build_cli", BUILD_CLI_PATH)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


build_cli = _load_build_cli()


class PackagedBrokerChild:
    """A subprocess child booted from the built zipapp artifact.

    stdout and stderr are merged into one pipe read via ``select`` + ``os.read``
    (raw fd reads — no buffered-reader/select skew) under an explicit deadline
    until the broker announces its ephemeral listen address. ``stop()`` is the
    G-010 kill/drain protocol: ``terminate()`` then a full ``communicate()``
    drain; it is idempotent and registered with ``addCleanup`` at construction,
    so no child leaks even when a boot fails or an assertion trips mid-test.
    """

    def __init__(self, testcase, artifact, env, cwd):
        self.testcase = testcase
        self.artifact = artifact
        self.proc = subprocess.Popen(
            [sys.executable, artifact],
            env=env, cwd=cwd,
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        )
        self.lines = []
        self.partial = b""
        self.drained = b""
        self.base_url = None
        self._eof = False
        self._stopped = False
        testcase.addCleanup(self.stop)
        self._await_listening(BOOT_DEADLINE_SECONDS)

    def _pump(self, timeout):
        """Move whatever is on the merged pipe into line storage.

        Returns False once the child's stream reached EOF; True otherwise
        (including when the poll window simply elapsed with no data).
        """
        if self._eof:
            return False
        fd = self.proc.stdout.fileno()
        ready, _, _ = select.select([fd], [], [], timeout)
        if not ready:
            return True
        try:
            chunk = os.read(fd, 4096)
        except OSError:
            chunk = b""
        if not chunk:
            self._eof = True
            return False
        self.partial += chunk
        while b"\n" in self.partial:
            raw, self.partial = self.partial.split(b"\n", 1)
            self.lines.append(raw.decode("utf-8", "replace"))
        return True

    def _await_listening(self, deadline_seconds):
        deadline = time.monotonic() + deadline_seconds
        cursor = len(self.lines)
        while time.monotonic() < deadline:
            alive = self._pump(POLL_SECONDS)
            while cursor < len(self.lines):
                line = self.lines[cursor]
                cursor += 1
                if line.startswith(LISTENING_PREFIX):
                    self.base_url = line.split(" on ", 1)[1].strip()
                    return
            if not alive:
                break
        self.stop()
        self.testcase.fail(
            "packaged broker did not announce %r within %.0fs "
            "(artifact=%s exit=%s)\n--- captured child output ---\n%s"
            % (LISTENING_PREFIX, deadline_seconds, self.artifact,
               self.proc.returncode, self.full_output())
        )

    def stop(self):
        """Kill/drain protocol (G-010): terminate, then fully drain."""
        if self._stopped:
            return
        self._stopped = True
        if self.proc.poll() is None:
            self.proc.terminate()
        try:
            remainder, _ = self.proc.communicate(timeout=DRAIN_TIMEOUT_SECONDS)
        except subprocess.TimeoutExpired:
            self.proc.kill()
            remainder, _ = self.proc.communicate()
        self.drained = remainder or b""

    def full_output(self):
        """Everything the child printed: complete lines + unterminated tail."""
        text = "\n".join(self.lines)
        tail = self.partial + self.drained
        if tail:
            text += "\n" + tail.decode("utf-8", "replace")
        return text

    def printed_admin_pass(self):
        values = [
            line.split("=", 1)[1]
            for line in self.lines
            if line.startswith(ADMIN_PASS_PREFIX)
        ]
        self.testcase.assertEqual(
            1, len(values),
            "expected exactly one %s line, got: %r" % (ADMIN_PASS_PREFIX, values)
        )
        return values[0]


class CliPackagingAcceptanceTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.build_root = tempfile.TemporaryDirectory(prefix="coreci-broker-pkg-")
        cls.addClassCleanup(cls.build_root.cleanup)
        cls.artifact = build_cli.build(
            VERSION, repo=REPO, dist_dir=cls.build_root.name
        )

    # -- harness helpers -------------------------------------------------

    def _temp_dir(self):
        handle = tempfile.TemporaryDirectory(prefix="coreci-broker-pkg-")
        self.addCleanup(handle.cleanup)
        return handle.name

    def _child_env(self, **overrides):
        """A child env with no inherited BROKER_* keys, then test overrides."""
        env = {
            key: value
            for key, value in os.environ.items()
            if not key.startswith("BROKER_")
        }
        env.update(overrides)
        return env

    def _seed_creds(self, state_dir):
        creds_path = os.path.join(state_dir, "creds.json")
        with open(creds_path, "w", encoding="utf-8") as handle:
            json.dump({PROXMOX_PATH: PROXMOX_SECRET, GITEA_PATH: GITEA_SECRET},
                      handle)
        return creds_path

    def _state_env(self, state_dir):
        """G-010: pin bind + state paths + a seeded file creds source."""
        return self._child_env(
            BROKER_BIND="127.0.0.1:0",
            BROKER_ENV_PATH=os.path.join(state_dir, ".env"),
            BROKER_DB_PATH=os.path.join(state_dir, "broker.db"),
            BROKER_AUDIT_LOG_PATH=os.path.join(state_dir, "broker.audit.log"),
            BROKER_CREDS_SOURCE="file",
            BROKER_CREDS_FILE=self._seed_creds(state_dir),
        )

    def _boot(self, env, cwd):
        return PackagedBrokerChild(self, self.artifact, env, cwd)

    def _boot_in_state(self, state_dir, cwd=None):
        """Boot with state pinned into state_dir; CWD is a separate temp dir."""
        return self._boot(self._state_env(state_dir),
                          cwd if cwd is not None else self._temp_dir())

    def _assert_state_files_0600_in(self, directory):
        for name in STATE_FILE_NAMES:
            path = os.path.join(directory, name)
            self.assertTrue(os.path.isfile(path), path)
            self.assertEqual(0o600, os.stat(path).st_mode & 0o777, path)

    def _request(self, base, method, path, payload=None, headers=None):
        data = None if payload is None else json.dumps(payload).encode("utf-8")
        request = urllib.request.Request(base + path, data=data, method=method)
        for key, value in (headers or {}).items():
            request.add_header(key, value)
        try:
            with urllib.request.urlopen(
                request, timeout=HTTP_TIMEOUT_SECONDS
            ) as response:
                return response.status, response.read(), response.headers
        except urllib.error.HTTPError as error:
            return error.code, error.read(), error.headers

    def _request_json(self, base, method, path, payload=None, headers=None):
        status, body, response_headers = self._request(
            base, method, path, payload, headers
        )
        return status, json.loads(body.decode("utf-8")), response_headers

    def _admin_login(self, base, password):
        """v0.1 pattern: POST login, assert the broker_session cookie is set."""
        status, body, headers = self._request_json(
            base, "POST", LOGIN_PATH,
            {"username": ADMIN_USER, "password": password},
        )
        self.assertEqual(200, status)
        self.assertEqual({"ok": True, "username": ADMIN_USER}, body)
        raw_cookie = headers.get("Set-Cookie")
        self.assertTrue(raw_cookie)
        self.assertTrue(raw_cookie.startswith("broker_session="))
        self.assertIn("HttpOnly", raw_cookie)
        self.assertIn("SameSite=Lax", raw_cookie)
        return {"Cookie": raw_cookie.split(";", 1)[0]}

    def _assert_tool_identity(self, entry, app_uuid):
        self.assertEqual("ceo", entry["user_id"])
        self.assertEqual(app_uuid, entry["app_registration_uuid"])
        self.assertEqual("proxmox", entry["agent_id"])

    def _read_repo_bytes(self, path):
        with open(path, "rb") as handle:
            return handle.read()

    # -- tests ------------------------------------------------------------

    def test_version_flag_subprocess_exits_zero_printing_version(self):
        completed = subprocess.run(
            [sys.executable, self.artifact, "--version"],
            capture_output=True, text=True, timeout=SUBPROCESS_TIMEOUT_SECONDS,
        )
        self.assertEqual(0, completed.returncode)
        self.assertEqual(VERSION, completed.stdout.strip())

    def test_first_boot_state_0600_cwd_independent_and_http_acceptance(self):
        state_dir = self._temp_dir()
        cwd_dir = self._temp_dir()  # deliberately NOT the state dir
        child = self._boot_in_state(state_dir, cwd=cwd_dir)
        base = child.base_url

        status, body, _ = self._request_json(base, "GET", "/healthz")
        self.assertEqual(200, status)
        self.assertEqual({"ok": True}, body)

        # UI pages served from the materialized assets (byte-exact vs repo)
        status, body, headers = self._request(base, "GET", UI_LOGIN_PATH)
        self.assertEqual(200, status)
        self.assertTrue(headers.get("Content-Type", "").startswith("text/html"))
        self.assertEqual(self._read_repo_bytes(REPO_LOGIN_PAGE_PATH), body)
        status, body, headers = self._request(base, "GET", UI_DASHBOARD_PATH)
        self.assertEqual(200, status)
        self.assertTrue(headers.get("Content-Type", "").startswith("text/html"))
        self.assertEqual(self._read_repo_bytes(REPO_DASHBOARD_PAGE_PATH), body)

        # packaged client module served byte-exact vs the repo file
        status, served, _ = self._request(base, "GET", CLIENT_STATIC_PATH)
        self.assertEqual(200, status)
        self.assertEqual(self._read_repo_bytes(REPO_CLIENT_PATH), served)

        # materialized assets under <state>/.coreci_broker_assets/<VERSION>/
        asset_base = os.path.join(state_dir, ASSET_DIR_NAME, VERSION)
        self.assertTrue(os.path.isdir(os.path.join(asset_base, "ui")))
        self.assertTrue(os.path.isdir(os.path.join(asset_base, "static")))
        for name in ("login.html", "dashboard.html"):
            self.assertTrue(
                os.path.isfile(os.path.join(asset_base, "ui", name)), name
            )
        self.assertTrue(os.path.isfile(
            os.path.join(asset_base, "static", "coreci_broker_client.py")
        ))

        # state resolved from BROKER_ENV_PATH: 0600 files in the state dir,
        # nothing whatsoever in the child's CWD
        self._assert_state_files_0600_in(state_dir)
        self.assertEqual([], os.listdir(cwd_dir))

        # pass printed exactly once — asserted on the FULLY drained output
        printed_pass = child.printed_admin_pass()
        child.stop()
        output = child.full_output()
        self.assertEqual(1, output.count(ADMIN_PASS_PREFIX), output)
        self.assertEqual(1, output.count(printed_pass))
        pepper = None
        with open(os.path.join(state_dir, ".env"), "r", encoding="utf-8") as handle:
            for line in handle:
                if line.startswith("BROKER_INSTANCE_PEPPER="):
                    pepper = line.split("=", 1)[1].strip()
        self.assertIsNotNone(pepper)
        self.assertNotIn(pepper, output)

    def test_v01_parity_flows_through_packaged_artifact(self):
        state_dir = self._temp_dir()
        child = self._boot_in_state(state_dir)
        base = child.base_url
        printed_pass = child.printed_admin_pass()
        cookie = self._admin_login(base, printed_pass)

        # seed allowlists first via the admin API (v0.1 pattern)
        app_uuid = str(uuidlib.uuid4())
        status, body, _ = self._request_json(
            base, "POST", APPS_PATH,
            {"uuid": app_uuid, "name": "proxmox-tool"}, headers=cookie,
        )
        self.assertEqual(201, status)
        self.assertEqual({"uuid": app_uuid, "status": "active"}, body)
        status, body, _ = self._request_json(
            base, "POST", USERS_PATH,
            {"user_id": "ceo", "email": "ceo@coreci.dev"}, headers=cookie,
        )
        self.assertEqual(201, status)

        # agent register flow
        status, body, _ = self._request_json(
            base, "POST", REGISTER_PATH,
            {"agent_id": "proxmox", "app_registration_uuid": app_uuid,
             "user_id": "ceo", "tags": ["coreci"]},
        )
        self.assertEqual(201, status)
        token = body["token"]
        app_header = {"Authorization": "App %s %s" % (app_uuid, token)}

        # state put/get roundtrip
        state_document = {"phase": "v0.2", "boot": "packaged", "items": ["parity"]}
        status, body, _ = self._request_json(
            base, "PUT", AGENT_STATE_PATH, state_document, headers=app_header
        )
        self.assertEqual(200, status)
        self.assertEqual({"ok": True}, body)
        status, body, _ = self._request_json(
            base, "GET", AGENT_STATE_PATH, headers=app_header
        )
        self.assertEqual(200, status)
        self.assertEqual(state_document, body)

        # one secret read via the seeded FileCredsSource
        status, body, _ = self._request_json(
            base, "GET", SECRETS_PATH, headers=app_header
        )
        self.assertEqual(200, status)
        self.assertEqual({"path", "data", "version", "updated_at"}, set(body))
        self.assertEqual(PROXMOX_PATH, body["path"])
        self.assertEqual(PROXMOX_SECRET, body["data"])
        self.assertRegex(body["version"], r"^[0-9a-f]{8}$")

        # audit lines exist for the driven flow
        status, body, _ = self._request_json(
            base, "GET", AUDIT_PATH + "?limit=1000", headers=cookie
        )
        self.assertEqual(200, status)
        entries = body["entries"]

        def select(action, result="ok"):
            return [entry for entry in entries
                    if entry["action"] == action and entry["result"] == result]

        logins = select("login")
        self.assertEqual(1, len(logins))
        self.assertEqual(LOGIN_PATH, logins[0]["path"])
        add_apps = select("add_app")
        self.assertEqual(1, len(add_apps))
        self.assertEqual(APPS_PATH + "/" + app_uuid, add_apps[0]["path"])
        add_users = select("add_user")
        self.assertEqual(1, len(add_users))
        self.assertEqual(USERS_PATH + "/ceo", add_users[0]["path"])
        registers = select("register")
        self.assertEqual(1, len(registers))
        self.assertEqual(REGISTER_PATH, registers[0]["path"])
        self._assert_tool_identity(registers[0], app_uuid)
        ok_reads = select("read")
        self.assertEqual(1, len(ok_reads))
        self.assertEqual(PROXMOX_PATH, ok_reads[0]["path"])
        self._assert_tool_identity(ok_reads[0], app_uuid)

        child.stop()
        self.assertEqual(1, child.full_output().count(ADMIN_PASS_PREFIX))

    def test_second_boot_silent_and_state_persists(self):
        state_dir = self._temp_dir()
        first = self._boot_in_state(state_dir)
        printed_pass = first.printed_admin_pass()
        cookie = self._admin_login(first.base_url, printed_pass)
        app_uuid = str(uuidlib.uuid4())
        status, _, _ = self._request_json(
            first.base_url, "POST", APPS_PATH,
            {"uuid": app_uuid, "name": "persist-check"}, headers=cookie,
        )
        self.assertEqual(201, status)

        # kill/drain protocol BEFORE any silence assertion (G-010)
        first.stop()
        first_output = first.full_output()
        self.assertEqual(1, first_output.count(ADMIN_PASS_PREFIX))
        self.assertIn(printed_pass, first_output)

        second = self._boot_in_state(state_dir)
        # the same pass still logs in on the second boot: .env persisted
        cookie = self._admin_login(second.base_url, printed_pass)
        status, body, _ = self._request_json(
            second.base_url, "GET", APPS_PATH, headers=cookie
        )
        self.assertEqual(200, status)
        listed = [row["uuid"] for row in body["registrations"]]
        self.assertIn(app_uuid, listed)

        second.stop()
        second_output = second.full_output()
        self.assertIn(LISTENING_PREFIX, second_output)
        self.assertNotIn(ADMIN_PASS_PREFIX, second_output)

    def test_user_env_ui_root_override_wins_over_materialized_default(self):
        state_dir = self._temp_dir()
        custom_root = os.path.join(state_dir, "custom_ui")
        os.makedirs(custom_root)
        marker = b"<h1>packaged-override-marker</h1>\n"
        with open(os.path.join(custom_root, "login.html"), "wb") as handle:
            handle.write(marker)
        # a user .env (at the child's BROKER_ENV_PATH) wins over the shim default
        with open(os.path.join(state_dir, ".env"), "w", encoding="utf-8") as handle:
            handle.write("BROKER_UI_ROOT=%s\n" % custom_root)

        child = self._boot_in_state(state_dir)
        status, body, headers = self._request(child.base_url, "GET", UI_LOGIN_PATH)
        self.assertEqual(200, status)
        self.assertTrue(headers.get("Content-Type", "").startswith("text/html"))
        self.assertEqual(marker, body)
        # the custom root is the whole UI root: no dashboard.html there
        status, _, _ = self._request(child.base_url, "GET", UI_DASHBOARD_PATH)
        self.assertEqual(404, status)
        # the materialized static default still applies (not overridden)
        status, served, _ = self._request(child.base_url, "GET", CLIENT_STATIC_PATH)
        self.assertEqual(200, status)
        self.assertEqual(self._read_repo_bytes(REPO_CLIENT_PATH), served)
        # materialized defaults still exist on disk; they were simply not forced
        self.assertTrue(os.path.isfile(os.path.join(
            state_dir, ASSET_DIR_NAME, VERSION, "ui", "login.html"
        )))
        child.stop()

    def test_default_anchoring_boot_writes_state_to_cwd(self):
        # the owner's literal happy path: no state overrides except the bind
        cwd_dir = self._temp_dir()
        env = self._child_env(BROKER_BIND="127.0.0.1:0")
        child = self._boot(env, cwd_dir)

        status, body, _ = self._request_json(child.base_url, "GET", "/healthz")
        self.assertEqual(200, status)
        self.assertEqual({"ok": True}, body)
        status, body, headers = self._request(child.base_url, "GET", UI_LOGIN_PATH)
        self.assertEqual(200, status)
        self.assertTrue(headers.get("Content-Type", "").startswith("text/html"))

        # state lands in the child's CWD, at 0600
        self._assert_state_files_0600_in(cwd_dir)

        child.stop()
        self.assertEqual(1, child.full_output().count(ADMIN_PASS_PREFIX))


if __name__ == "__main__":
    unittest.main()