From 8afbf1cd8354dfb56261115cb453ac6fb3567d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kenji=20Sato=20=E2=80=94=20PO=20researchx?= Date: Sat, 31 Oct 2026 01:17:32 +0000 Subject: [PATCH 1/3] release: deploy two services (backend + frontend) - phase 4 slice 2 (REQ-83) --- .ciagent/release/deploy.sh | 88 ++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 19 deletions(-) diff --git a/.ciagent/release/deploy.sh b/.ciagent/release/deploy.sh index 6a5bde1..1155437 100755 --- a/.ciagent/release/deploy.sh +++ b/.ciagent/release/deploy.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # ResearchX release deploy (ciagent release system) # -# Builds the frontend at a given version tag on the Docker host and -# runs it as a container. Idempotent: re-deploying a version replaces -# the running container, keeping the previous one as rollback. +# Builds the frontend and the backend at a given version tag on the +# Docker host and runs them as two containers. Idempotent: re-deploying +# a version replaces the running containers, keeping the previous ones +# as rollback. # # Usage: bash .ciagent/release/deploy.sh e.g. v0.1.4 # @@ -21,8 +22,14 @@ IMAGE_NAME="researchx" CONTAINER_NAME="researchx" CONTAINER_PREV="researchx-prev" HOST_PORT="3000" +IMAGE_BACKEND="researchx-backend" +CONTAINER_BACKEND="researchx-backend" +CONTAINER_BACKEND_PREV="researchx-backend-prev" +HOST_PORT_BACKEND="8000" +BACKEND_DATA_DIR="/var/lib/researchx" REMOTE_BASE="/opt/researchx" HEALTH_PATH="/ask" +BACKEND_HEALTH_PATH="/healthz" HEALTH_RETRIES=15 HEALTH_INTERVAL=2 # ----------------------------------------------------------------------- @@ -38,6 +45,7 @@ VERSION="$1" # ---- dependency checks -------------------------------------------------- command -v ssh >/dev/null || die "ssh not found" 2 +command -v uv >/dev/null || die "uv not found (backend gates need it: pip install uv)" 2 # ---- secrets (env file ONLY) ------------------------------------------- [ -f "$SECRETS_FILE" ] || die "secrets file missing: $SECRETS_FILE" 2 @@ -156,37 +164,66 @@ if [ "$(git -C "$REPO_ROOT" rev-parse HEAD)" != "$TAG_COMMIT" ]; then WORKTREE=$(mktemp -d) git -C "$REPO_ROOT" worktree add --detach "$WORKTREE" "$TAG_COMMIT" >/dev/null trap 'git -C "$REPO_ROOT" worktree remove --force "$WORKTREE" 2>/dev/null || true' EXIT - BUILD_DIR="$WORKTREE/frontend" + BUILD_ROOT="$WORKTREE" else - BUILD_DIR="$REPO_ROOT/frontend" + BUILD_ROOT="$REPO_ROOT" fi +BUILD_DIR="$BUILD_ROOT/frontend" +BACKEND_DIR="$BUILD_ROOT/backend" (cd "$BUILD_DIR" && pnpm install --frozen-lockfile >/dev/null 2>&1) (cd "$BUILD_DIR" && pnpm typecheck >/dev/null) || die "typecheck failed at $VERSION" 3 (cd "$BUILD_DIR" && pnpm lint >/dev/null) || die "lint failed at $VERSION" 3 (cd "$BUILD_DIR" && pnpm test >/dev/null 2>&1) || die "tests failed at $VERSION" 3 (cd "$BUILD_DIR" && pnpm build >/dev/null 2>&1) || die "build failed at $VERSION" 3 -log "gates passed: typecheck, lint, test, build" +log "frontend gates passed: typecheck, lint, test, build" cd "$REPO_ROOT" +# Backend gates: the repo's own commands, same as the CI backend job +# (.gitea/workflows/ci.yml), run from the committed lockfile. +(cd "$BACKEND_DIR" && uv sync --frozen >/dev/null 2>&1) || die "backend dependency sync failed at $VERSION" 3 +(cd "$BACKEND_DIR" && uv run ruff check src tests >/dev/null) || die "backend ruff failed at $VERSION" 3 +(cd "$BACKEND_DIR" && uv run mypy src >/dev/null) || die "backend mypy failed at $VERSION" 3 +(cd "$BACKEND_DIR" && uv run pytest >/dev/null) || die "backend pytest failed at $VERSION" 3 +log "backend gates passed: ruff, mypy, pytest" + # ---- remote host prep ---------------------------------------------------- log "connecting to $DEPLOY_USER@${DEPLOY_HOST%%:*} and preparing $REMOTE_BASE" SSH "mkdir -p '$REMOTE_BASE/releases/$BASE_VERSION'" || die "cannot reach host or create remote dir" 4 SSH "command -v docker >/dev/null" || die "docker not found on host" 4 # ---- sync source --------------------------------------------------------- -log "syncing source to host ($REMOTE_BASE/releases/$BASE_VERSION) via $SYNC_TRANSPORT" -sync_source "$BUILD_DIR" "$REMOTE_BASE/releases/$BASE_VERSION" \ - || die "source sync failed" 4 +RELEASE_DIR="$REMOTE_BASE/releases/$BASE_VERSION" +log "syncing source to host ($RELEASE_DIR) via $SYNC_TRANSPORT" +sync_source "$BUILD_DIR" "$RELEASE_DIR/frontend" || die "frontend source sync failed" 4 +sync_source "$BACKEND_DIR" "$RELEASE_DIR/backend" || die "backend source sync failed" 4 # ---- remote build -------------------------------------------------------- -log "building docker image $IMAGE_NAME:$VERSION on host (this takes a few minutes)" -SSH "cd '$REMOTE_BASE/releases/$BASE_VERSION' && \ +log "building docker images $IMAGE_BACKEND:$VERSION and $IMAGE_NAME:$VERSION on host (this takes a few minutes)" +SSH "cd '$RELEASE_DIR/backend' && \ + docker build -t '$IMAGE_BACKEND:$VERSION' -t '$IMAGE_BACKEND:latest' ." \ + || die "backend docker build failed on host" 4 +SSH "cd '$RELEASE_DIR/frontend' && \ docker build -t '$IMAGE_NAME:$VERSION' -t '$IMAGE_NAME:latest' ." \ - || die "docker build failed on host" 4 + || die "frontend docker build failed on host" 4 # ---- swap container ------------------------------------------------------ -log "swapping container (previous kept as $CONTAINER_PREV for rollback)" +log "swapping backend container (previous kept as $CONTAINER_BACKEND_PREV for rollback)" +SSH " + docker rm -f '$CONTAINER_BACKEND_PREV' 2>/dev/null || true + if docker ps -a --format '{{.Names}}' | grep -qx '$CONTAINER_BACKEND'; then + docker rename '$CONTAINER_BACKEND' '$CONTAINER_BACKEND_PREV' + docker stop '$CONTAINER_BACKEND_PREV' >/dev/null 2>&1 || true + fi + mkdir -p '$BACKEND_DATA_DIR' + docker run -d --name '$CONTAINER_BACKEND' --restart unless-stopped \ + -p '$HOST_PORT_BACKEND:8000' \ + -v '$BACKEND_DATA_DIR:$BACKEND_DATA_DIR' \ + -e RESEARCHX_DATA_DIR='$BACKEND_DATA_DIR' \ + '$IMAGE_BACKEND:$VERSION' +" || die "backend docker run failed" 5 + +log "swapping frontend container (previous kept as $CONTAINER_PREV for rollback)" SSH " docker rm -f '$CONTAINER_PREV' 2>/dev/null || true if docker ps -a --format '{{.Names}}' | grep -qx '$CONTAINER_NAME'; then @@ -195,19 +232,32 @@ SSH " fi docker run -d --name '$CONTAINER_NAME' --restart unless-stopped \ -p '$HOST_PORT:3000' '$IMAGE_NAME:$VERSION' -" || die "docker run failed" 5 +" || die "frontend docker run failed" 5 # ---- health check --------------------------------------------------------- -log "health-checking http://$DEPLOY_HOST:$HOST_PORT$HEALTH_PATH (retries: $HEALTH_RETRIES)" +log "health-checking backend http://$DEPLOY_HOST:$HOST_PORT_BACKEND$BACKEND_HEALTH_PATH (retries: $HEALTH_RETRIES)" +i=0 +while [ "$i" -lt "$HEALTH_RETRIES" ]; do + if SSH "curl -sf -o /dev/null --max-time 5 'http://127.0.0.1:$HOST_PORT_BACKEND$BACKEND_HEALTH_PATH'"; then + log "backend healthy after $((i + 1)) check(s)" + break + fi + i=$((i + 1)) + sleep "$HEALTH_INTERVAL" +done +[ "$i" -lt "$HEALTH_RETRIES" ] || die "backend container did not become healthy within $((HEALTH_RETRIES * HEALTH_INTERVAL))s" 5 + +log "health-checking frontend http://$DEPLOY_HOST:$HOST_PORT$HEALTH_PATH (retries: $HEALTH_RETRIES)" i=0 while [ "$i" -lt "$HEALTH_RETRIES" ]; do if SSH "curl -sf -o /dev/null --max-time 5 'http://127.0.0.1:$HOST_PORT$HEALTH_PATH'"; then - log "healthy after $((i + 1)) check(s)" - log "RELEASED: $VERSION — http://${DEPLOY_HOST%%:*}:$HOST_PORT (container $CONTAINER_NAME)" - log "rollback: ssh $DEPLOY_USER@${DEPLOY_HOST%%:*} 'docker stop $CONTAINER_NAME; docker rename $CONTAINER_PREV $CONTAINER_NAME; docker start $CONTAINER_NAME'" + log "frontend healthy after $((i + 1)) check(s)" + log "RELEASED: $VERSION — http://${DEPLOY_HOST%%:*}:$HOST_PORT (containers $CONTAINER_NAME, $CONTAINER_BACKEND)" + log "backend: http://${DEPLOY_HOST%%:*}:$HOST_PORT_BACKEND$BACKEND_HEALTH_PATH (container $CONTAINER_BACKEND)" + log "rollback: ssh $DEPLOY_USER@${DEPLOY_HOST%%:*} 'docker stop $CONTAINER_NAME $CONTAINER_BACKEND; docker rename $CONTAINER_PREV $CONTAINER_NAME; docker rename $CONTAINER_BACKEND_PREV $CONTAINER_BACKEND; docker start $CONTAINER_NAME $CONTAINER_BACKEND'" exit 0 fi i=$((i + 1)) sleep "$HEALTH_INTERVAL" done -die "container did not become healthy within $((HEALTH_RETRIES * HEALTH_INTERVAL))s" 5 \ No newline at end of file +die "frontend container did not become healthy within $((HEALTH_RETRIES * HEALTH_INTERVAL))s" 5 \ No newline at end of file -- 2.43.0 From 593be870ebe637e0f438bbbb4931877468bec8c5 Mon Sep 17 00:00:00 2001 From: coreci-researchx-po Date: Sat, 31 Oct 2026 03:00:22 +0000 Subject: [PATCH 2/3] fix(release): own the backend data dir for the non-root container user (phase 4 slice 2, REQ-83) --- .ciagent/release/deploy.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.ciagent/release/deploy.sh b/.ciagent/release/deploy.sh index 1155437..2d29b67 100755 --- a/.ciagent/release/deploy.sh +++ b/.ciagent/release/deploy.sh @@ -216,6 +216,12 @@ SSH " docker stop '$CONTAINER_BACKEND_PREV' >/dev/null 2>&1 || true fi mkdir -p '$BACKEND_DATA_DIR' + # The container runs as the image's non-root user and a bind mount + # SHADOWS the image's own (correctly owned) /var/lib/researchx, so a + # root-owned host dir makes every write to the data root (users.json, + # sessions.json, keys/) fail with EACCES while /healthz still answers + # 200. Take the owner from the image itself. + chown -R \$(docker run --rm --entrypoint stat '$IMAGE_BACKEND:$VERSION' -c '%u:%g' /var/lib/researchx) '$BACKEND_DATA_DIR' docker run -d --name '$CONTAINER_BACKEND' --restart unless-stopped \ -p '$HOST_PORT_BACKEND:8000' \ -v '$BACKEND_DATA_DIR:$BACKEND_DATA_DIR' \ @@ -247,6 +253,13 @@ while [ "$i" -lt "$HEALTH_RETRIES" ]; do done [ "$i" -lt "$HEALTH_RETRIES" ] || die "backend container did not become healthy within $((HEALTH_RETRIES * HEALTH_INTERVAL))s" 5 +# The health check above is a GET: it proves the app answers, not that it +# can persist. Probe the write path the auth/key stores use, so a +# mis-owned data root fails the deploy instead of the first signup. +log "checking the backend can write its data root" +SSH "docker exec '$CONTAINER_BACKEND' sh -c 'touch \"\$RESEARCHX_DATA_DIR/.write-probe\" && rm -f \"\$RESEARCHX_DATA_DIR/.write-probe\"'" \ + || die "backend data root is not writable by the container user" 5 + log "health-checking frontend http://$DEPLOY_HOST:$HOST_PORT$HEALTH_PATH (retries: $HEALTH_RETRIES)" i=0 while [ "$i" -lt "$HEALTH_RETRIES" ]; do -- 2.43.0 From 690925c6d147f01615a4aa23a7cd61a8c43efcac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kenji=20Sato=20=E2=80=94=20PO=20researchx?= Date: Sat, 31 Oct 2026 07:15:11 +0000 Subject: [PATCH 3/3] ci: port the phase-4 gate onto the two-service branch so the deploy script is gated before it merges --- .gitea/workflows/ci.yml | 162 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 .gitea/workflows/ci.yml diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..ac2719b --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,162 @@ +# .gitea/workflows/ci.yml - ResearchX phase-4 CI gate (researchx#24). +# +# Ported to the v0.3 line (milestone/v0.3-hosted): phase 4 ships on the +# milestone branch, and the pull_request trigger is extended to it so a +# phase-4 pull request gets a gate run there. The file itself is the same +# gate that ran green on main (researchx#24). +# +# This file IS the gate. The pre-code clause (coreci/company#105, c14057): +# a code phase opens only when the product repo's gating workflow file has +# dispatched at least one run on `main` and every job in that file is green +# in that run. Until this file exists, coreci/researchx has no workflow and +# no run at all - that is the case the clause was amended for (researchx#24, +# c14093). A half-green pipeline does not open the phase either. +# +# Two jobs: +# backend - uv + Python 3.12: ruff, mypy, pytest +# frontend - Node 22 + pnpm 12.6.0: typecheck, lint, test, build +# +# Runner label: `ubuntu-latest` - the org CI fleet's published label whose +# image is docker.gitea.com/runner-images:ubuntu-latest (label map read at +# source: coreci/nextcraft deploy/ci/docker-compose.yaml + deploy/ci/README.md, +# R-3). The previous string `employee-runner` was a fleet NAME, not a label: +# the employee fleet's labels are `employee-py` and `employee-coder` (read +# from a live job log of this seat's own home), and the CI fleet's are node / +# ubuntu-latest / python / release - no runner carried the old label, so a +# merged file asking for it could still buy no run. +# +# Toolchain: measured, not assumed. From the runner's own logs (job 56393, +# coreci/nova-platform, 2026-10-26T23:17Z, and job 57005, coreci/inboxos-v1, +# 2026-10-27T03:01Z): the image carries python 3.12.3, pip, git, node (the +# actions run in it), pypi REACHABLE and github REACHABLE; actions/checkout@v4, +# actions/setup-python@v5 and actions/setup-go@v5 all run inside it. It does +# NOT carry uv or pnpm, so this file installs them: uv from pypi (pip), pnpm +# 12.6.0 through corepack - the repo's own documented way (README command map). +# The gates are the repo's documented commands: backend `uv run ruff check src +# tests`, `uv run mypy src`, `uv run pytest`; frontend `pnpm typecheck`, +# `pnpm lint`, `pnpm test`, `pnpm build`. The frontend lane mirrors the +# release image's own recipe (frontend/Dockerfile: node:22-alpine, corepack, +# pnpm install --frozen-lockfile, pnpm build), so the gate tests what the +# product ships. +# +# Owner of a red leg (the clause asks for the owner of the fix next to the +# owner of the product line): toolchain / image / dispatch problems -> +# coreci-sre (employee-runner-fleet + gitea-instance); product-line problems +# -> coreci-researchx-po. Approval for this file rides the merge request +# (merge = approval, constitution v1.3). + +name: ci + +on: + push: + pull_request: + branches: ["main", "milestone/v0.3-hosted"] + workflow_dispatch: + +jobs: + backend: + name: "backend (uv, Python 3.12: ruff/mypy/pytest)" + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # uv is not in the CI image; pypi is reachable from it (measured). The + # install is a toolchain leg, not a product gate: the gates still run + # from the repo's own committed lockfile (backend/uv.lock). + - name: Install uv (from pypi) + run: | + set -eu + python -m pip install --upgrade uv + uv --version + + - name: Install dependencies (uv, from the committed lockfile) + working-directory: backend + run: uv sync --frozen + + - name: Lint (ruff) + working-directory: backend + run: uv run ruff check src tests + + - name: Typecheck (mypy) + working-directory: backend + run: uv run mypy src + + - name: Test (pytest) + working-directory: backend + run: uv run pytest + + frontend: + name: "frontend (Next.js on pnpm: typecheck/lint/test/build)" + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@v4 + + # Node 22 to mirror the release image (frontend/Dockerfile: node:22). + - name: Setup Node 22 + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Setup Python 3.12 (for the real-stack e2e bridge) + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv (from pypi) + run: | + set -eu + python -m pip install --upgrade uv + + # pnpm 12.6.0 is pinned by frontend/package.json `packageManager`; + # corepack reads it from there. If the image carries no corepack, npm + # installs the same pinned version. + - name: pnpm 12.6.0 (corepack, pinned by frontend/package.json) + working-directory: frontend + run: | + set -eu + node --version + corepack enable pnpm 2>/dev/null || true + if ! pnpm --version >/dev/null 2>&1; then + npm i -g pnpm@12.6.0 + fi + test "$(pnpm --version)" = "12.6.0" + + - name: Install dependencies (from the committed lockfile) + working-directory: frontend + run: pnpm install --frozen-lockfile + + # frontend/src/lib/api/e2e-real-stack.test.ts drives the real FastAPI + # stack: it spawns `uv run python -m researchx.api.dev_bridge` from + # ../backend (loopback :8000, fakes only, zero API keys). Without a + # synced backend venv that test cannot come up, so the lane provisions + # it. Provisioning only - the backend gates stay in the backend job. + - name: Backend venv for the real-stack e2e bridge + working-directory: backend + run: uv sync --frozen + + - name: Typecheck + working-directory: frontend + run: pnpm typecheck + + - name: Lint + working-directory: frontend + run: pnpm lint + + - name: Test + working-directory: frontend + run: pnpm test + + - name: Build + working-directory: frontend + env: + NEXT_TELEMETRY_DISABLED: "1" + run: pnpm build -- 2.43.0