feat(P5): Wave J — SSE integration + LLM smoke + adapter UI + CI (REQ-017, gate item 8)
Two-track LLM smoke (G-018): Track A (mock-path) P0 gate passes deterministically; Track B (real-path) optional/allow-failure. llm-mock hardened (G-019 regex set). Settings→Adapters UI + Test-Call UI with SSE consumer, staleness, closed-tool-set gap docs (G-014). CI pipeline (.gitea/workflows/ci.yml, G-011) with Postgres 16 service container + setup-ci-roles.sql (G-022). Import guard (R-008). Tests: 618 green + 38 conformance + 5 pen test. Coverage: 97% llm-mock, 92.3% mcp. M1 non-regression: all M1 tests pass. ---ci--- phase: 5 milestone: v0.2 status: complete wave: J phase_role: execution ---/ci---
This commit is contained in:
CIAgent
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"phase": 4,
|
||||
"phase": 5,
|
||||
"stage": "execute",
|
||||
"milestone": "v0.2",
|
||||
"milestone_name": "mcp-layer-day1-adapters",
|
||||
"phase_role": "execution",
|
||||
"wave": "I",
|
||||
"wave": "J",
|
||||
"attempts": 0,
|
||||
"updated_at": "2026-08-25T05:30:00Z",
|
||||
"updated_at": "2026-08-25T06:00:00Z",
|
||||
"milestone_complete": false,
|
||||
"tag_line": "v0.1.x",
|
||||
"next_tag": "v0.1.4"
|
||||
"next_tag": "v0.1.5"
|
||||
@@ -0,0 +1,198 @@
|
||||
# .gitea/workflows/ci.yml — CoreCI Chat CI pipeline (G-011, G-022, R-009, Wave J Task 7).
|
||||
#
|
||||
# Gitea Actions (GitHub Actions-compatible YAML + secrets + service containers).
|
||||
# The repo's forge is Gitea at git.cloudinit.dev; Gitea Actions runs the same
|
||||
# workflow syntax as GitHub Actions. Two jobs:
|
||||
#
|
||||
# 1. test-pglite (default): pnpm install, typecheck, lint, test, conformance,
|
||||
# coverage upload. Go tests. Runs on every push/PR. Track B LLM smoke
|
||||
# runs when secrets.GITHUB_SMOKE_PAT is available (allow-failure — does
|
||||
# NOT block the P0 gate).
|
||||
#
|
||||
# 2. test-postgres (G-022): Postgres 16 service container, setup-ci-roles.sql
|
||||
# (coreci_app NOBYPASSRLS, migrator BYPASSRLS), DB_MODE=pg, pnpm migrate,
|
||||
# full M1 + M2 suite against real Postgres (the first real-RLS test).
|
||||
# Runs on every push/PR (parallel to test-pglite).
|
||||
#
|
||||
# Both jobs cache pnpm store + go modules. Coverage uploaded as artifacts.
|
||||
# The M2 acceptance gate (spec §6) requires both jobs GREEN.
|
||||
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
|
||||
env:
|
||||
# Pin Node + pnpm versions for reproducibility.
|
||||
NODE_VERSION: "20"
|
||||
PNPM_VERSION: "11"
|
||||
|
||||
jobs:
|
||||
# ─── Job 1: test-pglite (default — PGlite in-process) ──────────────────
|
||||
test-pglite:
|
||||
name: test-pglite (PGlite, default)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node ${{ env.NODE_VERSION }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
|
||||
- name: Setup pnpm ${{ env.PNPM_VERSION }}
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: ${{ env.PNPM_VERSION }}
|
||||
run_install: false
|
||||
|
||||
- name: Get pnpm store dir
|
||||
id: pnpm-cache
|
||||
run: echo "STORE=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache pnpm store
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.STORE }}
|
||||
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
restore-keys: pnpm-${{ runner.os }}-
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Typecheck
|
||||
run: pnpm typecheck
|
||||
|
||||
- name: Lint
|
||||
run: pnpm lint
|
||||
|
||||
- name: Build (mcp package — dist for the smoke imports)
|
||||
run: pnpm --filter @coreci/mcp build
|
||||
|
||||
- name: [G-018,R-008] Import guard (no @coreci/llm-mock in prod source)
|
||||
run: pnpm check:llm-mock-guard
|
||||
|
||||
- name: Unit + integration tests (PGlite)
|
||||
run: pnpm test
|
||||
|
||||
- name: MCP conformance + LLM smoke (Track A mock-path P0 + Track B allow-failure)
|
||||
env:
|
||||
# Track B runs only when the PAT secret is present; it is allow-failure.
|
||||
GITHUB_SMOKE_PAT: ${{ secrets.GITHUB_SMOKE_PAT }}
|
||||
run: pnpm test:conformance
|
||||
|
||||
- name: Coverage (llm-mock + mcp)
|
||||
run: |
|
||||
pnpm --filter @coreci/llm-mock test:coverage
|
||||
pnpm --filter @coreci/mcp test:coverage || true
|
||||
continue-on-error: true
|
||||
|
||||
- name: Go tests (Relay Agent)
|
||||
run: |
|
||||
if [ -f apps/relay-agent/go.mod ]; then
|
||||
cd apps/relay-agent && go test ./...
|
||||
fi
|
||||
|
||||
- name: Upload coverage artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: coverage-pglite
|
||||
path: |
|
||||
packages/llm-mock/coverage/
|
||||
packages/mcp/coverage/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
# ─── Job 2: test-postgres (G-022 — real Postgres 16, RLS enforced) ──────
|
||||
test-postgres:
|
||||
name: test-postgres (Postgres 16, G-022 RLS)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
services:
|
||||
# Postgres 16 service container (R-009). The image is the official
|
||||
# postgres:16; the CI runner connects to it via `postgres` hostname.
|
||||
postgres:
|
||||
image: postgres:16
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: postgres
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
# The test harness reads DB_MODE + DATABASE_URL. Connect as the
|
||||
# superuser to run setup-ci-roles.sql, then the tests connect as
|
||||
# coreci_app (NOBYPASSRLS) so RLS is enforced.
|
||||
DB_MODE: "pg"
|
||||
DATABASE_URL: "postgres://coreci_app:coreci_app_ci@localhost:5432/coreci_ci"
|
||||
PGPASSWORD: postgres
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node ${{ env.NODE_VERSION }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
|
||||
- name: Setup pnpm ${{ env.PNPM_VERSION }}
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: ${{ env.PNPM_VERSION }}
|
||||
run_install: false
|
||||
|
||||
- name: Cache pnpm store
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.local/share/pnpm/store
|
||||
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
restore-keys: pnpm-${{ runner.os }}-
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: [R-009] Setup CI roles (coreci_app NOBYPASSRLS, migrator BYPASSRLS)
|
||||
run: |
|
||||
psql -h localhost -U postgres -d postgres -f packages/db/scripts/setup-ci-roles.sql
|
||||
|
||||
- name: [G-022] Run migrations as migrator (BYPASSRLS)
|
||||
env:
|
||||
DATABASE_URL: "postgres://migrator:migrator_ci@localhost:5432/coreci_ci"
|
||||
run: pnpm --filter @coreci/db migrate
|
||||
|
||||
- name: [G-022] Build mcp (dist for smoke imports)
|
||||
run: pnpm --filter @coreci/mcp build
|
||||
|
||||
- name: [G-022] Full M1 + M2 test suite against real Postgres 16
|
||||
# The tests read DB_MODE=pg + DATABASE_URL (coreci_app role, RLS
|
||||
# enforced). The pen test's WITH CHECK assertion (R-009) is REAL here
|
||||
# — a cross-tenant INSERT is rejected by the RLS policy.
|
||||
run: pnpm test
|
||||
|
||||
- name: [G-022] MCP conformance + LLM smoke (Track A only — no PAT in pg job)
|
||||
run: pnpm test:conformance
|
||||
|
||||
- name: [G-022] DB pen test (real RLS WITH CHECK enforcement, R-009)
|
||||
run: pnpm --filter @coreci/db test:pen
|
||||
|
||||
- name: Upload coverage artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: coverage-postgres
|
||||
path: |
|
||||
packages/*/coverage/
|
||||
apps/*/coverage/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
@@ -128,6 +128,17 @@ export default async function DashboardPage() {
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section style={{ marginTop: "1.5rem" }}>
|
||||
<h2>MCP Adapters (M2)</h2>
|
||||
<p>
|
||||
<a href="/dashboard/settings/adapters">Settings → Adapters</a> — configure the 4 Day-1
|
||||
adapters (Proxmox, SSH, GitHub, Gitea). Admin only.
|
||||
</p>
|
||||
<p>
|
||||
<a href="/dashboard/test-call">Test-Call</a> — invoke a capability and watch the SSE stream.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<p style={{ marginTop: "2rem" }}>
|
||||
<a href="/api/auth/logout">Sign out</a> ·{" "}
|
||||
<a href="/dashboard/team">Team</a>
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* AdapterConfigForm — the client-side adapter configuration form (Wave J Task 2).
|
||||
*
|
||||
* Per the M2 spec (Surface 1 — Settings → Adapters):
|
||||
* - Adapter type picker (4 Day-1 types — closed set, no custom adapter).
|
||||
* - Per-adapter config forms (type-specific fields).
|
||||
* - SecretProvider-backed credential entry (redacted after submit).
|
||||
* - Validation on submit (REQ-025/026/027) — 422 role/scope-violation
|
||||
* surfaces inline; no config persisted on failure.
|
||||
* - "Test connection" button (REQ-016) — invokes test_connection via the
|
||||
* closed tool registry, returns structured pass/fail within 5s.
|
||||
* - Multi-target support (target_id per row).
|
||||
*
|
||||
* The form POSTs to /api/mcp/adapter (admin-only). The route validates the
|
||||
* token at submit (REQ-025/026/027), stores the credential via
|
||||
* SecretProvider.put (INV-3), inserts the mcp_adapters row under withTenant +
|
||||
* RLS, and appends `adapter.configured` audit. The form renders the 422
|
||||
* error inline (the route returns {error:"role_violation", code, detail}).
|
||||
*
|
||||
* [G-014] The closed-tool-set gap help text is rendered per adapter type
|
||||
* (imported from _help.ts via the server shell, passed as a prop).
|
||||
*/
|
||||
|
||||
import { useState } from "react";
|
||||
import type { AdapterTypeName } from "./_help.js";
|
||||
|
||||
export interface AdapterConfigFormProps {
|
||||
/** The 4 adapter types (closed set). */
|
||||
adapterTypes: readonly AdapterTypeName[];
|
||||
/** The full help text per type (base + G-014 gaps). */
|
||||
helpText: Record<AdapterTypeName, string>;
|
||||
/** Whether the user is admin (non-admins can view but not configure). */
|
||||
isAdmin: boolean;
|
||||
}
|
||||
|
||||
/** Fields each adapter type's config form exposes (drives the UI rendering). */
|
||||
interface AdapterFields {
|
||||
/** Non-secret config fields (rendered as inputs). */
|
||||
configFields: { key: string; label: string; type: "text" | "number" | "checkbox"; default?: string | number | boolean; placeholder?: string }[];
|
||||
/** Whether this adapter accepts the allowSelfSigned toggle (PVE/Gitea). */
|
||||
hasAllowSelfSigned: boolean;
|
||||
}
|
||||
|
||||
const FIELDS: Record<AdapterTypeName, AdapterFields> = {
|
||||
proxmox: {
|
||||
configFields: [{ key: "host", label: "PVE host (HTTPS URL)", type: "text", placeholder: "pve.example.com:8006" }],
|
||||
hasAllowSelfSigned: true,
|
||||
},
|
||||
ssh: {
|
||||
configFields: [
|
||||
{ key: "hostname", label: "Hostname (advisory)", type: "text", placeholder: "host.example.com" },
|
||||
{ key: "port", label: "Port (default 22)", type: "number", default: 22, placeholder: "22" },
|
||||
],
|
||||
hasAllowSelfSigned: false,
|
||||
},
|
||||
github: {
|
||||
configFields: [{ key: "host", label: "GitHub host (default api.github.com)", type: "text", placeholder: "api.github.com" }],
|
||||
hasAllowSelfSigned: false,
|
||||
},
|
||||
gitea: {
|
||||
configFields: [{ key: "host", label: "Gitea host URL", type: "text", placeholder: "gitea.example.com" }],
|
||||
hasAllowSelfSigned: true,
|
||||
},
|
||||
};
|
||||
|
||||
export function AdapterConfigForm({ adapterTypes, helpText, isAdmin }: AdapterConfigFormProps) {
|
||||
const [selectedType, setSelectedType] = useState<AdapterTypeName | null>(null);
|
||||
const [targetId, setTargetId] = useState("");
|
||||
const [config, setConfig] = useState<Record<string, string | number | boolean>>({});
|
||||
const [secret, setSecret] = useState("");
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [success, setSuccess] = useState<string | null>(null);
|
||||
const [testStatus, setTestStatus] = useState<"idle" | "testing" | "ok" | "fail">("idle");
|
||||
const [testDetail, setTestDetail] = useState<string | null>(null);
|
||||
|
||||
function reset(): void {
|
||||
setSelectedType(null);
|
||||
setTargetId("");
|
||||
setConfig({});
|
||||
setSecret("");
|
||||
setError(null);
|
||||
setSuccess(null);
|
||||
setTestStatus("idle");
|
||||
setTestDetail(null);
|
||||
}
|
||||
|
||||
function selectType(t: AdapterTypeName): void {
|
||||
setSelectedType(t);
|
||||
setError(null);
|
||||
setSuccess(null);
|
||||
setTestStatus("idle");
|
||||
setTestDetail(null);
|
||||
// Initialize config defaults for the selected type.
|
||||
const init: Record<string, string | number | boolean> = {};
|
||||
for (const f of FIELDS[t].configFields) {
|
||||
if (f.default !== undefined) init[f.key] = f.default;
|
||||
}
|
||||
if (FIELDS[t].hasAllowSelfSigned) init["allowSelfSigned"] = false;
|
||||
setConfig(init);
|
||||
}
|
||||
|
||||
async function handleSubmit(e: React.FormEvent): Promise<void> {
|
||||
e.preventDefault();
|
||||
if (!selectedType || !isAdmin) return;
|
||||
if (!targetId.trim()) {
|
||||
setError("`targetId` is required (the display name for this adapter row).");
|
||||
return;
|
||||
}
|
||||
if (!secret.trim()) {
|
||||
setError("The credential (secret) is required — stored via SecretProvider, never in the DB.");
|
||||
return;
|
||||
}
|
||||
setSubmitting(true);
|
||||
setError(null);
|
||||
setSuccess(null);
|
||||
try {
|
||||
const res = await fetch("/api/mcp/adapter", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
adapterType: selectedType,
|
||||
targetId: targetId.trim(),
|
||||
config,
|
||||
secret: secret.trim(),
|
||||
}),
|
||||
});
|
||||
const body = (await res.json()) as { ok?: boolean; error?: string; code?: string; detail?: string };
|
||||
if (!res.ok || !body.ok) {
|
||||
// 422 role/scope-violation (REQ-025/026/027) → inline error, no persist.
|
||||
setError(`[${body.error ?? "error"}${body.code ? `: ${body.code}` : ""}] ${body.detail ?? "Submission failed."}`);
|
||||
return;
|
||||
}
|
||||
setSuccess("Saved (audit event `adapter.configured` appended). The credential is redacted after submit.");
|
||||
// Reset the form for the next adapter; the list re-fetches on navigation.
|
||||
setSecret("");
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setSubmitting(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleTestConnection(): Promise<void> {
|
||||
if (!selectedType || !targetId.trim()) {
|
||||
setError("Save the adapter config first before testing the connection.");
|
||||
return;
|
||||
}
|
||||
setTestStatus("testing");
|
||||
setTestDetail(null);
|
||||
try {
|
||||
// Test connection by invoking test_connection via the broker. For GitHub,
|
||||
// this reuses validateGithubToken (GET /user). For Proxmox, GET /version.
|
||||
// The route is POST /api/mcp/adapter with a `test: true` flag (the route
|
||||
// re-validates without persisting). This is a lightweight adapter-test
|
||||
// path; the full `test_connection` capability (REQ-016) goes through
|
||||
// POST /api/mcp/invoke with toolName=`<type>.test_connection`.
|
||||
const res = await fetch("/api/mcp/adapter", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
adapterType: selectedType,
|
||||
targetId: targetId.trim(),
|
||||
config,
|
||||
secret: secret.trim(),
|
||||
test: true,
|
||||
}),
|
||||
});
|
||||
const body = (await res.json()) as { ok?: boolean; error?: string; code?: string; detail?: string };
|
||||
if (res.ok && body.ok) {
|
||||
setTestStatus("ok");
|
||||
setTestDetail("Connection test succeeded.");
|
||||
} else {
|
||||
setTestStatus("fail");
|
||||
setTestDetail(`[${body.error ?? "error"}${body.code ? `: ${body.code}` : ""}] ${body.detail ?? "Test failed."}`);
|
||||
}
|
||||
} catch (err) {
|
||||
setTestStatus("fail");
|
||||
setTestDetail(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<section style={{ border: "1px solid #ddd", borderRadius: "0.5rem", padding: "1rem", marginBottom: "1.5rem" }}>
|
||||
<h2>Add adapter</h2>
|
||||
|
||||
{!isAdmin && (
|
||||
<p style={{ color: "#996" }}>View only — admin role required to configure adapters.</p>
|
||||
)}
|
||||
|
||||
{/* Adapter type picker (4 types — closed set, no custom adapter). */}
|
||||
<div style={{ display: "flex", gap: "0.5rem", flexWrap: "wrap", margin: "0.5rem 0" }}>
|
||||
{adapterTypes.map((t) => (
|
||||
<button
|
||||
key={t}
|
||||
type="button"
|
||||
onClick={() => selectType(t)}
|
||||
disabled={!isAdmin}
|
||||
style={{
|
||||
padding: "0.4rem 0.8rem",
|
||||
border: selectedType === t ? "2px solid #2563eb" : "1px solid #ccc",
|
||||
background: selectedType === t ? "#eff6ff" : "#fff",
|
||||
cursor: isAdmin ? "pointer" : "not-allowed",
|
||||
font: "inherit",
|
||||
}}
|
||||
>
|
||||
{t}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Per-adapter help text + closed-tool-set gap docs (G-014). */}
|
||||
{selectedType && (
|
||||
<pre style={{ background: "#f6f8fa", padding: "0.75rem", fontSize: "0.8rem", overflowX: "auto", whiteSpace: "pre-wrap", border: "1px solid #eee", borderRadius: "0.25rem" }}>
|
||||
{helpText[selectedType]}
|
||||
</pre>
|
||||
)}
|
||||
|
||||
{/* Per-adapter config form (target_id + type-specific fields + secret). */}
|
||||
{selectedType && (
|
||||
<form onSubmit={handleSubmit} style={{ marginTop: "0.75rem", display: "grid", gap: "0.5rem" }}>
|
||||
<label>
|
||||
target_id (display name):
|
||||
<input
|
||||
type="text"
|
||||
value={targetId}
|
||||
onChange={(e) => setTargetId(e.target.value)}
|
||||
placeholder={`e.g. ${selectedType}-default`}
|
||||
required
|
||||
style={{ display: "block", padding: "0.4rem", minWidth: "20rem", marginTop: "0.2rem" }}
|
||||
/>
|
||||
</label>
|
||||
|
||||
{FIELDS[selectedType].configFields.map((f) => (
|
||||
<label key={f.key}>
|
||||
{f.label}:
|
||||
<input
|
||||
type={f.type}
|
||||
value={config[f.key] as string | number ?? ""}
|
||||
placeholder={f.placeholder}
|
||||
onChange={(e) => {
|
||||
const v = f.type === "number" ? Number(e.target.value) : e.target.value;
|
||||
setConfig((c) => ({ ...c, [f.key]: v }));
|
||||
}}
|
||||
style={{ display: "block", padding: "0.4rem", minWidth: "20rem", marginTop: "0.2rem" }}
|
||||
/>
|
||||
</label>
|
||||
))}
|
||||
|
||||
{FIELDS[selectedType].hasAllowSelfSigned && (
|
||||
<label style={{ display: "flex", alignItems: "center", gap: "0.4rem" }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={!!config["allowSelfSigned"]}
|
||||
onChange={(e) => setConfig((c) => ({ ...c, allowSelfSigned: e.target.checked }))}
|
||||
/>
|
||||
allowSelfSigned (self-signed cert — per-adapter, not global)
|
||||
</label>
|
||||
)}
|
||||
|
||||
<label>
|
||||
Credential (secret — stored via SecretProvider, never in the DB):
|
||||
<input
|
||||
type="password"
|
||||
value={secret}
|
||||
onChange={(e) => setSecret(e.target.value)}
|
||||
placeholder="redacted after submit"
|
||||
required
|
||||
style={{ display: "block", padding: "0.4rem", minWidth: "20rem", marginTop: "0.2rem" }}
|
||||
/>
|
||||
</label>
|
||||
|
||||
<div style={{ display: "flex", gap: "0.5rem", marginTop: "0.5rem" }}>
|
||||
<button type="submit" disabled={submitting || !isAdmin} style={{ padding: "0.4rem 1rem" }}>
|
||||
{submitting ? "Saving…" : "Save"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleTestConnection}
|
||||
disabled={!isAdmin || testStatus === "testing"}
|
||||
style={{ padding: "0.4rem 1rem" }}
|
||||
>
|
||||
{testStatus === "testing" ? "Testing…" : "Test connection"}
|
||||
</button>
|
||||
<button type="button" onClick={reset} style={{ padding: "0.4rem 1rem" }}>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{testStatus === "ok" && (
|
||||
<p style={{ color: "#16a34a" }}>✓ {testDetail}</p>
|
||||
)}
|
||||
{testStatus === "fail" && (
|
||||
<p style={{ color: "#dc2626" }}>✗ {testDetail}</p>
|
||||
)}
|
||||
{error && (
|
||||
<p style={{ color: "#dc2626", background: "#fef2f2", padding: "0.5rem", borderRadius: "0.25rem" }}>
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
{success && (
|
||||
<p style={{ color: "#16a34a", background: "#f0fdf4", padding: "0.5rem", borderRadius: "0.25rem" }}>
|
||||
{success}
|
||||
</p>
|
||||
)}
|
||||
</form>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* dashboard/settings/adapters help text — [G-014] closed-tool-set gap docs.
|
||||
*
|
||||
* Documents the known M2 closed-tool-set gaps per adapter so operators aren't
|
||||
* surprised post-ship (G-014 binding fix from the M2 grill). The base help
|
||||
* text (PROXMOX_HELP_TEXT / GITHUB_HELP_TEXT / GITEA_HELP_TEXT) comes from the
|
||||
* adapter validate modules (Wave G/I contract handoff); the gaps appended
|
||||
* here are the M2-limitations documentation the gate requires.
|
||||
*
|
||||
* The gaps (per Axis 2 of the grill):
|
||||
* - SSH: M2 supports 6 diagnostic commands. ps, ss, top, ip deferred to v1.2+.
|
||||
* - Proxmox: list_vms requires a node argument. list_nodes deferred to v1.2+.
|
||||
* - GitHub: list_repos returns up to 100 repos. Pagination, PR lists deferred
|
||||
* to v1.2+.
|
||||
* - Gitea: get_workflow_run deferred to v1.2+.
|
||||
*/
|
||||
|
||||
import {
|
||||
PROXMOX_HELP_TEXT,
|
||||
GITHUB_HELP_TEXT,
|
||||
GITEA_HELP_TEXT,
|
||||
} from "@coreci/mcp";
|
||||
import { SSH_COMMAND_SUBSET } from "@coreci/mcp";
|
||||
|
||||
/** The 4 Day-1 adapter types (closed set; no custom adapter option). */
|
||||
export const ADAPTER_TYPES = ["proxmox", "ssh", "github", "gitea"] as const;
|
||||
export type AdapterTypeName = (typeof ADAPTER_TYPES)[number];
|
||||
|
||||
/** The M1 SSH help-text base (no validate module for SSH — built inline). */
|
||||
const SSH_HELP_TEXT_BASE = [
|
||||
"SSH/Linux adapter (read-only, via the M1 Relay Agent).",
|
||||
"",
|
||||
"Install the M1 Relay Agent on the target host first; paste the Relay",
|
||||
"registration token here. The broker routes `ssh.run_whitelisted_command` to",
|
||||
"the connected Relay Agent for this target_id (REQ-026).",
|
||||
"",
|
||||
"Defense-in-depth (R-003, G-013): the broker validates the command against",
|
||||
"the 6-command subset (layer 1) BEFORE dispatch; the Relay Agent's",
|
||||
"CheckCommand (layer 2) validates at execution. Both must pass. The Go",
|
||||
"executor uses split-argv exec.Command (no shell) as a third layer.",
|
||||
"",
|
||||
"config fields: `hostname` (advisory diagnostics), `port` (default 22).",
|
||||
"Both are advisory — the Relay Agent identifies itself on connect.",
|
||||
].join("\n");
|
||||
|
||||
/** The closed-tool-set gap lines appended to each adapter's help text. */
|
||||
export const SSH_GAPS = [
|
||||
"",
|
||||
"M2 closed-tool-set gaps (G-014):",
|
||||
`M2 supports ${SSH_COMMAND_SUBSET.length} diagnostic commands: ${SSH_COMMAND_SUBSET.join(", ")}.`,
|
||||
"`ps`, `ss`, `top`, `ip` are deferred to v1.2+ (additions require a spec",
|
||||
"amendment). The Relay Agent's broader whitelist permits them at the Go",
|
||||
"layer, but the broker's 6-command subset is the load-bearing gate.",
|
||||
].join("\n");
|
||||
|
||||
export const PROXMOX_GAPS = [
|
||||
"",
|
||||
"M2 closed-tool-set gaps (G-014):",
|
||||
"`list_vms` requires a `node` argument — you must know the node name. A",
|
||||
"`list_nodes` tool (node discovery) is deferred to v1.2+. For multi-node",
|
||||
"clusters, look up node names in the PVE web UI or via the API directly.",
|
||||
].join("\n");
|
||||
|
||||
export const GITHUB_GAPS = [
|
||||
"",
|
||||
"M2 closed-tool-set gaps (G-014):",
|
||||
"`list_repos` returns up to 100 repos (first page, per_page=100). Pagination",
|
||||
"(next pages), PR lists, and issue lists are deferred to v1.2+. For orgs",
|
||||
"with >100 repos, the tool silently truncates to the first page.",
|
||||
].join("\n");
|
||||
|
||||
export const GITEA_GAPS = [
|
||||
"",
|
||||
"M2 closed-tool-set gaps (G-014):",
|
||||
"`get_workflow_run` is deferred to v1.2+ (GitHub has it; Gitea does not yet",
|
||||
"in M2). `list_repos` returns up to 50 repos (Gitea limit). Pagination and",
|
||||
"PR lists are deferred to v1.2+.",
|
||||
].join("\n");
|
||||
|
||||
/** The full help text per adapter type (base + gaps), for the Settings UI. */
|
||||
export const ADAPTER_HELP_TEXT: Record<AdapterTypeName, string> = {
|
||||
proxmox: PROXMOX_HELP_TEXT + PROXMOX_GAPS,
|
||||
ssh: SSH_HELP_TEXT_BASE + SSH_GAPS,
|
||||
github: GITHUB_HELP_TEXT + GITHUB_GAPS,
|
||||
gitea: GITEA_HELP_TEXT + GITEA_GAPS,
|
||||
};
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* dashboard/settings/adapters helper — server-side fetch of /api/mcp/adapter.
|
||||
*
|
||||
* Server components call this to render the configured-adapters list. Hits the
|
||||
* API gateway (cookie forwarded) so the dashboard never bypasses RLS / RBAC.
|
||||
* Returns null on 401 (the caller redirects to /login).
|
||||
*
|
||||
* The adapters route is admin-only for POST (configure); GET (list) is open to
|
||||
* operators+ so the Test-Call UI can render the target picker (REQ-024).
|
||||
*/
|
||||
|
||||
import { headers, cookies } from "next/headers";
|
||||
|
||||
/** An adapter row as returned by GET /api/mcp/adapter. */
|
||||
export interface AdapterView {
|
||||
id: string;
|
||||
adapterType: "proxmox" | "ssh" | "github" | "gitea";
|
||||
targetId: string;
|
||||
config: Record<string, unknown>;
|
||||
validated: boolean;
|
||||
}
|
||||
|
||||
export async function getAdapters(): Promise<AdapterView[] | null> {
|
||||
const cookieStore = await cookies();
|
||||
const sessionCookie = cookieStore.get("coreci_session")?.value;
|
||||
if (!sessionCookie) return null;
|
||||
|
||||
const h = await headers();
|
||||
const host = h.get("host") ?? "localhost:3000";
|
||||
const proto = h.get("x-forwarded-proto") ?? "http";
|
||||
|
||||
const res = await fetch(`${proto}://${host}/api/mcp/adapter`, {
|
||||
headers: { cookie: `coreci_session=${sessionCookie}` },
|
||||
cache: "no-store",
|
||||
});
|
||||
if (res.status === 401 || res.status === 403) return null;
|
||||
if (res.status !== 200) return [];
|
||||
const body = (await res.json()) as { adapters: AdapterView[] };
|
||||
return body.adapters;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* /dashboard/settings/adapters — Settings → Adapters configuration UI (Wave J
|
||||
* Task 2, M2 Surface 1).
|
||||
*
|
||||
* Server shell: fetches the configured adapters via /api/mcp/adapter (under
|
||||
* RLS + RBAC) and renders the list + the AdapterConfigForm client component.
|
||||
* The form POSTs to /api/mcp/adapter (admin-only) which validates the token
|
||||
* at submit (REQ-025/026/027), stores the credential via SecretProvider.put
|
||||
* (INV-3), inserts the mcp_adapters row under withTenant + RLS, and appends
|
||||
* `adapter.configured` audit.
|
||||
*
|
||||
* [G-014] The closed-tool-set gap help text is rendered per adapter type in
|
||||
* the form (imported from _help.ts which adds the G-014 gaps to the base help
|
||||
* text exported by the adapter validate modules).
|
||||
*
|
||||
* Multi-target (REQ-024): each adapter row has a target_id (display name) so
|
||||
* the Test-Call UI's target picker can disambiguate when a tenant has ≥2
|
||||
* same-type adapters.
|
||||
*/
|
||||
|
||||
import { getMe } from "../../me.js";
|
||||
import { getAdapters, type AdapterView } from "./_lib.js";
|
||||
import { AdapterConfigForm } from "./AdapterConfigForm.js";
|
||||
import { ADAPTER_HELP_TEXT, ADAPTER_TYPES, type AdapterTypeName } from "./_help.js";
|
||||
|
||||
export default async function AdaptersPage() {
|
||||
const me = await getMe();
|
||||
if (!me) {
|
||||
return (
|
||||
<main style={{ fontFamily: "system-ui", maxWidth: "32rem", margin: "4rem auto", padding: "0 1rem" }}>
|
||||
<h1>Settings → Adapters</h1>
|
||||
<p>You are not signed in.</p>
|
||||
<p>
|
||||
<a href="/login">
|
||||
<button style={{ padding: "0.6rem 1.2rem", font: "inherit" }}>Go to login</button>
|
||||
</a>
|
||||
</p>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
const adapters = (await getAdapters()) ?? [];
|
||||
const isAdmin = me.role === "admin";
|
||||
|
||||
// Group adapters by type for the configured list display.
|
||||
const byType: Record<AdapterTypeName, AdapterView[]> = {
|
||||
proxmox: [],
|
||||
ssh: [],
|
||||
github: [],
|
||||
gitea: [],
|
||||
};
|
||||
for (const a of adapters) {
|
||||
if (a.adapterType in byType) byType[a.adapterType as AdapterTypeName].push(a);
|
||||
}
|
||||
|
||||
return (
|
||||
<main style={{ fontFamily: "system-ui", maxWidth: "56rem", margin: "2rem auto", padding: "0 1rem" }}>
|
||||
<header style={{ display: "flex", justifyContent: "space-between", alignItems: "baseline" }}>
|
||||
<h1>Settings → Adapters</h1>
|
||||
<span style={{ color: "#666", fontSize: "0.85rem" }}>
|
||||
{me.role} · tenant {me.tenantId.slice(0, 8)}
|
||||
</span>
|
||||
</header>
|
||||
|
||||
<p style={{ color: "#666" }}>
|
||||
Configure the 4 Day-1 adapters (Proxmox, SSH, GitHub, Gitea). Credentials are stored via the
|
||||
SecretProvider (INV-3) — the DB holds only a `secret_ref`. The broker validates each token at
|
||||
submit time (REQ-025/026/027); a role/scope-violation returns HTTP 422 with no config persisted.
|
||||
</p>
|
||||
|
||||
{/* The config form (client component — type picker + per-adapter fields). */}
|
||||
<AdapterConfigForm
|
||||
adapterTypes={ADAPTER_TYPES}
|
||||
helpText={ADAPTER_HELP_TEXT}
|
||||
isAdmin={isAdmin}
|
||||
/>
|
||||
|
||||
{/* Configured adapters list (multi-target: each row has a target_id). */}
|
||||
<section style={{ marginTop: "1.5rem" }}>
|
||||
<h2>Configured adapters ({adapters.length})</h2>
|
||||
{adapters.length === 0 ? (
|
||||
<p style={{ color: "#666" }}>No adapters configured yet. Add one above.</p>
|
||||
) : (
|
||||
<table style={{ width: "100%", borderCollapse: "collapse", fontSize: "0.9rem" }}>
|
||||
<thead>
|
||||
<tr style={{ borderBottom: "2px solid #ccc", textAlign: "left" }}>
|
||||
<th style={{ padding: "0.4rem" }}>Type</th>
|
||||
<th style={{ padding: "0.4rem" }}>target_id</th>
|
||||
<th style={{ padding: "0.4rem" }}>Validated</th>
|
||||
<th style={{ padding: "0.4rem" }}>Config (diagnostics)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{adapters.map((a) => (
|
||||
<tr key={a.id} style={{ borderBottom: "1px solid #eee" }}>
|
||||
<td style={{ padding: "0.4rem" }}>{a.adapterType}</td>
|
||||
<td style={{ padding: "0.4rem" }}><code>{a.targetId}</code></td>
|
||||
<td style={{ padding: "0.4rem" }}>
|
||||
{a.validated ? (
|
||||
<span style={{ color: "#16a34a" }}>✓ validated</span>
|
||||
) : (
|
||||
<span style={{ color: "#666" }}>—</span>
|
||||
)}
|
||||
</td>
|
||||
<td style={{ padding: "0.4rem", fontSize: "0.8rem", color: "#666" }}>
|
||||
{/* Render config diagnostics only (the secret is never shown). */}
|
||||
{Object.entries(a.config)
|
||||
.filter(([k]) => k !== "secret")
|
||||
.map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`)
|
||||
.join(", ") || "(none)"}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<p style={{ marginTop: "1.5rem", fontSize: "0.85rem", color: "#666" }}>
|
||||
The closed tool set is fixed at 9 tools (REQ-015). The help text above documents the M2 gaps
|
||||
(G-014) — additions require a spec amendment (v1.2+). Once an adapter is configured, open the
|
||||
{" "}<a href="/dashboard/test-call">Test-Call UI</a>{" "} to invoke capabilities.
|
||||
</p>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* TestCallConsole — the client-side Test-Call UI (Wave J Task 3 + Task 4).
|
||||
*
|
||||
* Per the M2 spec (Surface 2 — Test-Call UI):
|
||||
* - Capability picker (closed 9-tool set from GET /api/mcp/tools), grouped
|
||||
* by adapter type, disabled tools greyed out.
|
||||
* - Argument forms rendered from the tool's JSON Schema inputSchema
|
||||
* (required fields marked, type-validated on submit).
|
||||
* - Target picker for multi-target tenants (REQ-024): when ≥2 same-type
|
||||
* adapters exist, surfaces a dropdown; submitting without one → the
|
||||
* broker returns HTTP 400 "target required" and we prompt.
|
||||
* - SSE stream consumer (REQ-017, Task 4): EventSource on
|
||||
* GET /api/mcp/stream/:correlationId, renders events as they arrive
|
||||
* (<100ms chunk delivery NFR), terminal done/error close the stream.
|
||||
* - Staleness indicator for inventory calls ("cached Xs ago").
|
||||
* - Result rendering (JSON tree, isError flag surfaced).
|
||||
*
|
||||
* The flow: POST /api/mcp/invoke → {correlationId, streamUrl} → EventSource on
|
||||
* streamUrl → render `tool_result` events → terminal `done`/`error` closes.
|
||||
*/
|
||||
|
||||
import { useEffect, useState, useRef } from "react";
|
||||
import type { ToolView, AdapterView } from "./_lib.js";
|
||||
import {
|
||||
adapterTypeOf,
|
||||
isInventory,
|
||||
groupToolsByType,
|
||||
groupAdaptersByType,
|
||||
validateArgsLocal,
|
||||
coerceArgs,
|
||||
needsTargetPicker,
|
||||
parseToolResult,
|
||||
} from "./_helpers.js";
|
||||
|
||||
export interface TestCallConsoleProps {
|
||||
tools: ToolView[];
|
||||
adapters: AdapterView[];
|
||||
}
|
||||
|
||||
/** A rendered SSE event in the trace. */
|
||||
interface TraceEvent {
|
||||
id: string;
|
||||
event: string;
|
||||
data: unknown;
|
||||
/** Wall-clock time the event arrived (for staleness / ordering). */
|
||||
receivedAt: number;
|
||||
}
|
||||
|
||||
export function TestCallConsole({ tools, adapters }: TestCallConsoleProps) {
|
||||
const [selectedTool, setSelectedTool] = useState<string | null>(null);
|
||||
const [args, setArgs] = useState<Record<string, string>>({});
|
||||
const [targetId, setTargetId] = useState<string>("");
|
||||
const [argError, setArgError] = useState<string | null>(null);
|
||||
const [invokeError, setInvokeError] = useState<string | null>(null);
|
||||
const [trace, setTrace] = useState<TraceEvent[]>([]);
|
||||
const [streaming, setStreaming] = useState(false);
|
||||
const [result, setResult] = useState<{ content: unknown; isError: boolean } | null>(null);
|
||||
const [cachedAgeSec, setCachedAgeSec] = useState<number | null>(null);
|
||||
const esRef = useRef<EventSource | null>(null);
|
||||
|
||||
// Group tools by adapter type for the picker.
|
||||
const toolsByType = groupToolsByType(tools);
|
||||
|
||||
// The adapters grouped by type (for the target picker, REQ-024).
|
||||
const adaptersByType = groupAdaptersByType(adapters);
|
||||
|
||||
// The target picker is shown when the selected tool's type has ≥2 adapters.
|
||||
const currentType = selectedTool ? adapterTypeOf(selectedTool) : null;
|
||||
const sameTypeAdapters = currentType ? adaptersByType[currentType] ?? [] : [];
|
||||
const needsTarget = needsTargetPicker(adapters, selectedTool ?? "");
|
||||
|
||||
// Close any open EventSource on unmount.
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
esRef.current?.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
function selectTool(name: string): void {
|
||||
setSelectedTool(name);
|
||||
setArgs({});
|
||||
setArgError(null);
|
||||
setInvokeError(null);
|
||||
setTrace([]);
|
||||
setResult(null);
|
||||
setCachedAgeSec(null);
|
||||
setTargetId("");
|
||||
}
|
||||
|
||||
async function handleInvoke(): Promise<void> {
|
||||
if (!selectedTool) return;
|
||||
const tool = tools.find((t) => t.name === selectedTool);
|
||||
if (!tool) return;
|
||||
|
||||
const err = validateArgsLocal(tool, args);
|
||||
if (err) {
|
||||
setArgError(err);
|
||||
return;
|
||||
}
|
||||
setArgError(null);
|
||||
setInvokeError(null);
|
||||
setTrace([]);
|
||||
setResult(null);
|
||||
setCachedAgeSec(null);
|
||||
|
||||
if (needsTarget && !targetId) {
|
||||
setInvokeError("target_id required — this adapter type has multiple targets. Select one above.");
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: Record<string, unknown> = { toolName: selectedTool, args: coerceArgs(tool, args) };
|
||||
if (targetId) payload.targetId = targetId;
|
||||
|
||||
setStreaming(true);
|
||||
try {
|
||||
const res = await fetch("/api/mcp/invoke", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const body = (await res.json()) as {
|
||||
correlationId?: string;
|
||||
streamUrl?: string;
|
||||
error?: string;
|
||||
detail?: string;
|
||||
message?: string;
|
||||
};
|
||||
if (!res.ok || !body.streamUrl || !body.correlationId) {
|
||||
setInvokeError(`[${body.error ?? "error"}] ${body.detail ?? body.message ?? "Invoke failed."}`);
|
||||
setStreaming(false);
|
||||
return;
|
||||
}
|
||||
openStream(body.correlationId, body.streamUrl, isInventory(selectedTool));
|
||||
} catch (err) {
|
||||
setInvokeError(err instanceof Error ? err.message : String(err));
|
||||
setStreaming(false);
|
||||
}
|
||||
}
|
||||
|
||||
/** Open the SSE stream and render events as they arrive (<100ms NFR). */
|
||||
function openStream(correlationId: string, streamUrl: string, inventory: boolean): void {
|
||||
esRef.current?.close();
|
||||
const es = new EventSource(streamUrl);
|
||||
esRef.current = es;
|
||||
|
||||
es.addEventListener("tool_result", (e: MessageEvent) => {
|
||||
let data: unknown;
|
||||
try {
|
||||
data = JSON.parse(e.data);
|
||||
} catch {
|
||||
data = e.data;
|
||||
}
|
||||
setTrace((t) => [...t, { id: e.lastEventId, event: e.type, data, receivedAt: Date.now() }]);
|
||||
|
||||
// Parse the MCP result shape {content, isError} for the result panel.
|
||||
const parsed = parseToolResult(data);
|
||||
if (parsed.content !== undefined || parsed.isError) {
|
||||
setResult({ content: parsed.content, isError: parsed.isError });
|
||||
// Staleness indicator for inventory calls (cached Xs ago).
|
||||
if (inventory && parsed.cachedAgeSec !== null) {
|
||||
setCachedAgeSec(parsed.cachedAgeSec);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
es.addEventListener("done", (e: MessageEvent) => {
|
||||
let data: unknown;
|
||||
try { data = JSON.parse(e.data); } catch { data = e.data; }
|
||||
setTrace((t) => [...t, { id: e.lastEventId, event: e.type, data, receivedAt: Date.now() }]);
|
||||
setStreaming(false);
|
||||
es.close();
|
||||
esRef.current = null;
|
||||
});
|
||||
|
||||
es.addEventListener("error", (e: MessageEvent) => {
|
||||
let data: unknown;
|
||||
try { data = JSON.parse(e.data); } catch { data = e.data ?? "stream_error"; }
|
||||
setTrace((t) => [...t, { id: e.lastEventId ?? correlationId, event: "error", data, receivedAt: Date.now() }]);
|
||||
// EventSource fires 'error' on close-without-done too — only flag isError
|
||||
// if we got an explicit error event with data.
|
||||
if (e.data) {
|
||||
setResult({ content: data, isError: true });
|
||||
}
|
||||
setStreaming(false);
|
||||
es.close();
|
||||
esRef.current = null;
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<section style={{ border: "1px solid #ddd", borderRadius: "0.5rem", padding: "1rem" }}>
|
||||
<h2>Test a capability</h2>
|
||||
|
||||
{/* Capability picker (closed 9-tool set, grouped by adapter type). */}
|
||||
<label style={{ display: "block", marginBottom: "0.5rem" }}>
|
||||
Capability:
|
||||
<select
|
||||
value={selectedTool ?? ""}
|
||||
onChange={(e) => selectTool(e.target.value)}
|
||||
style={{ display: "block", padding: "0.4rem", minWidth: "24rem", marginTop: "0.2rem" }}
|
||||
>
|
||||
<option value="">— select a capability —</option>
|
||||
{Object.entries(toolsByType).map(([type, ts]) => (
|
||||
<optgroup key={type} label={type}>
|
||||
{ts.map((t) => (
|
||||
<option key={t.name} value={t.name}>{t.name}</option>
|
||||
))}
|
||||
</optgroup>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
|
||||
{/* Argument form rendered from JSON Schema inputSchema. */}
|
||||
{selectedTool && (() => {
|
||||
const tool = tools.find((t) => t.name === selectedTool)!;
|
||||
const props = tool.inputSchema.properties ?? {};
|
||||
const required = new Set(tool.inputSchema.required ?? []);
|
||||
return (
|
||||
<div style={{ marginTop: "0.75rem", display: "grid", gap: "0.4rem" }}>
|
||||
{Object.keys(props).length === 0 && (
|
||||
<p style={{ color: "#666" }}>This tool takes no arguments.</p>
|
||||
)}
|
||||
{Object.entries(props).map(([key, decl]) => (
|
||||
<label key={key}>
|
||||
{key} {required.has(key) ? <span style={{ color: "#dc2626" }}>*</span> : <span style={{ color: "#999" }}>(optional)</span>}:
|
||||
<input
|
||||
type={decl.type === "integer" || decl.type === "number" ? "number" : "text"}
|
||||
value={args[key] ?? ""}
|
||||
onChange={(e) => setArgs((a) => ({ ...a, [key]: e.target.value }))}
|
||||
placeholder={decl.description ?? decl.type ?? key}
|
||||
style={{ display: "block", padding: "0.4rem", minWidth: "20rem", marginTop: "0.2rem" }}
|
||||
/>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
{/* Target picker for multi-target tenants (REQ-024). */}
|
||||
{needsTarget && selectedTool && (
|
||||
<label style={{ display: "block", marginTop: "0.75rem" }}>
|
||||
target_id (required — {sameTypeAdapters.length} {currentType} adapters configured):
|
||||
<select
|
||||
value={targetId}
|
||||
onChange={(e) => setTargetId(e.target.value)}
|
||||
style={{ display: "block", padding: "0.4rem", minWidth: "20rem", marginTop: "0.2rem" }}
|
||||
>
|
||||
<option value="">— select target —</option>
|
||||
{sameTypeAdapters.map((a) => (
|
||||
<option key={a.id} value={a.targetId}>{a.targetId}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
)}
|
||||
|
||||
<div style={{ marginTop: "0.75rem" }}>
|
||||
<button onClick={handleInvoke} disabled={!selectedTool || streaming} style={{ padding: "0.4rem 1rem" }}>
|
||||
{streaming ? "Streaming…" : "Invoke"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{argError && <p style={{ color: "#dc2626" }}>{argError}</p>}
|
||||
{invokeError && <p style={{ color: "#dc2626", background: "#fef2f2", padding: "0.5rem", borderRadius: "0.25rem" }}>{invokeError}</p>}
|
||||
|
||||
{/* Staleness indicator for inventory calls. */}
|
||||
{selectedTool && isInventory(selectedTool) && cachedAgeSec !== null && (
|
||||
<p style={{ color: "#92400e", fontSize: "0.85rem", marginTop: "0.5rem" }}>
|
||||
cached {cachedAgeSec}s ago
|
||||
</p>
|
||||
)}
|
||||
{selectedTool && isInventory(selectedTool) && result && cachedAgeSec === null && (
|
||||
<p style={{ color: "#666", fontSize: "0.85rem", marginTop: "0.5rem" }}>
|
||||
fresh result (live path — not cached)
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* Result rendering (JSON tree, isError flag surfaced). */}
|
||||
{result && (
|
||||
<div style={{ marginTop: "0.75rem", border: `2px solid ${result.isError ? "#dc2626" : "#16a34a"}`, borderRadius: "0.25rem", padding: "0.75rem" }}>
|
||||
<strong style={{ color: result.isError ? "#dc2626" : "#16a34a" }}>
|
||||
{result.isError ? "✗ isError: true" : "✓ result"}
|
||||
</strong>
|
||||
<pre style={{ background: "#f6f8fa", padding: "0.5rem", fontSize: "0.8rem", overflowX: "auto", marginTop: "0.5rem" }}>
|
||||
{JSON.stringify(result.content, null, 2)}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* SSE event trace. */}
|
||||
{trace.length > 0 && (
|
||||
<div style={{ marginTop: "0.75rem" }}>
|
||||
<h3>SSE trace ({trace.length} events)</h3>
|
||||
<ul style={{ listStyle: "none", padding: 0, fontFamily: "monospace", fontSize: "0.8rem" }}>
|
||||
{trace.map((e, i) => (
|
||||
<li key={i} style={{ padding: "0.2rem 0", borderBottom: "1px solid #f0f0f0" }}>
|
||||
<span style={{ color: "#999" }}>[{e.event}]</span>{" "}
|
||||
<span style={{ color: "#666" }}>{e.id}</span>{" "}
|
||||
<code>{typeof e.data === "string" ? e.data : JSON.stringify(e.data)}</code>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* dashboard/test-call helpers — pure (testable) logic for the Test-Call UI
|
||||
* (Wave J Task 3). Extracted from the React component so the validation,
|
||||
* arg coercion, and adapter-type grouping can be unit-tested without a DOM.
|
||||
*
|
||||
* The TestCallConsole component imports these and stays thin (renders state).
|
||||
*/
|
||||
|
||||
import type { ToolView, AdapterView } from "./_lib.js";
|
||||
|
||||
/** The adapter type for a tool is the prefix before the first dot. */
|
||||
export function adapterTypeOf(toolName: string): string {
|
||||
return toolName.split(".")[0] ?? "";
|
||||
}
|
||||
|
||||
/** Whether a tool is an inventory call (list_* — gets the staleness indicator). */
|
||||
export function isInventory(toolName: string): boolean {
|
||||
return toolName.includes(".list_");
|
||||
}
|
||||
|
||||
/** Group tools by adapter type for the capability picker. */
|
||||
export function groupToolsByType(tools: ToolView[]): Record<string, ToolView[]> {
|
||||
const out: Record<string, ToolView[]> = {};
|
||||
for (const t of tools) {
|
||||
const k = adapterTypeOf(t.name);
|
||||
(out[k] ??= []).push(t);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Group adapters by type for the target picker (REQ-024). */
|
||||
export function groupAdaptersByType(adapters: AdapterView[]): Record<string, AdapterView[]> {
|
||||
const out: Record<string, AdapterView[]> = {};
|
||||
for (const a of adapters) {
|
||||
(out[a.adapterType] ??= []).push(a);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate args against the tool's JSON Schema inputSchema (Edge 4 → the
|
||||
* broker also validates; this is the client-side pre-check for fast feedback).
|
||||
* Returns null on success or an error message on failure.
|
||||
*/
|
||||
export function validateArgsLocal(tool: ToolView, raw: Record<string, string>): string | null {
|
||||
const schema = tool.inputSchema;
|
||||
const required = schema.required ?? [];
|
||||
for (const key of required) {
|
||||
const v = raw[key];
|
||||
if (v === undefined || v.trim() === "") {
|
||||
return `Missing required argument '${key}'.`;
|
||||
}
|
||||
}
|
||||
const props = schema.properties ?? {};
|
||||
for (const [key, decl] of Object.entries(props)) {
|
||||
const v = raw[key];
|
||||
if (v === undefined || v.trim() === "") continue;
|
||||
const t = decl.type;
|
||||
if (t === "integer" || t === "number") {
|
||||
const n = Number(v);
|
||||
if (!Number.isFinite(n)) return `Argument '${key}' must be a ${t} (got '${v}').`;
|
||||
if (t === "integer" && !Number.isInteger(n)) return `Argument '${key}' must be an integer.`;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce string args to the declared types for the invoke payload. Empty
|
||||
* strings are omitted (the broker's `additionalProperties: false` would reject
|
||||
* unknown keys, but empty optionals are fine to drop).
|
||||
*/
|
||||
export function coerceArgs(tool: ToolView, raw: Record<string, string>): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
const props = tool.inputSchema.properties ?? {};
|
||||
for (const [key, val] of Object.entries(raw)) {
|
||||
if (val.trim() === "") continue;
|
||||
const t = props[key]?.type;
|
||||
if (t === "integer" || t === "number") {
|
||||
const n = Number(val);
|
||||
out[key] = t === "integer" ? Math.trunc(n) : n;
|
||||
} else if (t === "boolean") {
|
||||
out[key] = val === "true";
|
||||
} else {
|
||||
out[key] = val;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Whether the target picker should be shown (≥2 same-type adapters, REQ-024). */
|
||||
export function needsTargetPicker(adapters: AdapterView[], toolName: string): boolean {
|
||||
const type = adapterTypeOf(toolName);
|
||||
return adapters.filter((a) => a.adapterType === type).length >= 2;
|
||||
}
|
||||
|
||||
/** Parse an SSE `tool_result` event's data into the MCP result shape. */
|
||||
export function parseToolResult(data: unknown): { content: unknown; isError: boolean; cachedAgeSec: number | null } {
|
||||
const r = data as { content?: unknown; isError?: boolean; cached?: { ageSec?: number } } | null;
|
||||
if (r === null || r === undefined || typeof r !== "object") {
|
||||
return { content: null, isError: false, cachedAgeSec: null };
|
||||
}
|
||||
return {
|
||||
content: r.content ?? null,
|
||||
isError: !!r.isError,
|
||||
cachedAgeSec: typeof r.cached?.ageSec === "number" ? r.cached.ageSec : null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* dashboard/test-call helper — server-side fetch of /api/mcp/tools (the closed
|
||||
* 9-tool set, MCP `tools/list` facade, REQ-015) + the configured adapters
|
||||
* (for the target picker, REQ-024).
|
||||
*
|
||||
* Server components call this to render the Test-Call UI. Hits the API gateway
|
||||
* (cookie forwarded) so the dashboard never bypasses RLS / RBAC. Returns null
|
||||
* on 401 (the caller redirects to /login).
|
||||
*/
|
||||
|
||||
import { headers, cookies } from "next/headers";
|
||||
|
||||
/** A tool from GET /api/mcp/tools (MCP tools/list shape). */
|
||||
export interface ToolView {
|
||||
name: string;
|
||||
description: string;
|
||||
inputSchema: {
|
||||
type: "object";
|
||||
properties?: Record<string, { type?: string; description?: string }>;
|
||||
required?: string[];
|
||||
[k: string]: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
export async function getTools(): Promise<ToolView[] | null> {
|
||||
const cookieStore = await cookies();
|
||||
const sessionCookie = cookieStore.get("coreci_session")?.value;
|
||||
if (!sessionCookie) return null;
|
||||
|
||||
const h = await headers();
|
||||
const host = h.get("host") ?? "localhost:3000";
|
||||
const proto = h.get("x-forwarded-proto") ?? "http";
|
||||
|
||||
const res = await fetch(`${proto}://${host}/api/mcp/tools`, {
|
||||
headers: { cookie: `coreci_session=${sessionCookie}` },
|
||||
cache: "no-store",
|
||||
});
|
||||
if (res.status === 401 || res.status === 403) return null;
|
||||
if (res.status !== 200) return [];
|
||||
const body = (await res.json()) as { tools: ToolView[] };
|
||||
return body.tools;
|
||||
}
|
||||
|
||||
/** Re-export the adapters fetcher for the target picker (same gateway path). */
|
||||
export { getAdapters } from "../settings/adapters/_lib.js";
|
||||
export type { AdapterView } from "../settings/adapters/_lib.js";
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* /dashboard/test-call — Test-Call UI (Wave J Task 3, M2 Surface 2).
|
||||
*
|
||||
* Server shell: fetches the closed 9-tool set from /api/mcp/tools (MCP
|
||||
* tools/list facade, REQ-015) + the configured adapters from /api/mcp/adapter
|
||||
* (for the target picker, REQ-024). Renders the TestCallConsole client
|
||||
* component which consumes the SSE stream (Task 4).
|
||||
*
|
||||
* The Test-Call UI is the M2 operator surface: pick a capability, enter args,
|
||||
* invoke, watch the SSE stream render results. Inventory calls (list_*) show
|
||||
* a "cached Xs ago" staleness indicator; live calls show fresh results.
|
||||
*/
|
||||
|
||||
import { getMe } from "../me.js";
|
||||
import { getTools, getAdapters } from "./_lib.js";
|
||||
import { TestCallConsole } from "./TestCallConsole.js";
|
||||
|
||||
export default async function TestCallPage() {
|
||||
const me = await getMe();
|
||||
if (!me) {
|
||||
return (
|
||||
<main style={{ fontFamily: "system-ui", maxWidth: "32rem", margin: "4rem auto", padding: "0 1rem" }}>
|
||||
<h1>Test-Call</h1>
|
||||
<p>You are not signed in.</p>
|
||||
<p>
|
||||
<a href="/login">
|
||||
<button style={{ padding: "0.6rem 1.2rem", font: "inherit" }}>Go to login</button>
|
||||
</a>
|
||||
</p>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
const [tools, adapters] = await Promise.all([getTools(), getAdapters()]);
|
||||
|
||||
return (
|
||||
<main style={{ fontFamily: "system-ui", maxWidth: "60rem", margin: "2rem auto", padding: "0 1rem" }}>
|
||||
<header style={{ display: "flex", justifyContent: "space-between", alignItems: "baseline" }}>
|
||||
<h1>Test-Call</h1>
|
||||
<span style={{ color: "#666", fontSize: "0.85rem" }}>
|
||||
{me.role} · tenant {me.tenantId.slice(0, 8)}
|
||||
</span>
|
||||
</header>
|
||||
|
||||
<p style={{ color: "#666" }}>
|
||||
Invoke a capability from the closed 9-tool set (REQ-015). The broker mints a ULID correlation
|
||||
ID, returns a stream URL, and the SSE stream renders results as they arrive (<100ms chunk
|
||||
delivery). Inventory calls (list_*) show a "cached Xs ago" staleness indicator.
|
||||
</p>
|
||||
|
||||
{(!tools || tools.length === 0) && (
|
||||
<p style={{ color: "#996" }}>
|
||||
No tools available. Configure an adapter in{" "}
|
||||
<a href="/dashboard/settings/adapters">Settings → Adapters</a>.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{tools && tools.length > 0 && (
|
||||
<TestCallConsole
|
||||
tools={tools}
|
||||
adapters={adapters ?? []}
|
||||
/>
|
||||
)}
|
||||
</main>
|
||||
);
|
||||
}
|
||||
@@ -6,6 +6,14 @@ import tseslint from "typescript-eslint";
|
||||
// pattern used by the other workspace packages. The Next.js ESLint plugin
|
||||
// (`eslint-plugin-next`) is not added here to avoid an extra devDependency; the
|
||||
// shared js.configs.recommended + tseslint recommended rules cover the TS code.
|
||||
//
|
||||
// [G-018, R-008] Import guard: `@coreci/llm-mock` is a CI-only devDependency.
|
||||
// It MUST NOT be imported from prod code (apps/control-plane/app/**, the route
|
||||
// handlers + React server components). The no-restricted-imports rule below
|
||||
// bans it in app/** and lib/** (the prod runtime path); tests/** are exempt
|
||||
// (the smoke imports the mock directly). A build-time grep in the root
|
||||
// `build` script additionally fails the prod build if `llm-mock` appears in
|
||||
// `.next/` output — defense-in-depth against a stray import slipping through.
|
||||
export default tseslint.config(
|
||||
js.configs.recommended,
|
||||
...tseslint.configs.recommended,
|
||||
@@ -18,6 +26,41 @@ export default tseslint.config(
|
||||
"@typescript-eslint/no-empty-function": "off",
|
||||
},
|
||||
},
|
||||
// [G-018, R-008] Prod import guard: ban @coreci/llm-mock from the runtime path.
|
||||
// The mock is CI-only (a devDependency); a prod import would bundle a fake
|
||||
// LLM into the real control plane. Tests may import it (the smoke uses it).
|
||||
{
|
||||
files: ["app/**", "lib/**", "ws-server.ts"],
|
||||
rules: {
|
||||
"no-restricted-imports": [
|
||||
"error",
|
||||
{
|
||||
paths: [
|
||||
{
|
||||
name: "@coreci/llm-mock",
|
||||
message:
|
||||
"@coreci/llm-mock is a CI-only devDependency (R-008). It MUST NOT be imported from prod runtime code (app/**, lib/**). The LLM smoke is a CI test that imports the mock directly.",
|
||||
},
|
||||
{
|
||||
name: "@coreci/llm-mock/server",
|
||||
message:
|
||||
"@coreci/llm-mock/server is CI-only (R-008). Import the patterns/retry modules from tests/** instead; prod runtime must not depend on the mock.",
|
||||
},
|
||||
{
|
||||
name: "@coreci/llm-mock/patterns",
|
||||
message:
|
||||
"@coreci/llm-mock/patterns is CI-only (R-008). Prod runtime must not depend on the mock LLM's pattern matcher.",
|
||||
},
|
||||
{
|
||||
name: "@coreci/llm-mock/retry",
|
||||
message:
|
||||
"@coreci/llm-mock/retry is CI-only (R-008). Prod runtime must not depend on the mock's retry policy.",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: [
|
||||
"dist/**",
|
||||
@@ -25,6 +68,7 @@ export default tseslint.config(
|
||||
".next/**",
|
||||
"coverage/**",
|
||||
"next-env.d.ts",
|
||||
"tests/**",
|
||||
],
|
||||
},
|
||||
);
|
||||
@@ -25,6 +25,7 @@
|
||||
"ws": "^8.18.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@coreci/llm-mock": "workspace:*",
|
||||
"@eslint/js": "9.39.5",
|
||||
"@types/node": "^22.0.0",
|
||||
"@types/react": "^19.0.0",
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* adapters _help test (Wave J Task 2, G-014) — closed-tool-set gap docs.
|
||||
* Asserts the G-014 binding fix: each adapter type's help text documents the
|
||||
* M2 limitations so operators aren't surprised post-ship.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
ADAPTER_HELP_TEXT,
|
||||
ADAPTER_TYPES,
|
||||
SSH_GAPS,
|
||||
PROXMOX_GAPS,
|
||||
GITHUB_GAPS,
|
||||
GITEA_GAPS,
|
||||
type AdapterTypeName,
|
||||
} from "../app/dashboard/settings/adapters/_help.js";
|
||||
|
||||
describe("Settings → Adapters help text (G-014)", () => {
|
||||
it("ADAPTER_TYPES is the closed 4-type set (no custom adapter)", () => {
|
||||
expect(ADAPTER_TYPES).toEqual(["proxmox", "ssh", "github", "gitea"]);
|
||||
});
|
||||
|
||||
it("each adapter type has full help text (base + gaps)", () => {
|
||||
for (const t of ADAPTER_TYPES) {
|
||||
expect(typeof ADAPTER_HELP_TEXT[t as AdapterTypeName]).toBe("string");
|
||||
expect(ADAPTER_HELP_TEXT[t as AdapterTypeName].length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("SSH gaps document the 6-command subset + deferred commands", () => {
|
||||
expect(SSH_GAPS).toContain("G-014");
|
||||
expect(SSH_GAPS).toContain("6 diagnostic commands");
|
||||
expect(SSH_GAPS).toContain("`ps`");
|
||||
expect(SSH_GAPS).toContain("`ss`");
|
||||
expect(SSH_GAPS).toContain("`top`");
|
||||
expect(SSH_GAPS).toContain("`ip`");
|
||||
expect(SSH_GAPS).toContain("v1.2+");
|
||||
});
|
||||
|
||||
it("Proxmox gaps document list_vms requires node + list_nodes deferred", () => {
|
||||
expect(PROXMOX_GAPS).toContain("G-014");
|
||||
expect(PROXMOX_GAPS).toContain("list_vms");
|
||||
expect(PROXMOX_GAPS).toContain("node");
|
||||
expect(PROXMOX_GAPS).toContain("list_nodes");
|
||||
expect(PROXMOX_GAPS).toContain("v1.2+");
|
||||
});
|
||||
|
||||
it("GitHub gaps document 100-repo limit + pagination/PR deferred", () => {
|
||||
expect(GITHUB_GAPS).toContain("G-014");
|
||||
expect(GITHUB_GAPS).toContain("100 repos");
|
||||
expect(GITHUB_GAPS).toContain("Pagination");
|
||||
expect(GITHUB_GAPS).toContain("PR lists");
|
||||
expect(GITHUB_GAPS).toContain("v1.2+");
|
||||
});
|
||||
|
||||
it("Gitea gaps document get_workflow_run deferred", () => {
|
||||
expect(GITEA_GAPS).toContain("G-014");
|
||||
expect(GITEA_GAPS).toContain("get_workflow_run");
|
||||
expect(GITEA_GAPS).toContain("v1.2+");
|
||||
});
|
||||
|
||||
it("the full help text includes BOTH base + gaps for each type", () => {
|
||||
// SSH: base mentions "Relay Agent"; gaps mention "G-014".
|
||||
expect(ADAPTER_HELP_TEXT.ssh).toContain("Relay Agent");
|
||||
expect(ADAPTER_HELP_TEXT.ssh).toContain("G-014");
|
||||
// Proxmox: base mentions "PVEAuditor"; gaps mention "list_vms".
|
||||
expect(ADAPTER_HELP_TEXT.proxmox).toContain("PVEAuditor");
|
||||
expect(ADAPTER_HELP_TEXT.proxmox).toContain("G-014");
|
||||
// GitHub: base mentions "fine-grained"; gaps mention "100 repos".
|
||||
expect(ADAPTER_HELP_TEXT.github).toContain("fine-grained");
|
||||
expect(ADAPTER_HELP_TEXT.github).toContain("G-014");
|
||||
// Gitea: base mentions "read:repository"; gaps mention "get_workflow_run".
|
||||
expect(ADAPTER_HELP_TEXT.gitea).toContain("read:repository");
|
||||
expect(ADAPTER_HELP_TEXT.gitea).toContain("G-014");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* test-call helpers test (Wave J Task 3) — pure logic for the Test-Call UI.
|
||||
* Covers arg validation, coercion, grouping, staleness, target-picker logic.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
adapterTypeOf,
|
||||
isInventory,
|
||||
groupToolsByType,
|
||||
groupAdaptersByType,
|
||||
validateArgsLocal,
|
||||
coerceArgs,
|
||||
needsTargetPicker,
|
||||
parseToolResult,
|
||||
} from "../app/dashboard/test-call/_helpers.js";
|
||||
import type { ToolView, AdapterView } from "../app/dashboard/test-call/_lib.js";
|
||||
|
||||
const listRepos: ToolView = {
|
||||
name: "github.list_repos",
|
||||
description: "list repos",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
};
|
||||
const getVmStatus: ToolView = {
|
||||
name: "proxmox.get_vm_status",
|
||||
description: "vm status",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
node: { type: "string", description: "the node" },
|
||||
vmid: { type: "integer", description: "the vmid" },
|
||||
},
|
||||
required: ["node", "vmid"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
};
|
||||
const recentRuns: ToolView = {
|
||||
name: "github.get_recent_ci_runs",
|
||||
description: "recent runs",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
owner: { type: "string" },
|
||||
repo: { type: "string" },
|
||||
per_page: { type: "integer" },
|
||||
},
|
||||
required: ["owner", "repo"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
};
|
||||
|
||||
describe("adapterTypeOf", () => {
|
||||
it("returns the prefix before the first dot", () => {
|
||||
expect(adapterTypeOf("github.list_repos")).toBe("github");
|
||||
expect(adapterTypeOf("proxmox.get_vm_status")).toBe("proxmox");
|
||||
expect(adapterTypeOf("ssh.run_whitelisted_command")).toBe("ssh");
|
||||
});
|
||||
it("returns the whole string when no dot (degenerate — tools always have a dot)", () => {
|
||||
expect(adapterTypeOf("bogus")).toBe("bogus");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isInventory", () => {
|
||||
it("true for list_* tools", () => {
|
||||
expect(isInventory("github.list_repos")).toBe(true);
|
||||
expect(isInventory("proxmox.list_vms")).toBe(true);
|
||||
expect(isInventory("gitea.list_repos")).toBe(true);
|
||||
});
|
||||
it("false for live tools", () => {
|
||||
expect(isInventory("github.get_recent_ci_runs")).toBe(false);
|
||||
expect(isInventory("proxmox.get_vm_status")).toBe(false);
|
||||
expect(isInventory("ssh.run_whitelisted_command")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("groupToolsByType", () => {
|
||||
it("groups tools by their adapter-type prefix", () => {
|
||||
const out = groupToolsByType([listRepos, getVmStatus, recentRuns]);
|
||||
expect(out.github).toEqual([listRepos, recentRuns]);
|
||||
expect(out.proxmox).toEqual([getVmStatus]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("groupAdaptersByType", () => {
|
||||
it("groups adapters by adapterType", () => {
|
||||
const adapters: AdapterView[] = [
|
||||
{ id: "1", adapterType: "proxmox", targetId: "pve1", config: {}, validated: true },
|
||||
{ id: "2", adapterType: "proxmox", targetId: "pve2", config: {}, validated: true },
|
||||
{ id: "3", adapterType: "github", targetId: "gh", config: {}, validated: true },
|
||||
];
|
||||
const out = groupAdaptersByType(adapters);
|
||||
expect(out.proxmox).toHaveLength(2);
|
||||
expect(out.github).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateArgsLocal", () => {
|
||||
it("returns null when all required args are present", () => {
|
||||
expect(validateArgsLocal(getVmStatus, { node: "pve1", vmid: "100" })).toBeNull();
|
||||
});
|
||||
it("returns an error when a required arg is missing", () => {
|
||||
expect(validateArgsLocal(getVmStatus, { node: "pve1" })).toContain("Missing required argument 'vmid'");
|
||||
});
|
||||
it("returns an error when a required arg is empty", () => {
|
||||
expect(validateArgsLocal(getVmStatus, { node: "pve1", vmid: "" })).toContain("Missing required argument 'vmid'");
|
||||
});
|
||||
it("returns an error when an integer arg is not a number", () => {
|
||||
expect(validateArgsLocal(getVmStatus, { node: "pve1", vmid: "abc" })).toContain("must be a integer");
|
||||
});
|
||||
it("returns an error when an integer arg is a float", () => {
|
||||
expect(validateArgsLocal(getVmStatus, { node: "pve1", vmid: "1.5" })).toContain("must be an integer");
|
||||
});
|
||||
it("returns null for optional args omitted", () => {
|
||||
expect(validateArgsLocal(recentRuns, { owner: "o", repo: "r" })).toBeNull();
|
||||
});
|
||||
it("returns null for a no-arg tool", () => {
|
||||
expect(validateArgsLocal(listRepos, {})).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("coerceArgs", () => {
|
||||
it("coerces integers", () => {
|
||||
expect(coerceArgs(getVmStatus, { node: "pve1", vmid: "100" })).toEqual({ node: "pve1", vmid: 100 });
|
||||
});
|
||||
it("truncates floats for integer fields", () => {
|
||||
expect(coerceArgs(getVmStatus, { node: "pve1", vmid: "100.9" })).toEqual({ node: "pve1", vmid: 100 });
|
||||
});
|
||||
it("keeps strings as strings", () => {
|
||||
expect(coerceArgs(recentRuns, { owner: "o", repo: "r" })).toEqual({ owner: "o", repo: "r" });
|
||||
});
|
||||
it("omits empty optional args", () => {
|
||||
expect(coerceArgs(recentRuns, { owner: "o", repo: "r", per_page: "" })).toEqual({ owner: "o", repo: "r" });
|
||||
});
|
||||
it("coerces optional integers when present", () => {
|
||||
expect(coerceArgs(recentRuns, { owner: "o", repo: "r", per_page: "50" })).toEqual({ owner: "o", repo: "r", per_page: 50 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("needsTargetPicker", () => {
|
||||
it("true when ≥2 same-type adapters", () => {
|
||||
const adapters: AdapterView[] = [
|
||||
{ id: "1", adapterType: "proxmox", targetId: "a", config: {}, validated: true },
|
||||
{ id: "2", adapterType: "proxmox", targetId: "b", config: {}, validated: true },
|
||||
];
|
||||
expect(needsTargetPicker(adapters, "proxmox.list_vms")).toBe(true);
|
||||
});
|
||||
it("false when only 1 same-type adapter", () => {
|
||||
const adapters: AdapterView[] = [
|
||||
{ id: "1", adapterType: "proxmox", targetId: "a", config: {}, validated: true },
|
||||
{ id: "2", adapterType: "github", targetId: "b", config: {}, validated: true },
|
||||
];
|
||||
expect(needsTargetPicker(adapters, "proxmox.list_vms")).toBe(false);
|
||||
});
|
||||
it("false when no adapters", () => {
|
||||
expect(needsTargetPicker([], "proxmox.list_vms")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseToolResult", () => {
|
||||
it("parses a success result", () => {
|
||||
const data = { content: [{ type: "text", text: "ok" }], isError: false };
|
||||
expect(parseToolResult(data)).toEqual({ content: data.content, isError: false, cachedAgeSec: null });
|
||||
});
|
||||
it("parses an error result", () => {
|
||||
const data = { content: [{ type: "text", text: "boom" }], isError: true };
|
||||
const r = parseToolResult(data);
|
||||
expect(r.isError).toBe(true);
|
||||
});
|
||||
it("parses a cached staleness age", () => {
|
||||
const data = { content: [], isError: false, cached: { ageSec: 42 } };
|
||||
expect(parseToolResult(data).cachedAgeSec).toBe(42);
|
||||
});
|
||||
it("returns null cachedAgeSec when no cached field", () => {
|
||||
const data = { content: [], isError: false };
|
||||
expect(parseToolResult(data).cachedAgeSec).toBeNull();
|
||||
});
|
||||
it("handles a malformed payload gracefully", () => {
|
||||
expect(parseToolResult(null)).toEqual({ content: null, isError: false, cachedAgeSec: null });
|
||||
expect(parseToolResult("not an object")).toEqual({ content: null, isError: false, cachedAgeSec: null });
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,15 @@ export default defineConfig({
|
||||
testTimeout: 30000,
|
||||
coverage: {
|
||||
provider: "v8",
|
||||
include: ["lib/**/*.ts", "ws-server.ts"],
|
||||
include: [
|
||||
"lib/**/*.ts",
|
||||
"ws-server.ts",
|
||||
// Wave J UI pure logic (extracted from React components for testability).
|
||||
"app/dashboard/settings/adapters/_help.ts",
|
||||
"app/dashboard/settings/adapters/_lib.ts",
|
||||
"app/dashboard/test-call/_helpers.ts",
|
||||
"app/dashboard/test-call/_lib.ts",
|
||||
],
|
||||
reporter: ["text", "json"],
|
||||
},
|
||||
},
|
||||
|
||||
+3
-1
@@ -12,12 +12,14 @@
|
||||
"test": "pnpm -r test",
|
||||
"migrate": "pnpm --filter @coreci/db migrate",
|
||||
"test:pen": "pnpm --filter @coreci/db test:pen",
|
||||
"test:conformance": "vitest run --config vitest.conformance.config.ts"
|
||||
"test:conformance": "vitest run --config vitest.conformance.config.ts",
|
||||
"check:llm-mock-guard": "node scripts/check-llm-mock-guard.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@coreci/llm-mock": "workspace:*",
|
||||
"@coreci/mcp": "workspace:*",
|
||||
"@eslint/js": "9.39.5",
|
||||
"@vitest/coverage-v8": "2.1.9",
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
-- packages/db/scripts/setup-ci-roles.sql — CI Postgres 16 role setup (R-009).
|
||||
--
|
||||
-- Run ONCE at CI job startup (before migrations) against the Postgres 16
|
||||
-- service container. Creates the two roles the M1+M2 app uses:
|
||||
--
|
||||
-- coreci_app — the runtime role. NOBYPASSRLS (RLS enforced even though
|
||||
-- the role owns no tables; the app connects as this role
|
||||
-- and every tenant-scoped query goes through withTenant,
|
||||
-- which sets app.tenant_id). This is the role RLS is tested
|
||||
-- against in the CI pen test (G-022, R-009).
|
||||
-- migrator — the migration role. BYPASSRLS so migrations can CREATE
|
||||
-- tables / policies / indexes that the app role cannot.
|
||||
-- `pnpm migrate` connects as this role in CI.
|
||||
--
|
||||
-- The CI job connects to the Postgres 16 service container as the `postgres`
|
||||
-- superuser and runs this script, then runs `pnpm migrate` as `migrator`,
|
||||
-- then runs the test suite as `coreci_app` (the test harness sets
|
||||
-- DATABASE_URL=postgres://coreci_app:<pwd>@localhost:5432/...).
|
||||
--
|
||||
-- This script is idempotent (CREATE ROLE IF NOT EXISTS + ALTER). Passwords
|
||||
-- are CI-only constants (the Postgres container is ephemeral; no prod
|
||||
-- secrets). The CI workflow sets these via the connection string.
|
||||
|
||||
-- The runtime role (NO BYPASSRLS — RLS enforced, the load-bearing CI test).
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreci_app') THEN
|
||||
CREATE ROLE coreci_app WITH LOGIN PASSWORD 'coreci_app_ci' NOBYPASSRLS;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- The migration role (BYPASSRLS — runs DDL the app role cannot).
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'migrator') THEN
|
||||
CREATE ROLE migrator WITH LOGIN PASSWORD 'migrator_ci' BYPASSRLS;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Grant schema + table privileges. The migrator creates tables; the app
|
||||
-- role gets DML (INSERT/SELECT/UPDATE/DELETE) on the tables it owns. RLS
|
||||
-- policies enforce tenant scoping (the WITH CHECK clause blocks cross-tenant
|
||||
-- writes even though the role has the DML privilege).
|
||||
GRANT USAGE ON SCHEMA public TO coreci_app, migrator;
|
||||
GRANT CREATE ON SCHEMA public TO migrator;
|
||||
|
||||
-- The app role gets DML on all current + future tables in public. The
|
||||
-- migrations CREATE TABLE with no explicit owner (migrator owns them); the
|
||||
-- app role connects and queries/inserts under RLS.
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO coreci_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public
|
||||
GRANT USAGE ON SEQUENCES TO coreci_app;
|
||||
|
||||
-- For tables created by migrations BEFORE this grant took effect, apply
|
||||
-- explicitly (the CI container runs this AFTER migrations in some flows;
|
||||
-- the GRANT below covers already-existing tables). Idempotent.
|
||||
DO $$
|
||||
DECLARE
|
||||
t text;
|
||||
BEGIN
|
||||
FOR t IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' LOOP
|
||||
EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.%I TO coreci_app', t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
-- Create the CI database (the service container's default is `postgres`;
|
||||
-- the CI workflow creates a separate `coreci_ci` database for the test run).
|
||||
-- This is optional — the workflow may set DATABASE_URL to point at any DB.
|
||||
SELECT 'CREATE DATABASE coreci_ci OWNER migrator'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'coreci_ci')\gexec
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* Cross-tenant isolation pen test — REQ-039, R-007.
|
||||
* Cross-tenant isolation pen test — REQ-039, R-007, R-009, G-022.
|
||||
*
|
||||
* Creates two tenants (T1, T2), each with a target. Issues queries as T1
|
||||
* attempting to read T2's data. Asserts every query returns zero T2 rows.
|
||||
@@ -9,9 +9,19 @@
|
||||
* tenant scoping as a defense-in-depth backstop. This test verifies the
|
||||
* APPLICATION-LAYER isolation that `withTenant` provides: every tenant-scoped
|
||||
* query runs inside withTenant, which sets app.tenant_id and scopes all queries.
|
||||
* A separate prod integration test (runs against real Postgres at M1 review)
|
||||
* verifies the RLS policies themselves enforce scoping even if a query
|
||||
* bypasses withTenant.
|
||||
*
|
||||
* ─── DB_MODE parameterization (G-022, R-009) ──────────────────────────────
|
||||
* The test runs in two modes:
|
||||
* - DB_MODE unset (default): PGlite — verifies app-layer withTenant scoping
|
||||
* (the placeholder WITH CHECK assertion stays a no-op; PGlite doesn't
|
||||
* enforce RLS WITH CHECK).
|
||||
* - DB_MODE=pg (CI test-postgres job): real Postgres 16 service container
|
||||
* with `setup-ci-roles.sql` (coreci_app NOBYPASSRLS, migrator BYPASSRLS).
|
||||
* The WITH CHECK assertion below is REAL: a cross-tenant INSERT under
|
||||
* withTenant(T1) with tenant_id=T2 is REJECTED by the RLS policy's WITH
|
||||
* CHECK clause (this is the R-009 deliverable — the M1 placeholder
|
||||
* `expect(true).toBe(true)` is replaced by a real RLS rejection assertion
|
||||
* when DB_MODE=pg).
|
||||
*
|
||||
* The withTenant + RLS model: withTenant is the primary enforcement (every
|
||||
* API call goes through it); RLS is the backstop (catches any bypass in prod).
|
||||
@@ -22,42 +32,71 @@ import { createDb } from "../../src/create-db.js";
|
||||
import { setDbClient, withTenant, getTenantContext } from "../../src/withTenant.js";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { readdir } from "node:fs/promises";
|
||||
|
||||
const T1 = "00000000-0000-0000-0000-000000000001";
|
||||
const T2 = "00000000-0000-0000-0000-000000000002";
|
||||
const U1 = "00000000-0000-0000-0000-000000000011";
|
||||
const U2 = "00000000-0000-0000-0000-000000000012";
|
||||
|
||||
/** DB_MODE env: 'pg' → real Postgres 16 (CI); unset → PGlite (dev). [G-022] */
|
||||
const isPgMode = process.env.DB_MODE === "pg";
|
||||
|
||||
describe("cross-tenant isolation (REQ-039 pen test)", () => {
|
||||
beforeAll(async () => {
|
||||
const db = await createDb({ mode: "pglite" });
|
||||
const db = await createDb(isPgMode ? { mode: "pg" } : { mode: "pglite" });
|
||||
setDbClient(db);
|
||||
const sql = await readFile(
|
||||
join(import.meta.dirname, "..", "..", "migrations", "0001_init.sql"),
|
||||
"utf8",
|
||||
);
|
||||
await db.exec(sql);
|
||||
|
||||
// Run ALL migrations (M1 0001_init + 0002_sessions + M2 0003_mcp_adapters)
|
||||
// so the schema matches prod. In PG mode the CI job has already run
|
||||
// `pnpm migrate` as the migrator role; in PGlite we run them in-process
|
||||
// (PGlite is a single role, BYPASSRLS not modeled — RLS still applies).
|
||||
if (!isPgMode) {
|
||||
const migrationsDir = join(import.meta.dirname, "..", "..", "migrations");
|
||||
const files = (await readdir(migrationsDir)).filter((f) => f.endsWith(".sql")).sort();
|
||||
for (const file of files) {
|
||||
const sql = await readFile(join(migrationsDir, file), "utf8");
|
||||
await db.exec(sql);
|
||||
}
|
||||
} else {
|
||||
// PG mode: the CI job ran migrations as `migrator` (BYPASSRLS) before
|
||||
// the test. The test connects as `coreci_app` (NOBYPASSRLS) so RLS is
|
||||
// enforced. Seed data must use withTenant (the app role cannot insert
|
||||
// outside a tenant scope — RLS WITH CHECK rejects it).
|
||||
}
|
||||
// Seed two tenants + users + memberships + one target each.
|
||||
// In PGlite RLS is not enforced on SELECT (0.5.7 limitation); we seed
|
||||
// directly and rely on withTenant's explicit scoping for the test.
|
||||
// In PG mode (coreci_app role), the tenants/users/memberships tables are
|
||||
// NOT tenant-scoped (they're the bootstrap tables), so direct inserts
|
||||
// work. The targets table IS tenant-scoped — seed via withTenant so the
|
||||
// RLS WITH CHECK passes.
|
||||
await db.query(
|
||||
`INSERT INTO tenants (id, name) VALUES ($1,'T1'), ($2,'T2')`,
|
||||
`INSERT INTO tenants (id, name) VALUES ($1,'T1'), ($2,'T2') ON CONFLICT DO NOTHING`,
|
||||
[T1, T2],
|
||||
);
|
||||
await db.query(
|
||||
`INSERT INTO users (id, email) VALUES ($1,'u1@t1.test'), ($2,'u2@t2.test')`,
|
||||
`INSERT INTO users (id, email) VALUES ($1,'u1@t1.test'), ($2,'u2@t2.test') ON CONFLICT DO NOTHING`,
|
||||
[U1, U2],
|
||||
);
|
||||
await db.query(
|
||||
`INSERT INTO tenant_memberships (tenant_id, user_id, role) VALUES ($1,$2,'admin'), ($3,$4,'admin')`,
|
||||
`INSERT INTO tenant_memberships (tenant_id, user_id, role) VALUES ($1,$2,'admin'), ($3,$4,'admin') ON CONFLICT DO NOTHING`,
|
||||
[T1, U1, T2, U2],
|
||||
);
|
||||
await db.query(
|
||||
`INSERT INTO targets (tenant_id, hostname, os_name, os_version, agent_version) VALUES
|
||||
($1,'t1-host','ubuntu','24.04','0.0.1'),
|
||||
($2,'t2-host','debian','12','0.0.1')`,
|
||||
[T1, T2],
|
||||
);
|
||||
// Seed targets via withTenant (RLS WITH CHECK requires the row's
|
||||
// tenant_id to match the current app.tenant_id).
|
||||
await withTenant(T1, async (c) => {
|
||||
await c.query(
|
||||
`INSERT INTO targets (tenant_id, hostname, os_name, os_version, agent_version) VALUES
|
||||
($1,'t1-host','ubuntu','24.04','0.0.1') ON CONFLICT DO NOTHING`,
|
||||
[T1],
|
||||
);
|
||||
});
|
||||
await withTenant(T2, async (c) => {
|
||||
await c.query(
|
||||
`INSERT INTO targets (tenant_id, hostname, os_name, os_version, agent_version) VALUES
|
||||
($1,'t2-host','debian','12','0.0.1') ON CONFLICT DO NOTHING`,
|
||||
[T2],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("T1 sees only T1 targets, not T2", async () => {
|
||||
@@ -108,15 +147,48 @@ describe("cross-tenant isolation (REQ-039 pen test)", () => {
|
||||
expect(ctx).toBe(null); // no active tenant context → prod RLS returns nothing
|
||||
});
|
||||
|
||||
it("T1 cannot INSERT a target row for T2 (application-layer check)", async () => {
|
||||
// In prod, RLS WITH CHECK blocks this. In PGlite (no RLS enforcement),
|
||||
// we verify the application layer rejects cross-tenant inserts: the
|
||||
// withTenant scope is T1, so inserting with tenant_id = T2 is a violation
|
||||
// the application must prevent. This test asserts the insert completes
|
||||
// (PGlite doesn't enforce RLS WITH CHECK) but documents that prod RLS
|
||||
// would reject it. The application's insert paths always use the scoped
|
||||
// tenant_id from withTenant, never a user-supplied tenant_id.
|
||||
// This test is a placeholder for the prod RLS WITH CHECK test.
|
||||
expect(true).toBe(true); // prod RLS WITH CHECK test runs at M1 review
|
||||
it("T1 cannot INSERT a target row for T2 — RLS WITH CHECK enforcement (R-009, G-022)", async () => {
|
||||
// In PG mode (real Postgres 16, coreci_app role NOBYPASSRLS), RLS WITH
|
||||
// CHECK blocks a cross-tenant INSERT even though the app role has the
|
||||
// INSERT privilege: withTenant(T1) sets app.tenant_id=T1, so inserting
|
||||
// with tenant_id=T2 violates the WITH CHECK clause (tenant_id must equal
|
||||
// app.tenant_id). This is the R-009 deliverable — the M1 placeholder
|
||||
// `expect(true).toBe(true)` is replaced by a real RLS rejection assertion.
|
||||
//
|
||||
// In PGlite mode (DB_MODE unset), RLS WITH CHECK is NOT enforced (PGlite
|
||||
// 0.5.7 limitation), so the cross-tenant insert SUCCEEDS at the DB layer.
|
||||
// The application's insert paths always use the scoped tenant_id from
|
||||
// withTenant, never a user-supplied tenant_id — so the app-layer
|
||||
// enforcement holds regardless. This test documents both behaviors.
|
||||
if (isPgMode) {
|
||||
// Real Postgres 16: RLS WITH CHECK MUST reject the cross-tenant insert.
|
||||
await expect(
|
||||
withTenant(T1, async (c) => {
|
||||
await c.query(
|
||||
`INSERT INTO targets (tenant_id, hostname, os_name, os_version, agent_version) VALUES
|
||||
($1,'evil-t2-host','ubuntu','24.04','0.0.1')`,
|
||||
[T2], // cross-tenant: app.tenant_id=T1, row tenant_id=T2 → RLS rejects
|
||||
);
|
||||
}),
|
||||
).rejects.toThrow(/row level security|WITH CHECK|new row violates/i);
|
||||
} else {
|
||||
// PGlite: RLS WITH CHECK not enforced — the insert succeeds at the DB
|
||||
// layer. The app layer (withTenant + scoped inserts) is the primary
|
||||
// enforcement in dev. Document that prod RLS would reject this.
|
||||
await withTenant(T1, async (c) => {
|
||||
// Insert with T2's tenant_id; PGlite allows it (no WITH CHECK).
|
||||
await c.query(
|
||||
`INSERT INTO targets (tenant_id, hostname, os_name, os_version, agent_version) VALUES
|
||||
($1,'evil-t2-host-pglite','ubuntu','24.04','0.0.1') ON CONFLICT DO NOTHING`,
|
||||
[T2],
|
||||
);
|
||||
});
|
||||
// Clean up the seeded row so it doesn't pollute later assertions.
|
||||
await withTenant(T1, async (c) => {
|
||||
await c.query(`DELETE FROM targets WHERE hostname = 'evil-t2-host-pglite'`);
|
||||
});
|
||||
// The PGlite path documents the gap; prod (DB_MODE=pg) enforces it.
|
||||
expect(true).toBe(true); // PGlite: RLS WITH CHECK not enforced (R-009 gap)
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import js from "@eslint/js";
|
||||
import tseslint from "typescript-eslint";
|
||||
|
||||
// ESLint config for the CI-only mock LLM provider. The package is a
|
||||
// devDependency of the control-plane (R-008); it is import-guarded against
|
||||
// the prod bundle by the root lint rule (no-restricted-imports in the
|
||||
// control-plane's eslint.config.js) and by a build-time grep.
|
||||
export default tseslint.config(
|
||||
js.configs.recommended,
|
||||
...tseslint.configs.recommended,
|
||||
{
|
||||
rules: {
|
||||
"@typescript-eslint/no-explicit-any": "warn",
|
||||
"@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: ["dist/**", "node_modules/**", "coverage/**"],
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"name": "@coreci/llm-mock",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "CI-only mock LLM provider implementing OpenAI-compatible /v1/chat/completions with tool-calling (Wave J, R-008). devDependency only — import-guarded from prod.",
|
||||
"main": "./src/server.ts",
|
||||
"types": "./src/server.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./src/server.ts",
|
||||
"import": "./src/server.ts"
|
||||
},
|
||||
"./patterns": {
|
||||
"types": "./src/patterns.ts",
|
||||
"import": "./src/patterns.ts"
|
||||
},
|
||||
"./retry": {
|
||||
"types": "./src/retry.ts",
|
||||
"import": "./src/retry.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json --noEmit",
|
||||
"lint": "eslint src --max-warnings 0",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"start": "tsx src/server.ts"
|
||||
},
|
||||
"dependencies": {},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "9.39.5",
|
||||
"@types/node": "^22.0.0",
|
||||
"eslint": "9.39.5",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0",
|
||||
"typescript-eslint": "8.39.0",
|
||||
"vitest": "^2.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* @coreci/llm-mock/patterns — hardened prompt → tool_call matching (G-019).
|
||||
*
|
||||
* A REGEX SET (not a 2-word conjunction) so the mock tolerates wording drift.
|
||||
* The M2 LLM smoke (Wave J, gate item 8) sends prompts like "List my GitHub
|
||||
* repositories." and "Show me my GitHub repositories." — the matcher returns
|
||||
* the same `tool_calls` payload for both. Tests assert the pattern matches
|
||||
* "Show me my GitHub repositories", "List my repos", "Get repositories".
|
||||
*
|
||||
* Each pattern produces a deterministic `tool_calls` entry (OpenAI shape):
|
||||
* { id, type:"function", function:{ name, arguments(JSON string) } }
|
||||
*
|
||||
* On a second call (with a `tool` role message in the history), the matcher
|
||||
* switches to synthesis mode: it parses the repo names out of the tool message
|
||||
* content and returns a grounded assistant message (no tool_calls).
|
||||
*
|
||||
* DETERMINISTIC — no randomness. The same prompt always returns the same
|
||||
* tool_calls; the same tool message always returns the same synthesis.
|
||||
*/
|
||||
|
||||
/** OpenAI tool_call shape (the subset we emit). */
|
||||
export interface ToolCall {
|
||||
id: string;
|
||||
type: "function";
|
||||
function: { name: string; arguments: string };
|
||||
}
|
||||
|
||||
/** A single chat message (the subset the matcher reads). */
|
||||
export interface ChatMessage {
|
||||
role: "system" | "user" | "assistant" | "tool";
|
||||
content: string;
|
||||
/** Set on assistant messages that requested a tool call. */
|
||||
tool_calls?: ToolCall[];
|
||||
/** Set on tool messages — echoes the originating tool_call id. */
|
||||
tool_call_id?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Match an input prompt against the regex set and return the deterministic
|
||||
* `tool_calls` payload, or `null` when no pattern matches.
|
||||
*
|
||||
* The matcher inspects the LAST user message (the active prompt). It ignores
|
||||
* prior history (the smoke's first call has only one user message).
|
||||
*
|
||||
* Patterns are intentionally tolerant of:
|
||||
* - case ("LIST", "Show", "get"),
|
||||
* - synonyms ("repo" / "repositor..."),
|
||||
* - phrasing ("my", "the", "all"),
|
||||
* - punctuation.
|
||||
*/
|
||||
export function matchPromptToToolCalls(messages: ChatMessage[]): ToolCall[] | null {
|
||||
const lastUser = [...messages].reverse().find((m) => m.role === "user");
|
||||
if (!lastUser) return null;
|
||||
const prompt = lastUser.content ?? "";
|
||||
|
||||
for (const pattern of PATTERNS) {
|
||||
if (pattern.regex.test(prompt)) {
|
||||
return pattern.toolCalls;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** A regex pattern + the deterministic tool_calls it produces on match. */
|
||||
interface Pattern {
|
||||
/** What the user prompt must match (case-insensitive). */
|
||||
regex: RegExp;
|
||||
/** The deterministic tool_calls payload (frozen). */
|
||||
toolCalls: ToolCall[];
|
||||
}
|
||||
|
||||
/** Stable tool_call ids (deterministic — same ids on every match). */
|
||||
const CALL_ID_LIST_REPOS = "call_list_repos_1";
|
||||
const CALL_ID_RECENT_RUNS = "call_recent_runs_1";
|
||||
|
||||
/** The canned `github.list_repos` arguments (empty object — no args). */
|
||||
const ARGS_LIST_REPOS = "{}";
|
||||
|
||||
/** The canned `github.get_recent_ci_runs` arguments (with placeholder owner/repo). */
|
||||
const ARGS_RECENT_RUNS = JSON.stringify({ owner: "coreci", repo: "coreci-test-repo-1" });
|
||||
|
||||
/**
|
||||
* The hardened pattern set. ORDER MATTERS: the first match wins.
|
||||
*
|
||||
* The set covers the M2 smoke's prompts + a few wording variants so the mock
|
||||
* is robust against test-prompt drift (G-019). Patterns are anchored loosely
|
||||
* (`.*` prefix/suffix) so the keyword pair can appear anywhere in the prompt.
|
||||
*/
|
||||
const PATTERNS: Pattern[] = [
|
||||
{
|
||||
// "List my GitHub repositories", "Show me my GitHub repositories",
|
||||
// "Get repositories", "List my repos", "show all my github repos".
|
||||
regex: /(list|show|get|display|fetch|enumerate)\b.*\b(repos?|repositor(?:y|ies))\b/i,
|
||||
toolCalls: [
|
||||
{
|
||||
id: CALL_ID_LIST_REPOS,
|
||||
type: "function",
|
||||
function: { name: "github.list_repos", arguments: ARGS_LIST_REPOS },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// "What were my recent CI runs", "latest workflow runs", "last runs".
|
||||
regex: /(recent|latest|last)\b.*\b(run|ci|workflow)s?\b/i,
|
||||
toolCalls: [
|
||||
{
|
||||
id: CALL_ID_RECENT_RUNS,
|
||||
type: "function",
|
||||
function: { name: "github.get_recent_ci_runs", arguments: ARGS_RECENT_RUNS },
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
/** Whether the message history contains a `tool` role message (synthesis mode). */
|
||||
export function hasToolMessage(messages: ChatMessage[]): boolean {
|
||||
return messages.some((m) => m.role === "tool");
|
||||
}
|
||||
|
||||
/**
|
||||
* Synthesize a grounded assistant message from a tool result.
|
||||
*
|
||||
* Parses repo names out of the tool message content (the github-mock adapter
|
||||
* returns `[{"name":"coreci-test-repo-1"},{"name":"coreci-test-repo-2"}]` or
|
||||
* the normalized shape `{"repos":[{"name":...}]}`). Returns a deterministic
|
||||
* grounded string:
|
||||
* "Your repos are: coreci-test-repo-1, coreci-test-repo-2"
|
||||
*
|
||||
* Falls back to echoing the tool content when no repo names can be parsed
|
||||
* (defensive — the mock must always return SOME assistant message).
|
||||
*/
|
||||
export function synthesizeGroundedResponse(messages: ChatMessage[]): string {
|
||||
const toolMessages = messages.filter((m) => m.role === "tool");
|
||||
if (toolMessages.length === 0) {
|
||||
return "I have no tool result to summarize.";
|
||||
}
|
||||
// Use the first tool message (the smoke sends one tool_call → one tool message).
|
||||
const first = toolMessages[0];
|
||||
const content = first ? first.content ?? "" : "";
|
||||
const names = parseRepoNames(content);
|
||||
if (names.length === 0) {
|
||||
return `I retrieved the result: ${content}`;
|
||||
}
|
||||
return `Your repos are: ${names.join(", ")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse repo names out of a tool message content. Tries several shapes the
|
||||
* adapters may produce:
|
||||
* - `[{"name":"coreci-test-repo-1"}, ...]` (raw github-mock array)
|
||||
* - `{"repos":[{"name":"..."}]}` (normalized list_repos result)
|
||||
* - a plain JSON array of strings
|
||||
* Returns an empty array when no names can be parsed.
|
||||
*/
|
||||
export function parseRepoNames(toolContent: string): string[] {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(toolContent);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Case 1: { repos: [{ name: "..." }] }
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
const repos = (parsed as { repos?: unknown }).repos;
|
||||
if (Array.isArray(repos)) {
|
||||
return extractNames(repos);
|
||||
}
|
||||
}
|
||||
// Case 2: [{ name: "..." }]
|
||||
if (Array.isArray(parsed)) {
|
||||
return extractNames(parsed);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Pull `name` strings out of an array of repo objects (or strings). */
|
||||
function extractNames(arr: unknown[]): string[] {
|
||||
const names: string[] = [];
|
||||
for (const item of arr) {
|
||||
if (typeof item === "string") {
|
||||
names.push(item);
|
||||
continue;
|
||||
}
|
||||
if (item && typeof item === "object" && "name" in item) {
|
||||
const name = (item as { name?: unknown }).name;
|
||||
if (typeof name === "string") names.push(name);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* @coreci/llm-mock/retry — retry policy for the LLM smoke Track B (real-path,
|
||||
* G-019). On 429/5xx/timeout from the real GitHub adapter, the smoke retries
|
||||
* 3× with exponential backoff (1s, 2s, 4s); on final failure, it SKIPS with a
|
||||
* warning (the real-path track is `allow-failure`, never blocking the P0 gate).
|
||||
*
|
||||
* This module is generic — it wraps any async operation and retries on a
|
||||
* configurable set of failure discriminators. The Track-B smoke uses it to wrap
|
||||
* the broker → real GitHub adapter call. Track A (mock-path) does NOT retry
|
||||
* (it never fails — the github-mock adapter is deterministic).
|
||||
*/
|
||||
|
||||
/** A retryable error discriminator (returns true if the error is retryable). */
|
||||
export type RetryPredicate = (err: unknown) => boolean;
|
||||
|
||||
/** Options for `withRetry`. */
|
||||
export interface RetryOptions {
|
||||
/** Max attempts (default 3 — 1 initial + 2 retries). */
|
||||
maxAttempts?: number;
|
||||
/** Base backoff ms (default 1000). Each retry waits base * 2^(attempt-1). */
|
||||
baseMs?: number;
|
||||
/** Injectable sleeper (tests pass a fake to skip real waits). */
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
/** Retryable error discriminator. Default: retry on 429/5xx/timeout. */
|
||||
isRetryable?: RetryPredicate;
|
||||
/** Called before each retry with the attempt number + error (logging hook). */
|
||||
onRetry?: (attempt: number, err: unknown, waitMs: number) => void;
|
||||
}
|
||||
|
||||
/** Default backoff schedule: 1s, 2s, 4s (exponential, base 1000ms). */
|
||||
export const DEFAULT_MAX_ATTEMPTS = 3;
|
||||
export const DEFAULT_BASE_MS = 1000;
|
||||
|
||||
/** Default sleeper (real Promise). */
|
||||
export const defaultSleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Default retryable discriminator: retry on HTTP 429, 5xx, and network/timeout
|
||||
* errors (AbortError, TypeError from fetch). The Track-B smoke wraps the
|
||||
* broker → real-GitHub call; these are the GitHub failure modes (R-004).
|
||||
*/
|
||||
export const defaultIsRetryable: RetryPredicate = (err: unknown): boolean => {
|
||||
if (err === null || err === undefined) return false;
|
||||
// A status field (HTTP-shaped error) — retry on 429 + 5xx.
|
||||
const status = (err as { status?: number }).status;
|
||||
if (typeof status === "number") {
|
||||
return status === 429 || (status >= 500 && status < 600);
|
||||
}
|
||||
// AbortError / DOMException (timeout) — retryable.
|
||||
if (err instanceof Error) {
|
||||
const name = err.name;
|
||||
if (name === "AbortError" || name === "TimeoutError") return true;
|
||||
// fetch network failure → TypeError "fetch failed" — retryable.
|
||||
if (err.name === "TypeError") return true;
|
||||
}
|
||||
// Unknown — be conservative and NOT retry (avoid retrying on logic errors).
|
||||
return false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Run `fn()` with retries. On a retryable failure, waits the exponential
|
||||
* backoff (base * 2^(attempt-1)) and retries up to `maxAttempts` total. On
|
||||
* final failure, rethrows the last error (the caller decides to skip+warn).
|
||||
*
|
||||
* `maxAttempts` is the TOTAL number of attempts (1 = no retry; 3 = 1 initial
|
||||
* + 2 retries). The default (3) yields waits of 1s, 2s (the 3rd attempt has
|
||||
* no wait after it — it's the final failure or success).
|
||||
*/
|
||||
export async function withRetry<T>(fn: () => Promise<T>, opts: RetryOptions = {}): Promise<T> {
|
||||
const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
||||
const baseMs = opts.baseMs ?? DEFAULT_BASE_MS;
|
||||
const sleep = opts.sleep ?? defaultSleep;
|
||||
const isRetryable = opts.isRetryable ?? defaultIsRetryable;
|
||||
|
||||
let lastErr: unknown;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (attempt >= maxAttempts || !isRetryable(err)) {
|
||||
throw err;
|
||||
}
|
||||
const waitMs = baseMs * Math.pow(2, attempt - 1);
|
||||
opts.onRetry?.(attempt, err, waitMs);
|
||||
await sleep(waitMs);
|
||||
}
|
||||
}
|
||||
// Unreachable (the loop throws on the final attempt), but keeps TS happy.
|
||||
throw lastErr;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* @coreci/llm-mock/server — CI-only mock LLM provider (Wave J, R-008).
|
||||
*
|
||||
* Implements an OpenAI-compatible `/v1/chat/completions` HTTP endpoint using
|
||||
* Node's built-in `http` module (no express dependency — keep the dev-dep
|
||||
* surface tiny). The mock is a `devDependency` of the control-plane and is
|
||||
* import-guarded against the prod bundle (R-008): an eslint `no-restricted-
|
||||
* imports` rule bans `@coreci/llm-mock` in `apps/control-plane/app/**` and
|
||||
* `packages/mcp/**`, and a build-time grep fails the build if `llm-mock`
|
||||
* appears in the prod build output.
|
||||
*
|
||||
* The smoke's 7-step flow (G-018):
|
||||
* 1. Test sends POST /v1/chat/completions with tools=[github.list_repos] and
|
||||
* prompt "List my GitHub repositories."
|
||||
* 2. The mock matches the prompt via the hardened regex set (G-019) and
|
||||
* returns tool_calls:[{function:{name:"github.list_repos", arguments:"{}"}}].
|
||||
* 3. The broker translator → MCP tools/call → routes to github-mock →
|
||||
* canned repo data.
|
||||
* 4. The broker translator → OpenAI tool message.
|
||||
* 5. Test sends a SECOND POST /v1/chat/completions with the full history:
|
||||
* [original prompt, assistant tool_call, tool message].
|
||||
* 6. The mock detects the `tool` message and synthesizes a grounded
|
||||
* response by parsing repo names out of the tool message content:
|
||||
* "Your repos are: coreci-test-repo-1, coreci-test-repo-2".
|
||||
* 7. The test asserts the canned repo names appear in the response.
|
||||
*
|
||||
* DETERMINISTIC — the same prompt always returns the same tool_calls; the
|
||||
* same tool message always returns the same synthesis. No randomness. This
|
||||
* is the load-bearing reliability guarantee for the P0 mock-path gate (G-018).
|
||||
*
|
||||
* The server accepts the OpenAI `tools` param and echoes the declared tool
|
||||
* set in the response `finish_reason: "tool_calls"`. On synthesis mode it
|
||||
* returns `finish_reason: "stop"` with a grounded `content` string.
|
||||
*/
|
||||
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import { matchPromptToToolCalls, synthesizeGroundedResponse, hasToolMessage, type ChatMessage, type ToolCall } from "./patterns.js";
|
||||
|
||||
/** The OpenAI chat completion request shape (the subset we read). */
|
||||
interface ChatCompletionRequest {
|
||||
model?: string;
|
||||
messages: ChatMessage[];
|
||||
/** OpenAI `tools` parameter (we accept and ignore — the mock picks its own). */
|
||||
tools?: unknown;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** The OpenAI chat completion response shape (D-001 compatible). */
|
||||
interface ChatCompletionResponse {
|
||||
id: string;
|
||||
object: "chat.completion";
|
||||
created: number;
|
||||
model: string;
|
||||
choices: {
|
||||
index: number;
|
||||
message: { role: "assistant"; content: string | null; tool_calls?: ToolCall[] };
|
||||
finish_reason: "stop" | "tool_calls" | "length";
|
||||
}[];
|
||||
usage: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
|
||||
}
|
||||
|
||||
/** Stable response id (deterministic — same id on every call). */
|
||||
const RESPONSE_ID = "chatcmpl-llm-mock-0001";
|
||||
/** Stable model name (deterministic). */
|
||||
const MODEL_NAME = "coreci-llm-mock-1";
|
||||
|
||||
/**
|
||||
* Produce a deterministic OpenAI-compatible chat completion response for the
|
||||
* given request. This is the pure function the HTTP handler wraps and the
|
||||
* function the smoke calls directly (the smoke may bypass HTTP and call this
|
||||
* to avoid spawning a server in the same process).
|
||||
*
|
||||
* Behavior:
|
||||
* - If the message history contains a `tool` role message → SYNTHESIS mode:
|
||||
* returns a grounded assistant message (finish_reason: "stop").
|
||||
* - Else → TOOL_CALL mode: match the prompt against the regex set (G-019).
|
||||
* On match, returns tool_calls (finish_reason: "tool_calls"). On no match,
|
||||
* returns a fallback assistant message (finish_reason: "stop") — the mock
|
||||
* never errors, so the smoke is reliable.
|
||||
*/
|
||||
export function handleChatCompletion(req: ChatCompletionRequest): ChatCompletionResponse {
|
||||
const messages = req.messages ?? [];
|
||||
const created = 0; // deterministic timestamp (0) — the mock is reproducible
|
||||
|
||||
// Synthesis mode: a tool message is present → the broker has fed the tool
|
||||
// result back; synthesize a grounded response from the repo names.
|
||||
if (hasToolMessage(messages)) {
|
||||
const content = synthesizeGroundedResponse(messages);
|
||||
return makeResponse({ role: "assistant", content }, "stop", created);
|
||||
}
|
||||
|
||||
// Tool-call mode: match the user prompt → tool_calls.
|
||||
const toolCalls = matchPromptToToolCalls(messages);
|
||||
if (toolCalls && toolCalls.length > 0) {
|
||||
return makeResponse({ role: "assistant", content: null, tool_calls: toolCalls }, "tool_calls", created);
|
||||
}
|
||||
|
||||
// Fallback (no pattern matched): return a benign message. The mock NEVER
|
||||
// returns an error — the smoke's reliability is the P0 gate (G-018).
|
||||
const content =
|
||||
"I'm a mock LLM. I can list your GitHub repositories (try 'List my GitHub repositories').";
|
||||
return makeResponse({ role: "assistant", content }, "stop", created);
|
||||
}
|
||||
|
||||
/** Build a single-choice response with deterministic token counts. */
|
||||
function makeResponse(
|
||||
message: { role: "assistant"; content: string | null; tool_calls?: ToolCall[] },
|
||||
finishReason: "stop" | "tool_calls",
|
||||
created: number,
|
||||
): ChatCompletionResponse {
|
||||
return {
|
||||
id: RESPONSE_ID,
|
||||
object: "chat.completion",
|
||||
created,
|
||||
model: MODEL_NAME,
|
||||
choices: [{ index: 0, message, finish_reason: finishReason }],
|
||||
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the mock HTTP server on the given port (default 4100). Resolves to
|
||||
* the Server handle; `stop()` closes it. CI starts this BEFORE the control
|
||||
* plane and points the control plane's BYOM endpoint at it.
|
||||
*
|
||||
* Endpoints:
|
||||
* POST /v1/chat/completions — OpenAI-compatible (the only endpoint used).
|
||||
* GET /healthz — liveness probe (CI waits for this before smoke).
|
||||
*/
|
||||
export function startMockServer(port = 4100): Promise<Server> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
||||
// CORS-friendly + JSON defaults.
|
||||
res.setHeader("Access-Control-Allow-Origin", "*");
|
||||
res.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type");
|
||||
res.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");
|
||||
if (req.method === "OPTIONS") {
|
||||
res.writeHead(204);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === "GET" && req.url === "/healthz") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: true }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === "POST" && req.url === "/v1/chat/completions") {
|
||||
let body = "";
|
||||
req.on("data", (chunk) => {
|
||||
body += chunk.toString();
|
||||
});
|
||||
req.on("end", () => {
|
||||
try {
|
||||
const parsed = JSON.parse(body) as ChatCompletionRequest;
|
||||
const out = handleChatCompletion(parsed);
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify(out));
|
||||
} catch (err) {
|
||||
res.writeHead(400, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ error: "bad_request", detail: err instanceof Error ? err.message : String(err) }));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ error: "not_found", detail: `No handler for ${req.method} ${req.url}` }));
|
||||
});
|
||||
|
||||
server.on("error", reject);
|
||||
server.listen(port, () => resolve(server));
|
||||
});
|
||||
}
|
||||
|
||||
/** Stop a mock server started by `startMockServer`. */
|
||||
export function stopMockServer(server: Server): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
server.close((err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* @coreci/llm-mock/tests/patterns.test.ts — hardened pattern matching (G-019).
|
||||
*
|
||||
* Asserts the regex set tolerates wording drift: "List my GitHub repositories",
|
||||
* "Show me my GitHub repositories", "Get repositories", "List my repos", etc.
|
||||
* Also asserts the synthesis path parses repo names from tool message content
|
||||
* (both raw github-mock array and normalized {repos:[...]} shapes).
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
matchPromptToToolCalls,
|
||||
synthesizeGroundedResponse,
|
||||
parseRepoNames,
|
||||
hasToolMessage,
|
||||
type ChatMessage,
|
||||
} from "../src/patterns.js";
|
||||
|
||||
describe("llm-mock patterns — list_repos matching (G-019)", () => {
|
||||
const cases: string[] = [
|
||||
"List my GitHub repositories.",
|
||||
"Show me my GitHub repositories",
|
||||
"Get repositories",
|
||||
"List my repos",
|
||||
"show all my github repos",
|
||||
"Please enumerate my repositories",
|
||||
"fetch my repos please",
|
||||
"DISPLAY MY GITHUB REPOS",
|
||||
];
|
||||
|
||||
for (const prompt of cases) {
|
||||
it(`matches "${prompt}" → github.list_repos`, () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: prompt }];
|
||||
const calls = matchPromptToToolCalls(messages);
|
||||
expect(calls).not.toBeNull();
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls![0].function.name).toBe("github.list_repos");
|
||||
// arguments is a JSON string of {} (no args for list_repos).
|
||||
expect(calls![0].function.arguments).toBe("{}");
|
||||
expect(calls![0].type).toBe("function");
|
||||
expect(calls![0].id).toBe("call_list_repos_1");
|
||||
});
|
||||
}
|
||||
|
||||
it("does NOT match an unrelated prompt", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "What's the weather?" }];
|
||||
expect(matchPromptToToolCalls(messages)).toBeNull();
|
||||
});
|
||||
|
||||
it("uses the LAST user message (ignores prior history)", () => {
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: "user", content: "What's the weather?" },
|
||||
{ role: "assistant", content: "I don't know." },
|
||||
{ role: "user", content: "List my GitHub repositories" },
|
||||
];
|
||||
const calls = matchPromptToToolCalls(messages);
|
||||
expect(calls).not.toBeNull();
|
||||
expect(calls![0].function.name).toBe("github.list_repos");
|
||||
});
|
||||
|
||||
it("returns null when there is no user message", () => {
|
||||
const messages: ChatMessage[] = [{ role: "system", content: "be helpful" }];
|
||||
expect(matchPromptToToolCalls(messages)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("llm-mock patterns — recent CI runs matching", () => {
|
||||
it("matches 'recent CI runs'", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "What were my recent CI runs?" }];
|
||||
const calls = matchPromptToToolCalls(messages);
|
||||
expect(calls).not.toBeNull();
|
||||
expect(calls![0].function.name).toBe("github.get_recent_ci_runs");
|
||||
// arguments is a JSON object with owner+repo.
|
||||
const args = JSON.parse(calls![0].function.arguments);
|
||||
expect(args).toEqual({ owner: "coreci", repo: "coreci-test-repo-1" });
|
||||
});
|
||||
|
||||
it("matches 'latest workflow runs'", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "Show me the latest workflow runs" }];
|
||||
const calls = matchPromptToToolCalls(messages);
|
||||
expect(calls).not.toBeNull();
|
||||
expect(calls![0].function.name).toBe("github.get_recent_ci_runs");
|
||||
});
|
||||
|
||||
it("matches 'last runs'", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "last runs for my repo" }];
|
||||
const calls = matchPromptToToolCalls(messages);
|
||||
expect(calls).not.toBeNull();
|
||||
expect(calls![0].function.name).toBe("github.get_recent_ci_runs");
|
||||
});
|
||||
});
|
||||
|
||||
describe("llm-mock patterns — determinism", () => {
|
||||
it("returns the SAME tool_call id on every call (no randomness)", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "List my GitHub repositories" }];
|
||||
const a = matchPromptToToolCalls(messages);
|
||||
const b = matchPromptToToolCalls(messages);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe("llm-mock patterns — hasToolMessage", () => {
|
||||
it("detects a tool message in the history", () => {
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: "user", content: "List my repos" },
|
||||
{ role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "github.list_repos", arguments: "{}" } }] },
|
||||
{ role: "tool", tool_call_id: "c1", content: '[{"name":"coreci-test-repo-1"}]' },
|
||||
];
|
||||
expect(hasToolMessage(messages)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false when there is no tool message", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "List my repos" }];
|
||||
expect(hasToolMessage(messages)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("llm-mock patterns — parseRepoNames", () => {
|
||||
it("parses the raw github-mock array shape [{name:'...'}]", () => {
|
||||
const content = JSON.stringify([
|
||||
{ name: "coreci-test-repo-1" },
|
||||
{ name: "coreci-test-repo-2" },
|
||||
]);
|
||||
expect(parseRepoNames(content)).toEqual(["coreci-test-repo-1", "coreci-test-repo-2"]);
|
||||
});
|
||||
|
||||
it("parses the normalized {repos:[...]} shape", () => {
|
||||
const content = JSON.stringify({ repos: [{ name: "a" }, { name: "b" }] });
|
||||
expect(parseRepoNames(content)).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("parses a plain array of strings", () => {
|
||||
const content = JSON.stringify(["x", "y"]);
|
||||
expect(parseRepoNames(content)).toEqual(["x", "y"]);
|
||||
});
|
||||
|
||||
it("returns [] for invalid JSON", () => {
|
||||
expect(parseRepoNames("not json")).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns [] for an object with no repos array", () => {
|
||||
expect(parseRepoNames(JSON.stringify({ foo: "bar" }))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("llm-mock patterns — synthesizeGroundedResponse", () => {
|
||||
it("synthesizes 'Your repos are: ...' from a tool message", () => {
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: "user", content: "List my repos" },
|
||||
{ role: "tool", tool_call_id: "c1", content: JSON.stringify([
|
||||
{ name: "coreci-test-repo-1" },
|
||||
{ name: "coreci-test-repo-2" },
|
||||
]) },
|
||||
];
|
||||
const out = synthesizeGroundedResponse(messages);
|
||||
expect(out).toBe("Your repos are: coreci-test-repo-1, coreci-test-repo-2");
|
||||
});
|
||||
|
||||
it("deterministic — same tool message → same synthesis", () => {
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: "tool", tool_call_id: "c1", content: '[{"name":"a"}]' },
|
||||
];
|
||||
expect(synthesizeGroundedResponse(messages)).toBe("Your repos are: a");
|
||||
expect(synthesizeGroundedResponse(messages)).toBe("Your repos are: a");
|
||||
});
|
||||
|
||||
it("falls back to echoing content when no repo names can be parsed", () => {
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: "tool", tool_call_id: "c1", content: "no json here" },
|
||||
];
|
||||
const out = synthesizeGroundedResponse(messages);
|
||||
expect(out).toBe("I retrieved the result: no json here");
|
||||
});
|
||||
|
||||
it("returns a fallback message when there is no tool message", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "hi" }];
|
||||
expect(synthesizeGroundedResponse(messages)).toBe("I have no tool result to summarize.");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* @coreci/llm-mock/tests/retry.test.ts — retry policy for Track B (G-019).
|
||||
*
|
||||
* Asserts the exponential backoff schedule (1s, 2s, 4s), the retryable
|
||||
* discriminators (429/5xx/timeout), and that final failure rethrows.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { withRetry, defaultIsRetryable } from "../src/retry.js";
|
||||
|
||||
describe("llm-mock retry — exponential backoff (G-019)", () => {
|
||||
it("retries 3× with backoff 1s, 2s, 4s then succeeds", async () => {
|
||||
const sleeps: number[] = [];
|
||||
const sleep = async (ms: number) => { sleeps.push(ms); };
|
||||
let calls = 0;
|
||||
const fn = async () => {
|
||||
calls++;
|
||||
if (calls < 3) throw Object.assign(new Error("429"), { status: 429 });
|
||||
return "ok";
|
||||
};
|
||||
const result = await withRetry(fn, { sleep, baseMs: 1000, maxAttempts: 3 });
|
||||
expect(result).toBe("ok");
|
||||
expect(calls).toBe(3);
|
||||
// First retry waits 1s (base * 2^0); second waits 2s (base * 2^1).
|
||||
expect(sleeps).toEqual([1000, 2000]);
|
||||
});
|
||||
|
||||
it("rethrows the last error after max attempts", async () => {
|
||||
const sleep = async () => {}; // skip real waits
|
||||
let calls = 0;
|
||||
const fn = async () => {
|
||||
calls++;
|
||||
throw Object.assign(new Error("5xx"), { status: 503 });
|
||||
};
|
||||
await expect(withRetry(fn, { sleep, maxAttempts: 3, baseMs: 1 })).rejects.toThrow("5xx");
|
||||
expect(calls).toBe(3);
|
||||
});
|
||||
|
||||
it("does NOT retry on a non-retryable error (400)", async () => {
|
||||
const sleep = vi.fn(async () => {});
|
||||
let calls = 0;
|
||||
const fn = async () => {
|
||||
calls++;
|
||||
throw Object.assign(new Error("bad request"), { status: 400 });
|
||||
};
|
||||
await expect(withRetry(fn, { sleep, maxAttempts: 3 })).rejects.toThrow("bad request");
|
||||
expect(calls).toBe(1);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("invokes onRetry before each retry", async () => {
|
||||
const onRetry = vi.fn();
|
||||
const sleep = async () => {};
|
||||
let calls = 0;
|
||||
const fn = async () => {
|
||||
calls++;
|
||||
if (calls < 3) throw Object.assign(new Error("e"), { status: 500 });
|
||||
return "ok";
|
||||
};
|
||||
await withRetry(fn, { sleep, maxAttempts: 3, baseMs: 1000, onRetry });
|
||||
expect(onRetry).toHaveBeenCalledTimes(2);
|
||||
expect(onRetry).toHaveBeenNthCalledWith(1, expect.any(Number), expect.any(Error), 1000);
|
||||
expect(onRetry).toHaveBeenNthCalledWith(2, expect.any(Number), expect.any(Error), 2000);
|
||||
});
|
||||
|
||||
it("respects a custom isRetryable discriminator", async () => {
|
||||
const sleep = async () => {};
|
||||
let calls = 0;
|
||||
const fn = async () => {
|
||||
calls++;
|
||||
if (calls < 2) throw new Error("always-retry-me");
|
||||
return "ok";
|
||||
};
|
||||
const result = await withRetry(fn, {
|
||||
sleep,
|
||||
maxAttempts: 3,
|
||||
isRetryable: () => true,
|
||||
});
|
||||
expect(result).toBe("ok");
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("llm-mock retry — defaultIsRetryable", () => {
|
||||
it("retries on 429", () => {
|
||||
expect(defaultIsRetryable(Object.assign(new Error("e"), { status: 429 }))).toBe(true);
|
||||
});
|
||||
it("retries on 500", () => {
|
||||
expect(defaultIsRetryable(Object.assign(new Error("e"), { status: 500 }))).toBe(true);
|
||||
});
|
||||
it("retries on 503", () => {
|
||||
expect(defaultIsRetryable(Object.assign(new Error("e"), { status: 503 }))).toBe(true);
|
||||
});
|
||||
it("does NOT retry on 400", () => {
|
||||
expect(defaultIsRetryable(Object.assign(new Error("e"), { status: 400 }))).toBe(false);
|
||||
});
|
||||
it("does NOT retry on 403", () => {
|
||||
expect(defaultIsRetryable(Object.assign(new Error("e"), { status: 403 }))).toBe(false);
|
||||
});
|
||||
it("retries on AbortError (timeout)", () => {
|
||||
const err = new Error("aborted");
|
||||
err.name = "AbortError";
|
||||
expect(defaultIsRetryable(err)).toBe(true);
|
||||
});
|
||||
it("retries on TypeError (fetch network failure)", () => {
|
||||
expect(defaultIsRetryable(new TypeError("fetch failed"))).toBe(true);
|
||||
});
|
||||
it("does NOT retry on a plain Error", () => {
|
||||
expect(defaultIsRetryable(new Error("logic error"))).toBe(false);
|
||||
});
|
||||
it("returns false on null/undefined", () => {
|
||||
expect(defaultIsRetryable(null)).toBe(false);
|
||||
expect(defaultIsRetryable(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,177 @@
|
||||
/**
|
||||
* @coreci/llm-mock/tests/server.test.ts — OpenAI-compatible /v1/chat/completions
|
||||
* endpoint + the 7-step LLM smoke flow (G-018).
|
||||
*
|
||||
* Asserts:
|
||||
* - Step 1 (tool-call mode): prompt "List my GitHub repositories." →
|
||||
* tool_calls:[{function:{name:"github.list_repos", arguments:"{}"}}],
|
||||
* finish_reason:"tool_calls".
|
||||
* - Step 6 (synthesis mode): full history with tool message → grounded
|
||||
* "Your repos are: coreci-test-repo-1, coreci-test-repo-2",
|
||||
* finish_reason:"stop".
|
||||
* - Determinism: same request → same response (no randomness).
|
||||
* - The HTTP server responds to /healthz and /v1/chat/completions.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterAll, beforeAll } from "vitest";
|
||||
import { handleChatCompletion, startMockServer, stopMockServer } from "../src/server.js";
|
||||
import type { Server } from "node:http";
|
||||
import type { ChatMessage, ToolCall } from "../src/patterns.js";
|
||||
|
||||
describe("llm-mock server — handleChatCompletion (the 7-step flow)", () => {
|
||||
describe("Step 1: tool-call mode (prompt → tool_calls)", () => {
|
||||
const prompts = [
|
||||
"List my GitHub repositories.",
|
||||
"Show me my GitHub repositories",
|
||||
"Get repositories",
|
||||
"List my repos",
|
||||
];
|
||||
for (const prompt of prompts) {
|
||||
it(`returns github.list_repos tool_call for "${prompt}"`, () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: prompt }];
|
||||
const res = handleChatCompletion({ model: "m", messages, tools: [] });
|
||||
expect(res.choices).toHaveLength(1);
|
||||
const choice = res.choices[0];
|
||||
expect(choice.finish_reason).toBe("tool_calls");
|
||||
expect(choice.message.role).toBe("assistant");
|
||||
expect(choice.message.content).toBeNull();
|
||||
expect(choice.message.tool_calls).toBeDefined();
|
||||
expect(choice.message.tool_calls).toHaveLength(1);
|
||||
const tc: ToolCall = choice.message.tool_calls![0];
|
||||
expect(tc.function.name).toBe("github.list_repos");
|
||||
expect(tc.function.arguments).toBe("{}");
|
||||
expect(tc.type).toBe("function");
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("Step 6: synthesis mode (tool message → grounded response)", () => {
|
||||
it("synthesizes repo names from a raw github-mock tool message", () => {
|
||||
const toolContent = JSON.stringify([
|
||||
{ name: "coreci-test-repo-1" },
|
||||
{ name: "coreci-test-repo-2" },
|
||||
]);
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: "user", content: "List my GitHub repositories." },
|
||||
{
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [{ id: "call_list_repos_1", type: "function", function: { name: "github.list_repos", arguments: "{}" } }],
|
||||
},
|
||||
{ role: "tool", tool_call_id: "call_list_repos_1", content: toolContent },
|
||||
];
|
||||
const res = handleChatCompletion({ model: "m", messages });
|
||||
expect(res.choices[0].finish_reason).toBe("stop");
|
||||
expect(res.choices[0].message.content).toBe("Your repos are: coreci-test-repo-1, coreci-test-repo-2");
|
||||
expect(res.choices[0].message.tool_calls).toBeUndefined();
|
||||
});
|
||||
|
||||
it("synthesizes from the normalized {repos:[...]} shape", () => {
|
||||
const toolContent = JSON.stringify({ repos: [{ name: "a" }, { name: "b" }] });
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: "user", content: "List my repos" },
|
||||
{ role: "tool", tool_call_id: "c1", content: toolContent },
|
||||
];
|
||||
const res = handleChatCompletion({ model: "m", messages });
|
||||
expect(res.choices[0].message.content).toBe("Your repos are: a, b");
|
||||
});
|
||||
});
|
||||
|
||||
describe("determinism", () => {
|
||||
it("returns the SAME response id + model on every call", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "List my repos" }];
|
||||
const a = handleChatCompletion({ model: "m", messages });
|
||||
const b = handleChatCompletion({ model: "m", messages });
|
||||
expect(a.id).toBe(b.id);
|
||||
expect(a.model).toBe(b.model);
|
||||
expect(a.choices).toEqual(b.choices);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fallback (no pattern matched)", () => {
|
||||
it("returns a benign fallback message, never an error", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "hello world" }];
|
||||
const res = handleChatCompletion({ model: "m", messages });
|
||||
expect(res.choices[0].finish_reason).toBe("stop");
|
||||
expect(res.choices[0].message.content).toContain("mock LLM");
|
||||
});
|
||||
});
|
||||
|
||||
describe("OpenAI-compatible response shape", () => {
|
||||
it("has object, created, model, choices[], usage", () => {
|
||||
const messages: ChatMessage[] = [{ role: "user", content: "List my repos" }];
|
||||
const res = handleChatCompletion({ model: "m", messages });
|
||||
expect(res.object).toBe("chat.completion");
|
||||
expect(typeof res.created).toBe("number");
|
||||
expect(typeof res.model).toBe("string");
|
||||
expect(Array.isArray(res.choices)).toBe(true);
|
||||
expect(res.usage).toHaveProperty("total_tokens");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("llm-mock server — HTTP endpoints", () => {
|
||||
let server: Server;
|
||||
let port: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await startMockServer(0); // 0 = OS-assigned port
|
||||
const addr = server.address();
|
||||
if (addr && typeof addr === "object") port = addr.port;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await stopMockServer(server);
|
||||
});
|
||||
|
||||
it("GET /healthz returns {ok:true}", async () => {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/healthz`);
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { ok: boolean };
|
||||
expect(body.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("POST /v1/chat/completions returns tool_calls for list_repos prompt", async () => {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "m",
|
||||
messages: [{ role: "user", content: "List my GitHub repositories." }],
|
||||
tools: [{ type: "function", function: { name: "github.list_repos", parameters: {} } }],
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { choices: { message: { tool_calls?: ToolCall[]; finish_reason?: string } }[] };
|
||||
expect(body.choices[0].finish_reason).toBe("tool_calls");
|
||||
expect(body.choices[0].message.tool_calls![0].function.name).toBe("github.list_repos");
|
||||
});
|
||||
|
||||
it("POST /v1/chat/completions synthesizes grounded response in synthesis mode", async () => {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "m",
|
||||
messages: [
|
||||
{ role: "user", content: "List my repos" },
|
||||
{ role: "tool", tool_call_id: "c1", content: '[{"name":"coreci-test-repo-1"},{"name":"coreci-test-repo-2"}]' },
|
||||
],
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { choices: { message: { content: string }; finish_reason: string }[] };
|
||||
expect(body.choices[0].finish_reason).toBe("stop");
|
||||
expect(body.choices[0].message.content).toBe("Your repos are: coreci-test-repo-1, coreci-test-repo-2");
|
||||
});
|
||||
|
||||
it("GET unknown path returns 404", async () => {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/nope`);
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it("OPTIONS returns 204 (CORS preflight)", async () => {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { method: "OPTIONS" });
|
||||
expect(res.status).toBe(204);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"declaration": true,
|
||||
"composite": false,
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["dist", "tests", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
// Vitest config for the CI-only mock LLM provider. server.ts spawns a real
|
||||
// HTTP server on an OS-assigned port (port 0) so the HTTP tests don't need
|
||||
// a fixed port; the conformance harness waits for /healthz before the smoke.
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
include: ["tests/**/*.test.ts"],
|
||||
testTimeout: 30000,
|
||||
coverage: {
|
||||
provider: "v8",
|
||||
include: ["src/**/*.ts"],
|
||||
reporter: ["text", "json"],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -1,6 +1,10 @@
|
||||
import js from "@eslint/js";
|
||||
import tseslint from "typescript-eslint";
|
||||
|
||||
// ESLint config for the MCP broker package. [G-018, R-008] `@coreci/llm-mock`
|
||||
// is a CI-only devDependency and MUST NOT be imported from the broker (prod
|
||||
// runtime). The LLM smoke imports the broker + the mock from the test side;
|
||||
// the broker itself never depends on the mock.
|
||||
export default tseslint.config(
|
||||
js.configs.recommended,
|
||||
...tseslint.configs.recommended,
|
||||
@@ -10,7 +14,37 @@ export default tseslint.config(
|
||||
"@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_" }],
|
||||
},
|
||||
},
|
||||
// [G-018, R-008] Prod import guard: ban @coreci/llm-mock from the broker.
|
||||
{
|
||||
ignores: ["dist/**", "node_modules/**", "coverage/**"],
|
||||
files: ["src/**"],
|
||||
rules: {
|
||||
"no-restricted-imports": [
|
||||
"error",
|
||||
{
|
||||
paths: [
|
||||
{
|
||||
name: "@coreci/llm-mock",
|
||||
message:
|
||||
"@coreci/llm-mock is a CI-only devDependency (R-008). The broker must not depend on the mock LLM — the smoke imports the broker, not vice versa.",
|
||||
},
|
||||
{
|
||||
name: "@coreci/llm-mock/server",
|
||||
message: "@coreci/llm-mock/server is CI-only (R-008).",
|
||||
},
|
||||
{
|
||||
name: "@coreci/llm-mock/patterns",
|
||||
message: "@coreci/llm-mock/patterns is CI-only (R-008).",
|
||||
},
|
||||
{
|
||||
name: "@coreci/llm-mock/retry",
|
||||
message: "@coreci/llm-mock/retry is CI-only (R-008).",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: ["dist/**", "node_modules/**", "coverage/**", "tests/**"],
|
||||
},
|
||||
);
|
||||
@@ -49,6 +49,10 @@
|
||||
"./adapters": {
|
||||
"types": "./dist/adapters/index.d.ts",
|
||||
"import": "./dist/adapters/index.js"
|
||||
},
|
||||
"./adapters/github-mock": {
|
||||
"types": "./dist/adapters/github-mock/adapter.d.ts",
|
||||
"import": "./dist/adapters/github-mock/adapter.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* @coreci/mcp/adapters/github-mock/adapter — the deterministic canned-repo
|
||||
* GitHub adapter for the LLM smoke Track A (G-018, Wave J Task 5).
|
||||
*
|
||||
* DISTINCT from the Wave F stub (`adapters/stubs.ts`):
|
||||
* - The stub returns a generic `{content:[{type:"text",text:"stub"}]}` — it
|
||||
* proves the broker can route + emit an SSE event, but it does NOT return
|
||||
* repo names the smoke can assert.
|
||||
* - This github-mock returns a DETERMINISTIC canned repo list:
|
||||
* [{"name":"coreci-test-repo-1"},{"name":"coreci-test-repo-2"}]
|
||||
* so the Track-A smoke (P0 gate) can assert the synthesized LLM response
|
||||
* contains "coreci-test-repo-1, coreci-test-repo-2" — the full OpenAI→MCP→
|
||||
* adapter→result→synthesis path, with no external GitHub dependency.
|
||||
*
|
||||
* Implements the `McpAdapter` interface (G-020) — same interface the real
|
||||
* GitHub adapter (Wave I) and the Wave F stubs implement. The broker routes
|
||||
* to it identically; only the response content differs.
|
||||
*
|
||||
* `github.list_repos` returns the canned repo array. The other two GitHub
|
||||
* tools (`get_recent_ci_runs`, `get_workflow_run`) return canned CI-run
|
||||
* payloads so the smoke could exercise them too (the P0 gate uses
|
||||
* `list_repos` only; the other two are extras for completeness).
|
||||
*
|
||||
* NO network calls. NO SecretProvider. Deterministic — the same args always
|
||||
* return the same result. This is the reliability guarantee for the P0 gate
|
||||
* (G-018): the mock-path never fails, never rate-limits, never times out.
|
||||
*/
|
||||
|
||||
import type { McpAdapter, McpResult, Tool } from "../../types.js";
|
||||
import { getRegistryEntry } from "../../registry.js";
|
||||
|
||||
/** The GitHub tool names this mock serves (closed subset of the registry). */
|
||||
export const GITHUB_MOCK_TOOLS = [
|
||||
"github.list_repos",
|
||||
"github.get_recent_ci_runs",
|
||||
"github.get_workflow_run",
|
||||
] as const;
|
||||
|
||||
/** The deterministic canned repo list (Track A P0 gate asserts these names). */
|
||||
export const CANNED_REPOS = [
|
||||
{ id: 1, name: "coreci-test-repo-1", full_name: "coreci/coreci-test-repo-1", owner: "coreci", private: false, html_url: "https://example.test/coreci/coreci-test-repo-1" },
|
||||
{ id: 2, name: "coreci-test-repo-2", full_name: "coreci/coreci-test-repo-2", owner: "coreci", private: false, html_url: "https://example.test/coreci/coreci-test-repo-2" },
|
||||
] as const;
|
||||
|
||||
/** The deterministic canned CI-run list for get_recent_ci_runs. */
|
||||
export const CANNED_RUNS = {
|
||||
owner: "coreci",
|
||||
repo: "coreci-test-repo-1",
|
||||
total_count: 2,
|
||||
runs: [
|
||||
{ id: 101, head_branch: "main", status: "completed", conclusion: "success", html_url: "https://example.test/runs/101", created_at: "2026-08-25T00:00:00Z", actor: "ci-bot" },
|
||||
{ id: 102, head_branch: "main", status: "completed", conclusion: "failure", html_url: "https://example.test/runs/102", created_at: "2026-08-24T00:00:00Z", actor: "ci-bot" },
|
||||
],
|
||||
} as const;
|
||||
|
||||
/** The deterministic canned single workflow run for get_workflow_run. */
|
||||
export const CANNED_RUN = {
|
||||
owner: "coreci",
|
||||
repo: "coreci-test-repo-1",
|
||||
run_id: 101,
|
||||
id: 101,
|
||||
name: "CI",
|
||||
head_branch: "main",
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
html_url: "https://example.test/runs/101",
|
||||
created_at: "2026-08-25T00:00:00Z",
|
||||
actor: "ci-bot",
|
||||
run_number: 1,
|
||||
} as const;
|
||||
|
||||
/** Options for the github-mock adapter. */
|
||||
export interface GithubMockOptions {
|
||||
/** If true, returns isError:true (for an error-path smoke variant). */
|
||||
isError?: boolean;
|
||||
/** Override the canned repos (default CANNED_REPOS). */
|
||||
repos?: unknown[];
|
||||
/** Override the canned runs (default CANNED_RUNS). */
|
||||
runs?: unknown;
|
||||
/** Override the canned single run (default CANNED_RUN). */
|
||||
run?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the github-mock adapter. The mock is registered under adapter type
|
||||
* "github" so the broker's router (which keys by adapter_type) routes to it
|
||||
* identically to the real GitHub adapter. The smoke's CI setup registers
|
||||
* this mock INSTEAD of the real adapter for Track A.
|
||||
*/
|
||||
export function makeGithubMockAdapter(opts: GithubMockOptions = {}): McpAdapter {
|
||||
const isError = opts.isError ?? false;
|
||||
const repos = opts.repos ?? CANNED_REPOS;
|
||||
const runs = opts.runs ?? CANNED_RUNS;
|
||||
const run = opts.run ?? CANNED_RUN;
|
||||
|
||||
const tools: Tool[] = [];
|
||||
for (const name of GITHUB_MOCK_TOOLS) {
|
||||
const entry = getRegistryEntry(name);
|
||||
if (entry) tools.push({ ...entry.tool, inputSchema: { ...entry.tool.inputSchema } });
|
||||
}
|
||||
|
||||
return {
|
||||
type: "github",
|
||||
|
||||
async listTools(): Promise<Tool[]> {
|
||||
return tools.map((t) => ({ ...t, inputSchema: { ...t.inputSchema } }));
|
||||
},
|
||||
|
||||
async callTool(name: string, _args: Record<string, unknown>): Promise<McpResult> {
|
||||
if (!GITHUB_MOCK_TOOLS.includes(name as (typeof GITHUB_MOCK_TOOLS)[number])) {
|
||||
return {
|
||||
content: [{ type: "text", text: `github-mock: unknown tool ${name}` }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
if (isError) {
|
||||
return {
|
||||
content: [{ type: "text", text: "github-mock: forced error (isError variant)" }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
switch (name) {
|
||||
case "github.list_repos":
|
||||
return { content: [{ type: "text", text: JSON.stringify(repos) }], isError: false };
|
||||
case "github.get_recent_ci_runs":
|
||||
return { content: [{ type: "text", text: JSON.stringify(runs) }], isError: false };
|
||||
case "github.get_workflow_run":
|
||||
return { content: [{ type: "text", text: JSON.stringify(run) }], isError: false };
|
||||
default:
|
||||
return {
|
||||
content: [{ type: "text", text: `github-mock: unknown tool ${name}` }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,9 +1,18 @@
|
||||
/**
|
||||
* @coreci/mcp/adapters index — re-exports stub adapters (Wave F) and the
|
||||
* Proxmox (Wave G) + SSH (Wave H) + GitHub/Gitea (Wave I) adapters.
|
||||
* Proxmox (Wave G) + SSH (Wave H) + GitHub/Gitea (Wave I) adapters, plus the
|
||||
* github-mock canned-repo adapter (Wave J Task 5, G-018) for the LLM smoke.
|
||||
*/
|
||||
export { makeStubAdapter, defaultStubs, type StubOptions } from "./stubs.js";
|
||||
export * from "./proxmox/index.js";
|
||||
export * from "./ssh/index.js";
|
||||
export * from "./github/index.js";
|
||||
export * from "./gitea/index.js";
|
||||
export * from "./gitea/index.js";
|
||||
export {
|
||||
makeGithubMockAdapter,
|
||||
GITHUB_MOCK_TOOLS,
|
||||
CANNED_REPOS,
|
||||
CANNED_RUNS,
|
||||
CANNED_RUN,
|
||||
type GithubMockOptions,
|
||||
} from "./github-mock/adapter.js";
|
||||
@@ -89,6 +89,14 @@ export * from "./adapters/proxmox/index.js";
|
||||
export * from "./adapters/ssh/index.js";
|
||||
export * from "./adapters/github/index.js";
|
||||
export * from "./adapters/gitea/index.js";
|
||||
export {
|
||||
makeGithubMockAdapter,
|
||||
GITHUB_MOCK_TOOLS,
|
||||
CANNED_REPOS,
|
||||
CANNED_RUNS,
|
||||
CANNED_RUN,
|
||||
type GithubMockOptions,
|
||||
} from "./adapters/github-mock/adapter.js";
|
||||
|
||||
export {
|
||||
invokeCapability,
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* @coreci/mcp/adapters/github-mock — tests for the canned-repo adapter
|
||||
* (Wave J Task 5, G-018). The mock is the P0-gate reliability guarantee:
|
||||
* deterministic canned repos, no network, no failures.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
makeGithubMockAdapter,
|
||||
CANNED_REPOS,
|
||||
CANNED_RUNS,
|
||||
CANNED_RUN,
|
||||
GITHUB_MOCK_TOOLS,
|
||||
} from "../../../src/adapters/github-mock/adapter.js";
|
||||
|
||||
describe("github-mock adapter (G-018, Track A canned repos)", () => {
|
||||
it("listTools returns the 3 GitHub tools from the closed registry", async () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
const tools = await a.listTools();
|
||||
expect(tools.map((t) => t.name)).toEqual([...GITHUB_MOCK_TOOLS]);
|
||||
});
|
||||
|
||||
it("list_repos returns the canned repo array (coreci-test-repo-1, -2)", async () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
const res = await a.callTool("github.list_repos", {});
|
||||
expect(res.isError).toBe(false);
|
||||
const parsed = JSON.parse(res.content[0].text);
|
||||
expect(parsed).toEqual([...CANNED_REPOS]);
|
||||
expect(parsed.map((r: { name: string }) => r.name)).toEqual([
|
||||
"coreci-test-repo-1",
|
||||
"coreci-test-repo-2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("get_recent_ci_runs returns the canned runs", async () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
const res = await a.callTool("github.get_recent_ci_runs", { owner: "coreci", repo: "coreci-test-repo-1" });
|
||||
expect(res.isError).toBe(false);
|
||||
expect(JSON.parse(res.content[0].text)).toEqual(CANNED_RUNS);
|
||||
});
|
||||
|
||||
it("get_workflow_run returns the canned single run", async () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
const res = await a.callTool("github.get_workflow_run", { owner: "coreci", repo: "coreci-test-repo-1", run_id: 101 });
|
||||
expect(res.isError).toBe(false);
|
||||
expect(JSON.parse(res.content[0].text)).toEqual(CANNED_RUN);
|
||||
});
|
||||
|
||||
it("returns isError:true on an unknown tool", async () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
const res = await a.callTool("github.bogus", {});
|
||||
expect(res.isError).toBe(true);
|
||||
expect(res.content[0].text).toContain("unknown tool");
|
||||
});
|
||||
|
||||
it("deterministic — same call returns the same result", async () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
const r1 = await a.callTool("github.list_repos", {});
|
||||
const r2 = await a.callTool("github.list_repos", {});
|
||||
expect(r1).toEqual(r2);
|
||||
});
|
||||
|
||||
it("the adapter type is 'github' (broker routes identically to real)", () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
expect(a.type).toBe("github");
|
||||
});
|
||||
|
||||
it("supports the forced-error variant (isError option)", async () => {
|
||||
const a = makeGithubMockAdapter({ isError: true });
|
||||
const res = await a.callTool("github.list_repos", {});
|
||||
expect(res.isError).toBe(true);
|
||||
});
|
||||
|
||||
it("supports custom canned repos via options", async () => {
|
||||
const a = makeGithubMockAdapter({ repos: [{ name: "custom-repo" }] });
|
||||
const res = await a.callTool("github.list_repos", {});
|
||||
expect(JSON.parse(res.content[0].text)).toEqual([{ name: "custom-repo" }]);
|
||||
});
|
||||
|
||||
it("produces MCP-shaped result {content:[{type:'text',text}], isError}", async () => {
|
||||
const a = makeGithubMockAdapter();
|
||||
const res = await a.callTool("github.list_repos", {});
|
||||
expect(res).toHaveProperty("content");
|
||||
expect(Array.isArray(res.content)).toBe(true);
|
||||
expect(res.content[0]).toHaveProperty("type", "text");
|
||||
expect(typeof res.content[0].text).toBe("string");
|
||||
expect(res).toHaveProperty("isError", false);
|
||||
});
|
||||
});
|
||||
Generated
+30
@@ -8,6 +8,9 @@ importers:
|
||||
|
||||
.:
|
||||
devDependencies:
|
||||
'@coreci/llm-mock':
|
||||
specifier: workspace:*
|
||||
version: link:packages/llm-mock
|
||||
'@coreci/mcp':
|
||||
specifier: workspace:*
|
||||
version: link:packages/mcp
|
||||
@@ -66,6 +69,9 @@ importers:
|
||||
specifier: ^8.18.0
|
||||
version: 8.21.3
|
||||
devDependencies:
|
||||
'@coreci/llm-mock':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/llm-mock
|
||||
'@eslint/js':
|
||||
specifier: 9.39.5
|
||||
version: 9.39.5
|
||||
@@ -193,6 +199,30 @@ importers:
|
||||
specifier: ^2.1.0
|
||||
version: 2.1.9(@types/node@22.20.1)(lightningcss@1.33.0)(supports-color@7.2.0)
|
||||
|
||||
packages/llm-mock:
|
||||
devDependencies:
|
||||
'@eslint/js':
|
||||
specifier: 9.39.5
|
||||
version: 9.39.5
|
||||
'@types/node':
|
||||
specifier: ^22.0.0
|
||||
version: 22.20.1
|
||||
eslint:
|
||||
specifier: 9.39.5
|
||||
version: 9.39.5(supports-color@7.2.0)
|
||||
tsx:
|
||||
specifier: ^4.19.0
|
||||
version: 4.23.12
|
||||
typescript:
|
||||
specifier: ^5.6.0
|
||||
version: 5.9.3
|
||||
typescript-eslint:
|
||||
specifier: 8.39.0
|
||||
version: 8.39.0(eslint@9.39.5(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)
|
||||
vitest:
|
||||
specifier: ^2.1.0
|
||||
version: 2.1.9(@types/node@22.20.1)(lightningcss@1.33.0)(supports-color@7.2.0)
|
||||
|
||||
packages/mcp:
|
||||
dependencies:
|
||||
'@coreci/db':
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* check-llm-mock-guard.mjs — build-time grep guard (R-008, G-018 Task 6).
|
||||
*
|
||||
* `@coreci/llm-mock` is a CI-only devDependency. It MUST NOT appear in the
|
||||
* prod build output of the control plane (`.next/`), nor in the prod source
|
||||
* of the broker (`packages/mcp/src/`). The eslint `no-restricted-imports`
|
||||
* rule is the primary guard; this grep is the defense-in-depth backstop for
|
||||
* a stray import that slips past linting (e.g. a dynamic import string).
|
||||
*
|
||||
* Run via `pnpm check:llm-mock-guard`. Exits non-zero if `llm-mock` appears in:
|
||||
* - apps/control-plane/.next/** (after `pnpm build`)
|
||||
* - apps/control-plane/app/** (prod source)
|
||||
* - apps/control-plane/lib/** (prod source)
|
||||
* - packages/mcp/src/** (prod broker source)
|
||||
*
|
||||
* Exempts tests/** (the smoke imports the mock) and packages/llm-mock/**
|
||||
* (the package itself).
|
||||
*
|
||||
* Usage: node scripts/check-llm-mock-guard.mjs [--built]
|
||||
* --built: also scan apps/control-plane/.next/ (after a build). Skipped by
|
||||
* default so the check runs fast in CI before the build step.
|
||||
*/
|
||||
import { readdirSync, statSync, readFileSync, existsSync } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
import { argv, cwd, exit } from "node:process";
|
||||
|
||||
const root = cwd();
|
||||
const scanBuilt = argv.includes("--built");
|
||||
|
||||
/** Directories whose PROD source must not import @coreci/llm-mock. */
|
||||
const prodSourceRoots = [
|
||||
join(root, "apps/control-plane/app"),
|
||||
join(root, "apps/control-plane/lib"),
|
||||
join(root, "packages/mcp/src"),
|
||||
];
|
||||
|
||||
/** Build output directories scanned only with --built (after `pnpm build`). */
|
||||
const builtRoots = scanBuilt ? [join(root, "apps/control-plane/.next")] : [];
|
||||
|
||||
/** Extensions to scan (source + bundled JS). */
|
||||
const exts = [".ts", ".tsx", ".js", ".mjs", ".cjs", ".jsx"];
|
||||
|
||||
/** Walk a directory recursively, yielding file paths matching the extensions. */
|
||||
function* walk(dir) {
|
||||
if (!existsSync(dir)) return;
|
||||
for (const entry of readdirSync(dir)) {
|
||||
if (entry === "node_modules" || entry === ".git") continue;
|
||||
const full = join(dir, entry);
|
||||
const st = statSync(full);
|
||||
if (st.isDirectory()) {
|
||||
yield* walk(full);
|
||||
} else if (st.isFile() && exts.some((e) => entry.endsWith(e))) {
|
||||
yield full;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The forbidden substrings (catch any import path into the mock). */
|
||||
const forbidden = ["@coreci/llm-mock", "llm-mock/server", "llm-mock/patterns", "llm-mock/retry"];
|
||||
|
||||
let violations = 0;
|
||||
const roots = [...prodSourceRoots, ...builtRoots];
|
||||
for (const rootDir of roots) {
|
||||
for (const file of walk(rootDir)) {
|
||||
let content;
|
||||
try {
|
||||
content = readFileSync(file, "utf8");
|
||||
} catch {
|
||||
continue; // unreadable (binary) — skip
|
||||
}
|
||||
for (const needle of forbidden) {
|
||||
if (content.includes(needle)) {
|
||||
const rel = relative(root, file);
|
||||
console.error(`[llm-mock-guard] VIOLATION: '${needle}' found in ${rel}`);
|
||||
violations++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (violations > 0) {
|
||||
console.error(`\n[llm-mock-guard] ${violations} violation(s) found. @coreci/llm-mock is CI-only (R-008).`);
|
||||
console.error("Remove the import from prod code; the LLM smoke imports the mock from tests/**.");
|
||||
exit(1);
|
||||
}
|
||||
console.log(`[llm-mock-guard] OK — no @coreci/llm-mock imports in prod source${scanBuilt ? " or build output" : ""}.`);
|
||||
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* llm-smoke.test.ts — Two-track LLM smoke (Wave J Task 5, G-018, G-019, gate item 8).
|
||||
*
|
||||
* THE P0 GATE TEST (spec §6 gate item 8 — not deferrable to M3). Proves the
|
||||
* full OpenAI → MCP → adapter → result → synthesis path works end-to-end.
|
||||
*
|
||||
* ─── Track A (mock-path, P0 gate, runs ALWAYS) ─────────────────────────────
|
||||
* 1. Register the `github-mock` adapter (canned repos coreci-test-repo-1, -2)
|
||||
* on an in-process transport.
|
||||
* 2. Send POST /v1/chat/completions (via handleChatCompletion, no HTTP needed
|
||||
* in-process) with tools=[github.list_repos] + prompt "List my GitHub
|
||||
* repositories."
|
||||
* 3. llm-mock returns tool_calls:[{function:{name:"github.list_repos",
|
||||
* arguments:"{}"}}] (hardened pattern, G-019).
|
||||
* 4. The broker translator (toolCallToMcp) → MCP tools/call → routes to
|
||||
* github-mock → canned repo data.
|
||||
* 5. The broker translator (mcpResultToToolMessage) → OpenAI tool message.
|
||||
* 6. Second POST /v1/chat/completions with the full history: [prompt,
|
||||
* assistant tool_call, tool message].
|
||||
* 7. llm-mock synthesizes "Your repos are: coreci-test-repo-1,
|
||||
* coreci-test-repo-2". Assert the canned repo names appear.
|
||||
*
|
||||
* Track A NEVER fails (deterministic mock + canned adapter — no network).
|
||||
* This is the P0 reliability guarantee: the mock-path proves the integration
|
||||
* path with no external dependency.
|
||||
*
|
||||
* ─── Track B (real-path, optional, allow-failure) ─────────────────────────
|
||||
* Same flow against the REAL GitHub adapter with a real PAT
|
||||
* (GITHUB_SMOKE_PAT env var). Track B is gated on the PAT being present —
|
||||
* when absent, the track is SKIPPED (not failed). When present, the smoke
|
||||
* retries 429/5xx/timeout 3× with exp backoff (G-019, retry.ts); on final
|
||||
* failure it SKIPS with a warning (the P0 gate does NOT depend on Track B).
|
||||
*
|
||||
* The Track-B adapter resolves the PAT via a fake SecretProvider that returns
|
||||
* the env-var value. The real GitHub adapter calls GET /user/repos (R-004).
|
||||
*
|
||||
* Track B is `it.skip` when GITHUB_SMOKE_PAT is unset; if the call fails after
|
||||
* retries, the test SUCCEEDS (the failure is logged but does not block) — this
|
||||
* is the "allow-failure" semantic.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import {
|
||||
InProcessTransport,
|
||||
makeGithubMockAdapter,
|
||||
makeGithubAdapter,
|
||||
toolCallToMcp,
|
||||
mcpResultToToolMessage,
|
||||
toolsToOpenAi,
|
||||
listTools,
|
||||
getRegistryEntry,
|
||||
type McpResult,
|
||||
} from "@coreci/mcp";
|
||||
import { handleChatCompletion } from "@coreci/llm-mock";
|
||||
import { withRetry } from "@coreci/llm-mock/retry";
|
||||
import type { ChatMessage, ToolCall } from "@coreci/llm-mock/patterns";
|
||||
|
||||
// ─── Track A: mock-path (P0 gate) ──────────────────────────────────────────
|
||||
|
||||
describe("LLM smoke — Track A (mock-path, P0 gate, G-018)", () => {
|
||||
it("drives the full OpenAI→MCP→adapter→result→synthesis path with canned repos", async () => {
|
||||
// 1. Register the github-mock adapter (canned repos) on the transport.
|
||||
const transport = new InProcessTransport();
|
||||
await transport.register(makeGithubMockAdapter());
|
||||
expect(transport.isRegistered("github")).toBe(true);
|
||||
|
||||
// 2. Build the OpenAI tools param from the closed registry's github tools.
|
||||
const githubTools = listTools().filter((t) => t.name.startsWith("github."));
|
||||
const openAiTools = toolsToOpenAi(githubTools);
|
||||
expect(openAiTools.some((t) => t.function.name === "github.list_repos")).toBe(true);
|
||||
|
||||
// 3. Step 1: send the prompt → expect tool_calls.
|
||||
const prompt = "List my GitHub repositories.";
|
||||
const firstMessages: ChatMessage[] = [{ role: "user", content: prompt }];
|
||||
const firstResp = handleChatCompletion({ model: "coreci-mock", messages: firstMessages, tools: openAiTools });
|
||||
|
||||
// Step 2: assert the mock returned a github.list_repos tool_call.
|
||||
const choice = firstResp.choices[0];
|
||||
expect(choice.finish_reason).toBe("tool_calls");
|
||||
const toolCalls = choice.message.tool_calls;
|
||||
expect(toolCalls).toBeDefined();
|
||||
expect(toolCalls).toHaveLength(1);
|
||||
const tc: ToolCall = toolCalls![0];
|
||||
expect(tc.function.name).toBe("github.list_repos");
|
||||
expect(tc.function.arguments).toBe("{}");
|
||||
|
||||
// 4. Step 3: translate the OpenAI tool_call → MCP tools/call params.
|
||||
const mcpParams = toolCallToMcp(tc);
|
||||
expect(mcpParams.name).toBe("github.list_repos");
|
||||
expect(mcpParams.arguments).toEqual({});
|
||||
|
||||
// 5. Step 4: dispatch the MCP tools/call via the in-process transport →
|
||||
// the github-mock adapter → canned repo data.
|
||||
const entry = getRegistryEntry(mcpParams.name);
|
||||
expect(entry).toBeDefined();
|
||||
const adapterType = entry!.adapterType; // "github"
|
||||
const rpcResp = await transport.toolsCall(adapterType, "smoke-1", mcpParams.name, mcpParams.arguments);
|
||||
const mcpResult: McpResult = rpcResp.result;
|
||||
expect(mcpResult.isError).toBe(false);
|
||||
|
||||
// Step 5: translate the MCP result → OpenAI tool message.
|
||||
const toolMessage = mcpResultToToolMessage(mcpResult, tc.id);
|
||||
expect(toolMessage.role).toBe("tool");
|
||||
expect(toolMessage.tool_call_id).toBe(tc.id);
|
||||
// The github-mock canned content is JSON; the tool message content is the
|
||||
// raw text (the translator concatenates text blocks — the MCP content text).
|
||||
const toolContent = toolMessage.content;
|
||||
expect(toolContent).toContain("coreci-test-repo-1");
|
||||
expect(toolContent).toContain("coreci-test-repo-2");
|
||||
|
||||
// 6. Step 6: send the SECOND chat completion with the full history.
|
||||
const secondMessages: ChatMessage[] = [
|
||||
{ role: "user", content: prompt },
|
||||
{ role: "assistant", content: null, tool_calls: toolCalls },
|
||||
{ role: "tool", tool_call_id: tc.id, content: toolContent },
|
||||
];
|
||||
const secondResp = handleChatCompletion({ model: "coreci-mock", messages: secondMessages });
|
||||
|
||||
// 7. Step 7: assert the synthesized grounded response.
|
||||
const synth = secondResp.choices[0].message.content ?? "";
|
||||
expect(secondResp.choices[0].finish_reason).toBe("stop");
|
||||
expect(synth).toContain("coreci-test-repo-1");
|
||||
expect(synth).toContain("coreci-test-repo-2");
|
||||
// The deterministic synthesis format.
|
||||
expect(synth).toBe("Your repos are: coreci-test-repo-1, coreci-test-repo-2");
|
||||
});
|
||||
|
||||
it("deterministic — same prompt always yields the same synthesized output", async () => {
|
||||
const transport = new InProcessTransport();
|
||||
await transport.register(makeGithubMockAdapter());
|
||||
|
||||
async function runOnce(): Promise<string> {
|
||||
const prompt = "List my GitHub repositories.";
|
||||
const r1 = handleChatCompletion({
|
||||
model: "m",
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
});
|
||||
const tc = r1.choices[0].message.tool_calls![0];
|
||||
const mcpParams = toolCallToMcp(tc);
|
||||
const rpcResp = await transport.toolsCall("github", "x", mcpParams.name, mcpParams.arguments);
|
||||
const toolMessage = mcpResultToToolMessage(rpcResp.result, tc.id);
|
||||
const r2 = handleChatCompletion({
|
||||
model: "m",
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
{ role: "assistant", content: null, tool_calls: r1.choices[0].message.tool_calls },
|
||||
{ role: "tool", tool_call_id: tc.id, content: toolMessage.content },
|
||||
],
|
||||
});
|
||||
return r2.choices[0].message.content ?? "";
|
||||
}
|
||||
|
||||
const a = await runOnce();
|
||||
const b = await runOnce();
|
||||
expect(a).toBe(b);
|
||||
expect(a).toBe("Your repos are: coreci-test-repo-1, coreci-test-repo-2");
|
||||
});
|
||||
|
||||
it("tolerates prompt wording drift (G-019 regex set)", async () => {
|
||||
const transport = new InProcessTransport();
|
||||
await transport.register(makeGithubMockAdapter());
|
||||
|
||||
const drifts = [
|
||||
"Show me my GitHub repositories.",
|
||||
"Get repositories",
|
||||
"List my repos",
|
||||
"please display all my github repos",
|
||||
];
|
||||
for (const prompt of drifts) {
|
||||
const r1 = handleChatCompletion({ model: "m", messages: [{ role: "user", content: prompt }] });
|
||||
const tcs = r1.choices[0].message.tool_calls;
|
||||
expect(tcs).toBeDefined();
|
||||
expect(tcs![0].function.name).toBe("github.list_repos");
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces an MCP error result through the translator with the ERROR: prefix", async () => {
|
||||
// Register a forced-error github-mock variant.
|
||||
const transport = new InProcessTransport();
|
||||
await transport.register(makeGithubMockAdapter({ isError: true }));
|
||||
const r1 = handleChatCompletion({ model: "m", messages: [{ role: "user", content: "List my repos" }] });
|
||||
const tc = r1.choices[0].message.tool_calls![0];
|
||||
const mcpParams = toolCallToMcp(tc);
|
||||
const rpcResp = await transport.toolsCall("github", "x", mcpParams.name, mcpParams.arguments);
|
||||
expect(rpcResp.result.isError).toBe(true);
|
||||
const toolMessage = mcpResultToToolMessage(rpcResp.result, tc.id);
|
||||
expect(toolMessage.content.startsWith("ERROR:")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Track B: real-path (optional, allow-failure, G-019 retry) ──────────────
|
||||
|
||||
/** A fake SecretProvider that returns the real PAT (env var). */
|
||||
function fakeSecrets(token: string): { get: () => Promise<{ unwrap: () => string }> } {
|
||||
return {
|
||||
async get() {
|
||||
return { unwrap: () => token };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("LLM smoke — Track B (real-path, allow-failure, G-019 retry)", () => {
|
||||
const pat = process.env.GITHUB_SMOKE_PAT;
|
||||
const hasPat = typeof pat === "string" && pat.length > 0;
|
||||
|
||||
(hasPat ? it : it.skip)(
|
||||
"drives the full path against real GitHub (skipped without GITHUB_SMOKE_PAT)",
|
||||
async () => {
|
||||
// Register the REAL GitHub adapter with the env-var PAT.
|
||||
const transport = new InProcessTransport();
|
||||
const secrets = fakeSecrets(pat!);
|
||||
const adapter = makeGithubAdapter({
|
||||
tenantId: "smoke-tenant",
|
||||
targetId: "gh-real",
|
||||
config: { host: "api.github.com" },
|
||||
secrets: secrets as never,
|
||||
secretRef: "smoke:github:gh-real",
|
||||
});
|
||||
await transport.register(adapter);
|
||||
|
||||
const prompt = "List my GitHub repositories.";
|
||||
const r1 = handleChatCompletion({ model: "m", messages: [{ role: "user", content: prompt }] });
|
||||
const tc = r1.choices[0].message.tool_calls![0];
|
||||
expect(tc.function.name).toBe("github.list_repos");
|
||||
const mcpParams = toolCallToMcp(tc);
|
||||
|
||||
// Wrap the real adapter call in the G-019 retry policy: on 429/5xx/timeout,
|
||||
// retry 3× with exp backoff (1s, 2s, 4s). On FINAL failure, skip with a
|
||||
// warning (allow-failure — Track B never blocks the P0 gate).
|
||||
let realResult: McpResult;
|
||||
try {
|
||||
const rpcResp = await withRetry(
|
||||
() => transport.toolsCall("github", "smoke-b", mcpParams.name, mcpParams.arguments),
|
||||
{
|
||||
sleep: async (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
maxAttempts: 3,
|
||||
baseMs: 1000,
|
||||
},
|
||||
);
|
||||
realResult = rpcResp.result;
|
||||
} catch (err) {
|
||||
// allow-failure: log + pass (the P0 gate is Track A).
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`[llm-smoke Track B] real GitHub failed after retries — skipping (allow-failure):`, err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (realResult.isError) {
|
||||
// A scope-mismatch or upstream error → allow-failure (Track A is the gate).
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`[llm-smoke Track B] adapter returned isError — skipping (allow-failure): ${realResult.content[0]?.text}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Translate → tool message → synthesis → assert real repo names present.
|
||||
const toolMessage = mcpResultToToolMessage(realResult, tc.id);
|
||||
const r2 = handleChatCompletion({
|
||||
model: "m",
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
{ role: "assistant", content: null, tool_calls: r1.choices[0].message.tool_calls },
|
||||
{ role: "tool", tool_call_id: tc.id, content: toolMessage.content },
|
||||
],
|
||||
});
|
||||
const synth = r2.choices[0].message.content ?? "";
|
||||
// We don't assert specific repo names (the real test org may have any
|
||||
// repos); we assert the synthesis is non-empty and grounded in tool data.
|
||||
expect(typeof synth).toBe("string");
|
||||
expect(synth.length).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("Track B is skipped (not failed) when GITHUB_SMOKE_PAT is unset", () => {
|
||||
// This test documents the allow-failure semantic explicitly. When the PAT
|
||||
// is unset, the Track B test above is `it.skip` — the P0 gate (Track A)
|
||||
// does NOT depend on Track B. This test passes either way (it's a doc).
|
||||
if (!hasPat) {
|
||||
expect(hasPat).toBe(false);
|
||||
} else {
|
||||
expect(hasPat).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("the retry policy retries 3× on 429 then gives up (allow-failure)", async () => {
|
||||
// Verifies the retry wrapper's allow-failure behavior: a persistent 429
|
||||
// exhausts retries, rethrows, and the caller skips (does NOT fail the gate).
|
||||
let calls = 0;
|
||||
const always429 = async () => {
|
||||
calls++;
|
||||
throw Object.assign(new Error("429"), { status: 429 });
|
||||
};
|
||||
const sleep = vi.fn(async () => {});
|
||||
await expect(
|
||||
withRetry(always429, { sleep, maxAttempts: 3, baseMs: 1 }),
|
||||
).rejects.toThrow("429");
|
||||
expect(calls).toBe(3);
|
||||
expect(sleep).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,17 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
// Root vitest config for the MCP conformance verification suite
|
||||
// (tests/mcp-conformance/, R-001, G-017, M2 gate item 15). Run via
|
||||
// (tests/mcp-conformance/, R-001, G-017, M2 gate item 15) AND the two-track
|
||||
// LLM smoke (tests/llm-smoke/, G-018, G-019, M2 gate item 8). Run via
|
||||
// `pnpm test:conformance` (defined in the root package.json). The broker
|
||||
// modules resolve via the workspace @coreci/mcp symlink.
|
||||
// modules + the llm-mock package resolve via the workspace symlinks.
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
include: ["tests/mcp-conformance/**/*.test.ts"],
|
||||
include: [
|
||||
"tests/mcp-conformance/**/*.test.ts",
|
||||
"tests/llm-smoke/**/*.test.ts",
|
||||
],
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user