Merge milestone/v1.0-initial into main (InboxOS v1.0 release)
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
{
|
||||
"projects": [
|
||||
{
|
||||
"slug": "coreci",
|
||||
"name": "CoreCI",
|
||||
"milestone": "v1.8",
|
||||
"status": "execute"
|
||||
},
|
||||
{
|
||||
"slug": "ci",
|
||||
"name": "CIAgent",
|
||||
"milestone": "v0.13",
|
||||
"status": "specify"
|
||||
},
|
||||
{
|
||||
"slug": "foo",
|
||||
"name": "Foo",
|
||||
"milestone": "v0.1",
|
||||
"status": "specify"
|
||||
},
|
||||
{
|
||||
"slug": "inboxos",
|
||||
"name": "InboxOS",
|
||||
"milestone": "v1.0",
|
||||
"status": "specify"
|
||||
}
|
||||
],
|
||||
"active_project": "inboxos",
|
||||
"active_projects": ["inboxos"],
|
||||
"autonomy": {
|
||||
"level": "full",
|
||||
"escalation_hooks": ["deploy", "delete_data", "merge_to_main"],
|
||||
"clarify_budget": 10,
|
||||
"decision_confidence_threshold": 0.6,
|
||||
"max_revision_iterations": 3,
|
||||
"max_verification_retries": 2,
|
||||
"escalation_timeout_ms": 300000
|
||||
},
|
||||
"model_profile": "quality",
|
||||
"parallelization": {
|
||||
"enabled": true,
|
||||
"max_concurrent_agents": 5,
|
||||
"min_plans_for_parallel": 2,
|
||||
"max_concurrent_projects": 3
|
||||
},
|
||||
"verification": {
|
||||
"automated_only": true,
|
||||
"escalate_visual": true,
|
||||
"escalate_external_integration": true,
|
||||
"test_first": false
|
||||
},
|
||||
"security": {
|
||||
"auto_accept_low_severity": true,
|
||||
"auto_mitigate_medium_severity": true,
|
||||
"escalate_high_severity": true
|
||||
},
|
||||
"git": {
|
||||
"branching_strategy": "phase",
|
||||
"auto_commit": true,
|
||||
"auto_push": true
|
||||
},
|
||||
"sessions": {
|
||||
"max_concurrent_sessions": 3,
|
||||
"session_timeout_ms": 3600000,
|
||||
"session_isolation": "branch"
|
||||
},
|
||||
"personas": {
|
||||
"enabled": true,
|
||||
"territory_enforcement": "warn",
|
||||
"personas": [
|
||||
{
|
||||
"name": "lead-developer",
|
||||
"domain": "coordination",
|
||||
"frameworks": [],
|
||||
"constraints": ["pragmatic", "battle-tested defaults"],
|
||||
"territory": []
|
||||
},
|
||||
{
|
||||
"name": "data-engineer",
|
||||
"domain": "data",
|
||||
"frameworks": ["drizzle", "postgresql"],
|
||||
"constraints": ["schema-first", "type-safe ORM", "migration-driven"],
|
||||
"territory": ["**/migrations/**", "**/schema/**", "**/models/**", "**/db/**", "drizzle/**", "**/*.sql"]
|
||||
},
|
||||
{
|
||||
"name": "backend-engineer",
|
||||
"domain": "backend",
|
||||
"frameworks": ["fastify", "hono"],
|
||||
"constraints": ["api-first", "strict-typing", "dependency-injection"],
|
||||
"territory": ["**/api/**", "**/routes/**", "**/services/**", "**/middleware/**", "**/controllers/**", "**/auth/**"]
|
||||
},
|
||||
{
|
||||
"name": "frontend-engineer",
|
||||
"domain": "frontend",
|
||||
"frameworks": ["react", "next.js"],
|
||||
"constraints": ["component-first", "server-components", "minimal-client-js"],
|
||||
"territory": ["**/components/**", "**/pages/**", "**/hooks/**", "**/styles/**", "**/*.tsx", "**/*.css"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gitea": {
|
||||
"base_url": "https://git.cloudinit.dev",
|
||||
"api_token_env": "b8fa72220e350314704a8a28a0e570b418bb4655",
|
||||
"owner": "cto-inboxos",
|
||||
"repo": "inboxos"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
# InboxOS — Architecture (initial)
|
||||
|
||||
> Initial architecture, may be incomplete. Refine in subsequent phases.
|
||||
|
||||
## System Diagram
|
||||
|
||||
```
|
||||
┌────────────────────┐
|
||||
│ Gmail (IMAP) │
|
||||
└─────────┬──────────┘
|
||||
│ IMAP poll (cron */5 * * * *)
|
||||
▼
|
||||
┌────────────────────┐
|
||||
│ live-pipeline.yml │ (Gitea Actions, self-hosted runner)
|
||||
│ live_pipeline.py │
|
||||
└─────┬───────┬──────┘
|
||||
│ │
|
||||
archive .eml│ │extract via Ollama
|
||||
▼ ▼
|
||||
┌──────────┐ ┌──────────────────┐
|
||||
│ SeaweedFS│ │ Ollama │
|
||||
│ (S3 API) │ │ gemma4:e4b │
|
||||
└──────────┘ │ inboxos:classify│
|
||||
│ inboxos:extract │
|
||||
└────────┬─────────┘
|
||||
│ structured record
|
||||
▼
|
||||
┌────────────────────┐
|
||||
│ NocoDB │
|
||||
│ finance │
|
||||
│ crm │
|
||||
│ intel │
|
||||
│ promotions │
|
||||
└────────────────────┘
|
||||
|
||||
historical (workflow_dispatch):
|
||||
.mbox file → historical_pipeline.py → mbox_stream.py → same extract+upsert path
|
||||
```
|
||||
|
||||
## Components
|
||||
|
||||
### Ingest
|
||||
|
||||
- **Gmail IMAP** (single inbox). The only external dependency that touches PII.
|
||||
- **Live pipeline** (`pipelines/live_pipeline.py`): Gitea Actions `cron */5 * * * *`, fetches new mail, archives to SeaweedFS, classifies + extracts via Ollama, upserts to NocoDB.
|
||||
- **Historical pipeline** (`pipelines/historical_pipeline.py` + `pipelines/mbox_stream.py`): Gitea Actions `workflow_dispatch`, streams a local `.mbox` end-to-end with the same extract+upsert logic.
|
||||
|
||||
### Storage
|
||||
|
||||
- **SeaweedFS** (`docker/docker-compose.yml`): S3-compatible object store. Holds raw `.eml` and attachments, keyed by Message-ID and content hash. Archive of record.
|
||||
- **NocoDB** (`docker/docker-compose.yml`): Structured store. Four module tables: `finance`, `crm`, `intel`, `promotions`. Idempotency keyed by Message-ID via `nocodb_upsert` v2 bulk-PATCH format.
|
||||
|
||||
### Extraction (LLM)
|
||||
|
||||
- **Ollama** (`docker/docker-compose.yml`): Local LLM runtime. Single pinned model: `gemma4:e4b`. No model-escalation chain in production.
|
||||
- **Prompt family** (`.opencode/commands/`): prompt-as-code.
|
||||
- `inboxos:classify` — picks the module.
|
||||
- `inboxos:extract-<module>` — produces the structured record per module schema.
|
||||
- **Contract**: `docs/extraction-contract.md`. Source of truth for the prompt family and the record shape.
|
||||
|
||||
### Document Store (curated)
|
||||
|
||||
- **Papra** (`docker/docker-compose.yml`): Curated document store for selected attachments + bodies. **First ship is dry-run** — the pipeline emits Papra write intent but does not call the real Papra API. See `docs/papra-write-policy.md`.
|
||||
|
||||
**Live vs historical asymmetry.** The historical pipeline captures
|
||||
`Papra_Link` during the pre-classify attachment walk (see
|
||||
`pipelines/historical_pipeline.py:296-336`); the live pipeline calls
|
||||
`papra_decide` once after classify and does not pre-walk
|
||||
attachments. This asymmetry is intentional — see
|
||||
`docs/extraction-contract.md §Live vs historical: Papra_Link
|
||||
asymmetry` and `docs/papra-write-policy.md §Rule 1 — attachments
|
||||
present`.
|
||||
|
||||
### Orchestration
|
||||
|
||||
- **Gitea + Gitea Actions** (self-hosted, two runners). Live + historical can run concurrently.
|
||||
- **Gitea Actions 1.x / GHES compatibility**: action versions pinned to v3.
|
||||
|
||||
### Tests / CI
|
||||
|
||||
- **PR CI** (`.gitea/workflows/ci-pr.yml`): fast unit tests + 100MB e2e + live Ollama smoke.
|
||||
- **Synthetic UPSERT test**: `INB-12` acceptance criterion — confirms idempotent upsert end-to-end.
|
||||
|
||||
## Boundaries
|
||||
|
||||
- **PII boundary**: only the IMAP connection to Gmail leaves the self-hosted perimeter. All downstream processing is local.
|
||||
- **Idempotency boundary**: Message-ID is the universal key, from archive through NocoDB upsert.
|
||||
- **Prompt boundary**: prompts are versioned in `.opencode/commands/`; the pipeline invokes them by name; the model is pinned.
|
||||
|
||||
## Open Questions (to refine)
|
||||
|
||||
- Papra live-write gate (when to flip from dry-run).
|
||||
- Retention / rotation policy for SeaweedFS archive.
|
||||
- Observability surface (logs only today; metrics/tracing TBD).
|
||||
- Disaster recovery posture (backups of NocoDB, SeaweedFS, Gitea).
|
||||
@@ -0,0 +1,134 @@
|
||||
docs(research): InboxOS v1.0 research findings + PERSONAS.md
|
||||
|
||||
---ci---
|
||||
project: inboxos
|
||||
phase: 0
|
||||
milestone: v1.0
|
||||
status: research
|
||||
findings:
|
||||
stack:
|
||||
python: 3.12 (pinned across all three workflows)
|
||||
deps: [boto3, requests, tenacity, imapclient]
|
||||
llm: ollama @ gemma4:e4b (single pinned, no escalation)
|
||||
storage: seaweedfs (S3), nocodb (REST, default sqlite), papra (dry-run)
|
||||
orchestrator: gitea + gitea-actions 1.x, two self-hosted runners
|
||||
architecture:
|
||||
pii_boundary: only IMAP/993 to Gmail leaves self-hosted perimeter
|
||||
idempotency_boundary: Message-ID across S3 keys, NocoDB rows, Papra writes
|
||||
prompt_boundary: .opencode/commands/*.md, version-controlled, single model pin
|
||||
runners: gitea-runner + gitea-runner-2 (identical label set enables concurrency)
|
||||
features:
|
||||
wired: [live IMAP poll, historical mbox backfill, SeaweedFS archive, Ollama extract, Papra decision, Gitea REST, PR CI]
|
||||
dry_run: [Papra writes (PAPRA_DRY_RUN=true default)]
|
||||
planned: [Papra live (Phase 6), papra_index (INB-27.b), observability/DR (Phase 7)]
|
||||
pitfalls:
|
||||
- id: P-1
|
||||
severity: high
|
||||
summary: Model pin drift — PROJECT/REQUIREMENTS/common.py say gemma4:e4b; pipelines/README.md:185 and docs/extraction-contract.md:36,44 still reference ollama/lfm2.5:8b
|
||||
- id: P-2
|
||||
severity: high
|
||||
summary: Extraction contract doc is stale on LLM backend (still describes OpenCode as the LLM caller; production path is ollama)
|
||||
- id: P-3
|
||||
severity: medium
|
||||
summary: live vs historical pipeline asymmetry — historical writes Papra_Link on attachment upload; live does not
|
||||
- id: P-4
|
||||
severity: medium
|
||||
summary: OLLAMA_BASE_URL has /v1 suffix in runbook+common.py but not in workflow env defaults (works, but inconsistent)
|
||||
- id: P-5
|
||||
severity: low
|
||||
summary: docker-compose.yml Papra INTAKE_EMAILS_IS_ENABLED=true; policy says catch-all was cancelled
|
||||
- id: P-6
|
||||
severity: low
|
||||
summary: State-file field names in runbook drift from actual code
|
||||
- id: P-7
|
||||
severity: medium
|
||||
summary: live papra_decide is called once pre-classify with modules=[]; does not re-evaluate after classify. Body writes silently skipped for crm/finance no-attachment emails on live.
|
||||
personas:
|
||||
active: [lead-developer, data-engineer, backend-engineer, devops-engineer, ml-engineer, security-engineer]
|
||||
deactivated: [frontend-engineer]
|
||||
revised: [data-engineer, backend-engineer]
|
||||
new: [devops-engineer, ml-engineer, security-engineer]
|
||||
lsp_observations:
|
||||
note: pre-existing import errors in pipelines/*.py (tenacity, imapclient, email.message) — local interpreter missing runtime deps; not in scope of this commit
|
||||
execution_decisions:
|
||||
- id: ED-1
|
||||
summary: T1.6 (live pipeline post-classify papra_decide re-evaluation) is a no-op
|
||||
rationale: |
|
||||
The live pipeline already calls papra_decide ONCE with modules=modules
|
||||
(live_pipeline.py:490), which is the post-classify call. The historical
|
||||
pipeline calls it TWICE: once pre-classify in the attachment walk (which
|
||||
captures Papra_Link) and once post-classify for body writes. The live
|
||||
pipeline has no pre-classify attachment walk, so adding a second
|
||||
papra_decide call would be a no-op. The "asymmetry" is structural
|
||||
(one call vs two) and is fully captured by the T1.3 documentation
|
||||
(extraction-contract.md §Live vs historical: Papra_Link asymmetry +
|
||||
ARCHITECTURE.md §Document Store + papra-write-policy.md §Rule 1 live
|
||||
note). No code change required.
|
||||
effect: T1.6 was planned as a code change; it is now a documentation change (T1.3 already covers it). Plan must-have #19 is closed by T1.3 + this decision.
|
||||
verification:
|
||||
verdict: PASS
|
||||
confidence: high
|
||||
layers:
|
||||
structural: PASS (20/20)
|
||||
behavioral: PASS (20/20)
|
||||
security: PASS (REQ-INB-9, REQ-INB-10, REQ-INB-12, secret scan)
|
||||
quality: PASS (14/14 REQ coverage, 6/6 persona coverage, 11 well-formed ---ci--- blocks, no blocking TODO/FIXME)
|
||||
must_haves_closed: 20
|
||||
blockers: []
|
||||
notes:
|
||||
- T1.6 / PR #F was a no-op (ED-1); the asymmetry is captured by the T1.3 documentation chain.
|
||||
- T1.4 / PR #D was folded into PR #A (commit 4062f5b); both touch pipelines/README.md and the OLLAMA_BASE_URL canonicalisation landed in the same commit.
|
||||
- All 11 execution-pass commits have well-formed ---ci--- blocks; the YAML parses cleanly.
|
||||
- .env is git-ignored; no real secrets have ever been committed in any branch.
|
||||
- Pre-existing non-blocking: pipelines/_tests/test_upsert_synthetic.py:90-95 documents a known NocoDB v2026.04.3 single-record PATCH issue; production nocodb_upsert uses the bulk-PATCH format and is not affected.
|
||||
review:
|
||||
verdict: READY-WITH-NITS
|
||||
confidence: high
|
||||
p0_fixes: 0
|
||||
p1_issues: 6
|
||||
p2_issues: 3
|
||||
audit:
|
||||
reconstruction: PASS (15/15 ---ci--- blocks, status progression specify→clarify→research→plan→execute→verify, ROADMAP.md says "executed 2026-06-08", CI-LOG captures all 6 stages)
|
||||
file_discipline: PASS (all .ciagent/, .opencode/commands/, .gitea/workflows/ files present; inb-28-report.md present and non-empty)
|
||||
commit_discipline: PASS (15/15 commits have ---ci--- blocks; 15/15 follow <type>(<scope>): convention; max 2 files per commit; branch is milestone/v1.0-initial)
|
||||
post_hoc_review:
|
||||
phase_6_backlog:
|
||||
- id: P1-1
|
||||
summary: T1.3 plan-vs-PR mapping split (PR #C touched 2 files; the 3rd was in PR #B)
|
||||
- id: P1-2
|
||||
summary: Plan §9.2 row 16 cited runbook:487; actual location is §6.1 (the runbook grew 13 lines). Fixed in 32c076c.
|
||||
- id: P1-3
|
||||
summary: inb-28-report.md "Per-table row counts at freeze" is empty (header present, no actual counts). Future: mine the test's run history.
|
||||
- id: P1-4
|
||||
summary: Runbook §1 system diagram showed OpenCode CLI as a node. Fixed in 32c076c.
|
||||
- id: P1-5
|
||||
summary: .env.example lines 96-99 have stale lfm2.5:8b / gemma4:12b model comments. Pre-existing, not in milestone diff.
|
||||
- id: P1-6
|
||||
summary: CI-LOG.md ED-1 effect: was multi-line YAML continuation (interpreted as nested keys). Fixed in 32c076c.
|
||||
- id: P2-1
|
||||
summary: Runbook §9.1 #12 glob {live,historical,ci-pr}.yml is misleading (ci-pr.yml doesn't use upload-artifact)
|
||||
- id: P2-2
|
||||
summary: Runbook §6.1 prose says 21 distinct secrets; the table has 22 rows. Off-by-one.
|
||||
- id: P2-4
|
||||
summary: Plan #19 persona (backend-engineer) is implicit in the runbook renumber. The plan's #19 was the live code change; the runbook renumbered #19 to documentation.
|
||||
fixed_in_32c076c: [P1-2, P1-4, P1-6]
|
||||
remaining_for_phase_6: [P1-1, P1-3, P1-5, P2-1, P2-2, P2-4]
|
||||
complete:
|
||||
status: complete
|
||||
date: 2026-06-08
|
||||
milestone: v1.0
|
||||
requirements_covered: [REQ-INB-1, REQ-INB-2, REQ-INB-3, REQ-INB-4, REQ-INB-5, REQ-INB-6, REQ-INB-7, REQ-INB-8, REQ-INB-9, REQ-INB-10, REQ-INB-11, REQ-INB-12, REQ-INB-13, REQ-INB-14]
|
||||
requirements_partial: []
|
||||
version: v1.0.0
|
||||
branch: milestone/v1.0-initial
|
||||
commits: 17
|
||||
execution_pass_commits: 13
|
||||
ship_status: documented-not-merged
|
||||
ship_note: |
|
||||
The milestone branch milestone/v1.0-initial is the canonical
|
||||
close branch. The merge-to-main step is gated by the
|
||||
autonomy.escalation_hooks rule "merge_to_main" and requires
|
||||
explicit human approval. All execution work is on the
|
||||
milestone branch; main is left untouched. The follow-up
|
||||
run can tag v1.0.0 and merge when the operator confirms.
|
||||
---/ci---
|
||||
@@ -0,0 +1,243 @@
|
||||
---
|
||||
active:
|
||||
- lead-developer
|
||||
- data-engineer
|
||||
- backend-engineer
|
||||
- devops-engineer
|
||||
- ml-engineer
|
||||
- security-engineer
|
||||
phase_specific:
|
||||
- persona: frontend-engineer
|
||||
phases: []
|
||||
reason: "deactivated — InboxOS has no React/Next.js/JS frontend; the operator surface is the Gitea Actions UI, Gitea web UI, NocoDB web UI, and the Papra web UI. The default frameworks (react, next.js) are not present in the repository."
|
||||
- persona: data-engineer
|
||||
phases: [1, 2, 3, 4, 5, 6]
|
||||
reason: "re-scoped rather than deactivated — the structured store is NocoDB (not Postgres/Drizzle), accessed via REST, with Message-ID as the natural key and nocodb_upsert as the only write path. Schema lives in docs/schemas.md and scripts/apply_schemas.py. The persona's frameworks and territory are revised below to match."
|
||||
reason: |
|
||||
InboxOS is a self-hosted Python pipeline orchestrated by Gitea Actions YAML and shell.
|
||||
There is no TypeScript/Node service, no React UI, no Postgres, no Drizzle, no Fastify/Hono.
|
||||
The "frontend" is the Gitea/NocoDB/Papra web UIs operated on; the "backend" is two Python
|
||||
scripts plus a 3rd CI workflow. The default 4 personas in config.json (lead-developer,
|
||||
data-engineer, backend-engineer, frontend-engineer) are framework-bound to TypeScript/React
|
||||
stacks; for InboxOS three custom personas are required (devops for Gitea Actions + Docker,
|
||||
ml for Ollama + prompts, security for the PII boundary) and the frontend-engineer must
|
||||
be deactivated. The data-engineer and backend-engineer personas are retained but their
|
||||
frameworks and territory are revised to match the actual stack (NocoDB REST, Python pipelines,
|
||||
Gitea Actions YAML, prompt .md files).
|
||||
---
|
||||
|
||||
# InboxOS — Active personas
|
||||
|
||||
This document is the active persona roster for the InboxOS project (`v1.0`,
|
||||
Phase 5 — Full Deployment). It is derived from the default roster in
|
||||
`.ciagent/config.json` and adjusted to match the actual project stack as
|
||||
documented in `PROJECT.md`, `ARCHITECTURE.md`, `REQUIREMENTS.md`, and the
|
||||
source files under `pipelines/`, `.gitea/workflows/`, `docker/`, and `docs/`.
|
||||
|
||||
## 1. lead-developer (active)
|
||||
|
||||
- **name**: lead-developer
|
||||
- **domain**: coordination
|
||||
- **frameworks**: []
|
||||
- **constraints**:
|
||||
- pragmatic
|
||||
- battle-tested defaults
|
||||
- idempotency-first (REQ-INB-3)
|
||||
- self-hosted-PII-only (REQ-INB-10)
|
||||
- **territory**:
|
||||
- `**/*.md` (project docs)
|
||||
- `.ciagent/inboxos/**`
|
||||
- `README.md`
|
||||
|
||||
The lead owns the cross-cutting invariants: the Message-ID idempotency
|
||||
contract, the PII boundary, and the prompt-as-code governance model.
|
||||
Coordinates the devops, ml, data, and security personas on shared
|
||||
boundaries (archive key derivation, schema drift, secret rotation).
|
||||
|
||||
## 2. data-engineer (active, RE-SCOPED)
|
||||
|
||||
- **name**: data-engineer
|
||||
- **domain**: data
|
||||
- **frameworks**:
|
||||
- `nocodb` (REST API, xc-token, `/api/v2/tables/<id>/records`)
|
||||
- `boto3` (SeaweedFS S3 API — archive of record)
|
||||
- `tenacity` (retry policy for S3 / NocoDB writes)
|
||||
- `json` (structured log events, state files)
|
||||
- **constraints**:
|
||||
- Message-ID as the natural key (REQ-INB-3, REQ-INB-5)
|
||||
- GET-then-PATCH/POST for every write (no plain INSERT path — `common.py:206-256`)
|
||||
- deterministic object keys (`eml/<kind>/<yyyy>/<mm>/<dd>/<mid>.eml`)
|
||||
- schema-first — `docs/schemas.md` is the source of truth, NocoDB is the target
|
||||
- **territory**:
|
||||
- `pipelines/common.py` (nocodb_upsert, s3_* helpers, state load/save)
|
||||
- `pipelines/live_pipeline.py` (NocoDB write + state flush)
|
||||
- `pipelines/historical_pipeline.py` (NocoDB write + state flush)
|
||||
- `pipelines/mbox_stream.py` (offset-based checkpoint semantics)
|
||||
- `pipelines/_tests/test_historical_pipeline_offline.py` (the merge gate for UPSERT idempotency)
|
||||
- `pipelines/_tests/test_upsert_synthetic.py`
|
||||
- `docs/schemas.md`
|
||||
- `scripts/apply_schemas.py`
|
||||
- `docs/phase-1.1-verification/inb-28-report.md` (locks in table IDs and column counts)
|
||||
|
||||
**Revision rationale**: the default `data-engineer` was scoped to Postgres + Drizzle +
|
||||
migrations, which is not the InboxOS stack. InboxOS stores structured records in
|
||||
NocoDB (a self-hosted Airtable-like UI, REST API) and the archive of record in
|
||||
SeaweedFS (S3-compatible). The persona is retained because the data-layer
|
||||
invariants (idempotency, schema-first, deterministic keys) translate 1:1; only
|
||||
the frameworks and territory change.
|
||||
|
||||
## 3. backend-engineer (active, RE-SCOPED)
|
||||
|
||||
- **name**: backend-engineer
|
||||
- **domain**: backend
|
||||
- **frameworks**:
|
||||
- `python` 3.12 (pinned in all three workflows)
|
||||
- `imapclient` (IMAP polling)
|
||||
- `requests` (NocoDB, Ollama, Papra, Gitea REST)
|
||||
- `boto3` (SeaweedFS)
|
||||
- `tenacity` (retry/backoff)
|
||||
- `gitea-actions` (workflow YAML, dispatch, secrets, vars)
|
||||
- `docker-compose` (self-hosted infra)
|
||||
- **constraints**:
|
||||
- "api-first" applies to the NocoDB + Gitea + Papra + Ollama + OpenAI-compatible interfaces the pipeline calls, not to a project-owned HTTP server
|
||||
- strict-typing (dataclasses throughout `common.py`, `live_pipeline.py`, `historical_pipeline.py`)
|
||||
- dependency-injection (config via `XConfig.from_env()` dataclasses)
|
||||
- air-gapped: only the IMAP connection to Gmail leaves the self-hosted perimeter (REQ-INB-10)
|
||||
- **territory**:
|
||||
- `pipelines/common.py`
|
||||
- `pipelines/live_pipeline.py`
|
||||
- `pipelines/historical_pipeline.py`
|
||||
- `pipelines/mbox_stream.py`
|
||||
- `pipelines/_tests/**`
|
||||
- `.gitea/workflows/live-pipeline.yml`
|
||||
- `.gitea/workflows/historical-pipeline.yml`
|
||||
- `.gitea/workflows/ci-pr.yml`
|
||||
- `docker/docker-compose.yml`
|
||||
|
||||
**Revision rationale**: the default `backend-engineer` was scoped to Fastify/Hono
|
||||
+ REST routes + services. InboxOS has no application HTTP server — the "backend"
|
||||
is two Python scripts (live, historical) that call out to NocoDB / Ollama / Papra /
|
||||
Gitea REST / SeaweedFS, plus three Gitea Actions workflow YAMLs that orchestrate
|
||||
them. The persona is retained because the engineering discipline (strict typing,
|
||||
DI, idempotent side effects, explicit failure handling) translates; the
|
||||
frameworks and territory change.
|
||||
|
||||
## 4. frontend-engineer (DEACTIVATED for this project)
|
||||
|
||||
- **name**: frontend-engineer
|
||||
- **domain**: frontend
|
||||
- **frameworks**: [`react`, `next.js`] (carried from default; not used)
|
||||
- **constraints**: [`component-first`, `server-components`, `minimal-client-js`] (carried from default; not used)
|
||||
- **territory**: [] (empty — no React/TSX/CSS files exist in this project)
|
||||
- **status**: **deactivated** in `active:` list. The default frameworks (React,
|
||||
Next.js) are not present anywhere in the repository. The operator-facing
|
||||
surfaces are the self-hosted Gitea web UI, NocoDB web UI, and Papra web UI —
|
||||
all third-party products, not code owned by this project. Listed in
|
||||
`phase_specific` with `phases: []` to make the deactivation explicit.
|
||||
|
||||
## 5. devops-engineer (active, NEW)
|
||||
|
||||
- **name**: devops-engineer
|
||||
- **domain**: devops / infrastructure
|
||||
- **frameworks**:
|
||||
- `gitea-actions` (self-hosted runners, `act_runner`, workflow YAML v3 pinning for GHES compat)
|
||||
- `docker-compose` (the `inboxos-net` bridge, named volumes, FRP-tunneled Ollama)
|
||||
- `seaweedfs` (master/volume/filer topology, S3 API)
|
||||
- `caddy` (commented-out reverse proxy in `docker-compose.yml`)
|
||||
- `bash` (the per-run logs upload + `apply_schemas.py` + `setup_gitea_secrets.sh` + `push_to_gitea.sh`)
|
||||
- **constraints**:
|
||||
- Gitea Actions 1.x / GHES compatibility — `actions/upload-artifact@v3`, never v4+ (REQ-INB-12)
|
||||
- per-job secrets via `secrets.X` and `vars.X`, never committed
|
||||
- two-runner topology (REQ-INB-7, INB-33) — `gitea-runner` + `gitea-runner-2`
|
||||
- Basic Auth for Gitea REST (`GITEA_USER`/`GITEA_PASS`) — Gitea does not surface the plaintext of freshly-issued PATs
|
||||
- self-hosted PII boundary — only the IMAP connection leaves the network
|
||||
- **territory**:
|
||||
- `docker/docker-compose.yml`
|
||||
- `.gitea/workflows/live-pipeline.yml`
|
||||
- `.gitea/workflows/historical-pipeline.yml`
|
||||
- `.gitea/workflows/ci-pr.yml`
|
||||
- `docs/gitea-actions.md`
|
||||
- `docs/operations-runbook.md` (services + observability + failure modes)
|
||||
- `scripts/push_to_gitea.sh`
|
||||
- `scripts/setup_gitea_secrets.sh`
|
||||
|
||||
**Why this persona is required**: InboxOS is a workflow-as-code project. The
|
||||
"deployment surface" is two self-hosted runner containers, six service
|
||||
containers, a Caddy reverse proxy, and three workflow YAML files. None of the
|
||||
default personas cover this. The devops-engineer owns the runner provisioning,
|
||||
the artifact-upload v3 pin (which is the single most common CI failure mode
|
||||
referenced in the codebase), the FRP-tunneled Ollama URL, the Papra intake
|
||||
catch-all cancellation, and the per-run log artifact pipeline.
|
||||
|
||||
## 6. ml-engineer (active, NEW)
|
||||
|
||||
- **name**: ml-engineer
|
||||
- **domain**: ml / llm
|
||||
- **frameworks**:
|
||||
- `ollama` (OpenAI-compatible `/v1/chat/completions` endpoint)
|
||||
- `gemma4:e4b` (single pinned model, REQ-INB-11)
|
||||
- `tenacity` (3 attempts, exponential backoff, in `common.py:525-530`)
|
||||
- prompt-frontmatter YAML (model declaration in `.opencode/commands/*.md`)
|
||||
- **constraints**:
|
||||
- single-model pin, no escalation chain (REQ-INB-11, `common.py:439-443`)
|
||||
- prompt-as-code — `.opencode/commands/*.md` is the source of truth, version-controlled in the same repo
|
||||
- classify-first, then extract-per-module (not "one big extract all")
|
||||
- strict-JSON output, parsed by `json.JSONDecoder.raw_decode` over the model's reply (`common.py:613-624`)
|
||||
- `INBOXOS_BODY_CHARS=2000` cap on the email body sent to the LLM (prevents 86KB Substack OOM)
|
||||
- `INBOXOS_LLM_MAX_TOKENS=4096` headroom for 11-field extract schema
|
||||
- **territory**:
|
||||
- `.opencode/commands/inboxos:classify.md`
|
||||
- `.opencode/commands/inboxos:extract-finance.md`
|
||||
- `.opencode/commands/inboxos:extract-crm.md`
|
||||
- `.opencode/commands/inboxos:extract-intel.md`
|
||||
- `.opencode/commands/inboxos:extract-promotions.md`
|
||||
- `pipelines/common.py::llm_run_command` (line 632)
|
||||
- `pipelines/common.py::ollama_run_command` (line 531)
|
||||
- `pipelines/common.py::_load_prompt` (line 498)
|
||||
- `pipelines/common.py::LLMConfig` (line 414)
|
||||
- `docs/extraction-contract.md`
|
||||
|
||||
**Why this persona is required**: the extraction contract is the heart of the
|
||||
project. The 5 prompt files are co-located with the code that invokes them
|
||||
(prompt-as-code, REQ-INB-14), and the LLM backend is a self-hosted Ollama
|
||||
running `gemma4:e4b` with no cloud fallback. No default persona covers Ollama
|
||||
prompt-engineering or model-pin governance.
|
||||
|
||||
## 7. security-engineer (active, NEW)
|
||||
|
||||
- **name**: security-engineer
|
||||
- **domain**: security
|
||||
- **frameworks**:
|
||||
- `imap` (Gmail IMAP, App Password, 2FA required)
|
||||
- `frp` (Fast Reverse Proxy — Ollama tunnel termination)
|
||||
- `gitea-basic-auth` (HTTP Basic for Gitea REST, see `common.gitea_request`)
|
||||
- `boto3` (S3 access keys — long-lived, single-operator)
|
||||
- `xc-token` (NocoDB API token, `xc-token` header)
|
||||
- **constraints**:
|
||||
- REQ-INB-10: no third-party SaaS shall process PII
|
||||
- the only outbound network call carrying PII is IMAP/993 to Gmail
|
||||
- Gitea service account is scoped to `cto-inboxos/inboxos` repo only (no instance admin)
|
||||
- all secret reads via `${{ secrets.X }}` — never committed, never logged
|
||||
- the prompt boundary is a hard boundary: prompts are co-located, version-controlled, never embedded in workflow YAML or Python strings
|
||||
- **territory**:
|
||||
- `docs/operations-runbook.md` (the entire §6 — Secret inventory, Why we never commit secrets, Data-sovereignty posture)
|
||||
- `docs/gitea-actions.md` (auth model §5, Basic Auth rationale)
|
||||
- `docs/papra-write-policy.md` (PII flow: attachment vs body write decisions)
|
||||
- `docker/docker-compose.yml` (network isolation `inboxos-net`, named volumes)
|
||||
- `.gitea/workflows/*.yml` (where `${{ secrets.* }}` is referenced)
|
||||
- `pipelines/common.py::gitea_request` (Basic Auth + retry policy, lines 305-339)
|
||||
- `pipelines/common.py::GiteaConfig` (lines 278-302)
|
||||
- `.env.example` and `.gitignore` (placeholder-only, excludes `.env` and `.env.*`)
|
||||
|
||||
**Why this persona is required**: InboxOS's PII boundary is its core value
|
||||
proposition. The "no third-party SaaS touches PII" rule is enforced in three
|
||||
places: the Docker network isolation, the secret discipline (no committed
|
||||
secrets, Basic Auth instead of PATs), and the data-sovereignty posture (FRP
|
||||
tunnel for Ollama). No default persona owns this boundary.
|
||||
|
||||
## Cross-references
|
||||
|
||||
- `PROJECT.md` (REQ-INB-1 through REQ-INB-14) — requirements each persona guards
|
||||
- `ARCHITECTURE.md` — boundaries (PII, idempotency, prompt) that map to persona constraints
|
||||
- `config.json::personas` — the roster this document supersedes
|
||||
- `pipelines/common.py` — the single shared module all personas touch (lead owns; others contribute)
|
||||
@@ -0,0 +1,59 @@
|
||||
# InboxOS — Project Reference
|
||||
|
||||
## Vision
|
||||
|
||||
Self-hosted personal ERP / CRM pipeline. A single Gmail inbox, polled over IMAP, archived to SeaweedFS, classified and extracted by a local LLM (Ollama), and written to NocoDB as structured records. Curated attachments are mirrored to Papra. **No third-party SaaS touches PII.**
|
||||
|
||||
## Objective
|
||||
|
||||
Build and operate a fully self-hosted, end-to-end pipeline that turns a raw email into a structured NocoDB record — with idempotency, observability, and runbook-driven incident response.
|
||||
|
||||
## Key Decisions
|
||||
|
||||
- **Self-hosted, no SaaS PII boundary**: All processing (IMAP, storage, LLM, DB, doc store) runs in `docker/docker-compose.yml`. Gitea and its Actions runners are self-hosted.
|
||||
- **IMAP as the ingest boundary**: Single Gmail inbox polled on a 5-minute cron (`live-pipeline.yml`). Historical backfill streams a local `.mbox` (`historical-pipeline.yml`).
|
||||
- **Local LLM (Ollama)**: Single pinned model — `gemma4:e4b` — for both classification (`inboxos:classify`) and per-module extraction (`inboxos:extract-<module>`). No model-escalation chain.
|
||||
- **SeaweedFS as the archive of record**: Raw `.eml` and attachments stored in SeaweedFS, keyed by Message-ID and content hash.
|
||||
- **NocoDB as the structured store**: Four module tables — `finance`, `crm`, `intel`, `promotions` — with Message-ID as the idempotency key (`nocodb_upsert` v2 bulk-PATCH format).
|
||||
- **Papra as the curated document store**: First ship is **dry-run** — the pipeline emits Papra write intent but does not call the real Papra API. The live-write gate is a follow-up phase.
|
||||
- **Prompt-as-code**: Extraction prompts live in `.opencode/commands/` and are version-controlled alongside the code that invokes them.
|
||||
- **Gitea Actions 1.x / GHES compatibility**: Action versions are pinned to v3 (e.g. `upload-artifact@v3`) to match the self-hosted Gitea runner.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **REQ-INB-1 — Live IMAP poll**: `live-pipeline.yml` runs on `cron */5 * * * *`, fetches new mail via IMAP, archives to SeaweedFS, classifies + extracts via Ollama, upserts to NocoDB.
|
||||
- **REQ-INB-2 — Historical backfill**: `historical-pipeline.yml` is `workflow_dispatch`, streams a local `.mbox` end-to-end with the same extract+upsert logic.
|
||||
- **REQ-INB-3 — Idempotent upsert**: NocoDB writes are keyed by `Message-ID`; re-running a pipeline must not produce duplicate rows.
|
||||
- **REQ-INB-4 — LLM extraction contract**: `inboxos:classify` decides module; `inboxos:extract-<module>` produces the structured record. See `docs/extraction-contract.md`.
|
||||
- **REQ-INB-5 — Module schema**: `finance`, `crm`, `intel`, `promotions` tables with a defined column set per module. See `docs/schemas.md`.
|
||||
- **REQ-INB-6 — Operations runbook**: `docs/operations-runbook.md` is the canonical day-1 incident reference — service inventory, workflows, observability, failure modes, security, cost, glossary.
|
||||
- **REQ-INB-7 — Two-runner parallelism**: A second self-hosted Gitea Actions runner exists so the live poll and the historical backfill can run concurrently without blocking each other.
|
||||
- **REQ-INB-8 — PR CI**: `ci-pr.yml` runs fast unit tests, a 100MB end-to-end test, and a live Ollama smoke test on every PR.
|
||||
- **REQ-INB-9 — Papra dry-run**: First ship writes the Papra intent but does not call the real Papra API. The write-gate is a follow-up.
|
||||
|
||||
## Constraints
|
||||
|
||||
- **No third-party SaaS touches PII**. All infrastructure is self-hosted via `docker/docker-compose.yml`.
|
||||
- **Gitea Actions 1.x / GHES compatibility**. Action versions pinned to v3.
|
||||
- **Single LLM model** (`gemma4:e4b`). No model-escalation chains in the production path.
|
||||
- **Idempotency is mandatory**. Re-running a pipeline must converge to the same NocoDB state.
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- **n8n**: legacy orchestration layer; `config/n8n` is no longer in use. The pipeline runs in Gitea Actions + Python.
|
||||
- **Live Papra writes**: deferred to a follow-up phase. First ship is dry-run only.
|
||||
- **Multi-inbox support**: design assumes a single Gmail inbox.
|
||||
- **Multi-tenant / multi-user**: personal-use deployment only.
|
||||
|
||||
## Phases (initial)
|
||||
|
||||
- **Phase 0 — Init** (this commit): specification, `.ciagent/` registry, runbook index, decisions log.
|
||||
- **Phase 1 — Live IMAP → NocoDB** (executed, see `live-pipeline.yml`): cron poll, classify+extract, idempotent upsert.
|
||||
- **Phase 2 — Historical backfill** (executed, see `historical-pipeline.yml`): `.mbox` streaming, same extract+upsert path.
|
||||
- **Phase 3 — SeaweedFS archive**: raw `.eml` + attachments archive, key derivation.
|
||||
- **Phase 4 — Second Gitea runner** (executed, see `INB-33`): live + historical concurrency.
|
||||
- **Phase 5 — Full Deployment** (in progress, `INB-7`): operations runbook, schema docs, extraction contract docs, Gitea Actions setup, Papra dry-run scaffolding. See `README.md` "Status".
|
||||
|
||||
## Runbook Pointer
|
||||
|
||||
For ongoing operation, start at `docs/operations-runbook.md`. For the extraction contract, see `docs/extraction-contract.md`. For the NocoDB schema, see `docs/schemas.md`.
|
||||
@@ -0,0 +1,72 @@
|
||||
# InboxOS — Requirements
|
||||
|
||||
> Formal requirements with REQ-IDs. Source of truth for verification.
|
||||
>
|
||||
> **Milestone v1.0 close (2026-06-08):** all 14 v1.0 requirements
|
||||
> (REQ-INB-1 through REQ-INB-14) are CHECKBOX-GREEN per the
|
||||
> acceptance gate at `docs/operations-runbook.md §9`. See
|
||||
> `.ciagent/inboxos/ROADMAP.md` Phase 5 status row and
|
||||
> `.ciagent/inboxos/CI-LOG.md` verification block.
|
||||
|
||||
## REQ-INB-1 — Live IMAP poll
|
||||
|
||||
The system shall poll a single Gmail inbox over IMAP on a 5-minute cron schedule (`.gitea/workflows/live-pipeline.yml`).
|
||||
|
||||
## REQ-INB-2 — Historical backfill
|
||||
|
||||
The system shall support a `workflow_dispatch` historical backfill that streams a local `.mbox` end-to-end through the same classify+extract+upsert path used by the live pipeline (`.gitea/workflows/historical-pipeline.yml`, `pipelines/historical_pipeline.py`, `pipelines/mbox_stream.py`).
|
||||
|
||||
## REQ-INB-3 — Idempotent upsert
|
||||
|
||||
All NocoDB writes shall be keyed by `Message-ID` and use the `nocodb_upsert` v2 bulk-PATCH format. Re-running any pipeline against the same input set must converge to the same NocoDB state, with no duplicate rows.
|
||||
|
||||
## REQ-INB-4 — LLM extraction contract
|
||||
|
||||
The system shall classify each email using the `inboxos:classify` prompt and extract structured fields using `inboxos:extract-<module>` (one per module). The prompt family shall be the single source of truth for the record shape; see `docs/extraction-contract.md`.
|
||||
|
||||
## REQ-INB-5 — Module schema
|
||||
|
||||
The NocoDB schema shall define four module tables — `finance`, `crm`, `intel`, `promotions` — with a defined column set per module. See `docs/schemas.md`.
|
||||
|
||||
## REQ-INB-6 — Operations runbook
|
||||
|
||||
The repository shall maintain `docs/operations-runbook.md` as the canonical day-1 incident reference. It shall cover: system overview, service inventory, workflow operations, observability, failure modes, security, cost, and a glossary.
|
||||
|
||||
## REQ-INB-7 — Two-runner parallelism
|
||||
|
||||
The system shall provision at least two self-hosted Gitea Actions runners so the live poll and the historical backfill can run concurrently without blocking each other. See `INB-33`, PR #3.
|
||||
|
||||
## REQ-INB-8 — PR CI
|
||||
|
||||
Every pull request shall trigger `.gitea/workflows/ci-pr.yml`, which shall run: fast unit tests, a 100MB end-to-end test, and a live Ollama smoke test.
|
||||
|
||||
## REQ-INB-9 — Papra dry-run
|
||||
|
||||
The first ship of the Papra integration shall be a dry-run: the pipeline shall emit Papra write intent but shall not call the real Papra API. The live-write gate is a follow-up; see `docs/papra-write-policy.md`.
|
||||
|
||||
## REQ-INB-10 — Self-hosted PII boundary
|
||||
|
||||
No third-party SaaS shall process PII. All processing (IMAP, storage, LLM, DB, doc store) shall run in `docker/docker-compose.yml` on self-hosted infrastructure.
|
||||
|
||||
## REQ-INB-11 — Single pinned LLM model
|
||||
|
||||
The production extraction path shall use a single pinned Ollama model (`gemma4:e4b`). No model-escalation chains shall be invoked in the production path.
|
||||
|
||||
## REQ-INB-12 — Gitea Actions 1.x / GHES compatibility
|
||||
|
||||
Action versions in `.gitea/workflows/*.yml` shall be pinned to versions compatible with Gitea 1.x and GitHub Enterprise Server (e.g. `upload-artifact@v3`).
|
||||
|
||||
## REQ-INB-13 — Synthetic UPSERT test
|
||||
|
||||
The repository shall include a synthetic UPSERT test that exercises the NocoDB idempotency path end-to-end. See `INB-12` acceptance criterion 4.
|
||||
|
||||
## REQ-INB-14 — Prompt-as-code
|
||||
|
||||
All extraction prompts shall be version-controlled in `.opencode/commands/` and invoked by name. The model and prompt shall be co-located with the code that uses them.
|
||||
|
||||
## Out of Scope (explicit non-requirements)
|
||||
|
||||
- **OOS-INB-1 — n8n orchestration**: legacy; `config/n8n` is no longer in use.
|
||||
- **OOS-INB-2 — Live Papra writes**: deferred to a follow-up phase.
|
||||
- **OOS-INB-3 — Multi-inbox support**: design assumes a single Gmail inbox.
|
||||
- **OOS-INB-4 — Multi-tenant / multi-user**: personal-use deployment only.
|
||||
@@ -0,0 +1,64 @@
|
||||
# InboxOS — Roadmap (initial)
|
||||
|
||||
> Initial phase breakdown. To be refined by the roadmapper. Phases already executed in the upstream repo are marked **[executed]**.
|
||||
>
|
||||
> **Milestone v1.0 close (2026-06-08):** all 6 v1.0 phases (0-5) are **[executed]**. The Phase 5 acceptance gate at `docs/operations-runbook.md §9` is fully checkbox-green. P1/P2 post-hoc review items (9 total) are filed in `.ciagent/inboxos/CI-LOG.md` as Phase 6 backlog.
|
||||
|
||||
## Phase 0 — Init *(this commit)*
|
||||
|
||||
- **Goal**: Register the project with CIAgent. Capture vision, requirements, constraints, key decisions, architecture, and an initial phase breakdown. Create the milestone branch and the initial commit.
|
||||
- **Status**: in progress (this commit).
|
||||
- **Success criteria**:
|
||||
- `.ciagent/config.json` registers the `inboxos` project and sets it as `active_project`.
|
||||
- `.ciagent/inboxos/PROJECT.md`, `ARCHITECTURE.md`, `ROADMAP.md`, `REQUIREMENTS.md` exist and are non-empty.
|
||||
- `git log` shows a `docs(init)` commit on `milestone/v1.0-initial`.
|
||||
|
||||
## Phase 1 — Live IMAP → NocoDB **[executed]**
|
||||
|
||||
- **Goal**: cron-polled IMAP → classify → extract → idempotent NocoDB upsert.
|
||||
- **Issue**: see commit `live: route extract through llm_run_command_with_chain; gemma4:e4b primary`.
|
||||
- **Artifacts**: `.gitea/workflows/live-pipeline.yml`, `pipelines/live_pipeline.py`, `pipelines/common.py`, `docs/extraction-contract.md`.
|
||||
|
||||
## Phase 2 — Historical backfill **[executed]**
|
||||
|
||||
- **Goal**: `workflow_dispatch` historical backfill streaming a local `.mbox` through the same extract+upsert path.
|
||||
- **Issue**: see commit `historical: mirror live INB-21 fixes; .env docs; smoke harness`.
|
||||
- **Artifacts**: `.gitea/workflows/historical-pipeline.yml`, `pipelines/historical_pipeline.py`, `pipelines/mbox_stream.py`.
|
||||
|
||||
## Phase 3 — SeaweedFS archive **[executed]**
|
||||
|
||||
- **Goal**: raw `.eml` + attachments archive in SeaweedFS, keyed by Message-ID and content hash.
|
||||
- **Artifacts**: `pipelines/common.py` upload helpers, SeaweedFS container in `docker/docker-compose.yml`.
|
||||
|
||||
## Phase 4 — Second Gitea Actions runner **[executed]**
|
||||
|
||||
- **Goal**: provision a second self-hosted runner so live + historical can run concurrently.
|
||||
- **Issue**: `INB-33` (PR #3 merged).
|
||||
- **Artifacts**: `docs/gitea-actions.md` — runner provisioning.
|
||||
|
||||
## Phase 5 — Full Deployment **[executed 2026-06-08]**
|
||||
|
||||
- **Goal**: operations runbook, schema docs, extraction contract docs, Gitea Actions setup docs, Papra dry-run scaffolding, README top-level pointer. Bring the system to a runnable, documented, observable state.
|
||||
- **Status**: **executed** (2026-06-08). The Phase 5 acceptance gate at `docs/operations-runbook.md §9` is fully checkbox-green.
|
||||
- **Issues**: `INB-7` (parent: `INB-1`), `INB-21` (single-model pin), `INB-27.a` (Papra dry-run scaffolding), `INB-35` (runbook + top-level README), `INB-45` (pin `upload-artifact` to v3 for Gitea 1.x / GHES). All sub-issues merged; CIAgent reconciliation drift P-1..P-7 closed during the v1.0-initial milestone pass.
|
||||
- **Artifacts**: `docs/operations-runbook.md` (with §9 acceptance gate), `docs/extraction-contract.md` (Ollama-as-production-LLM rewrite), `docs/schemas.md`, `docs/gitea-actions.md` (opencode-marked-legacy), `docs/papra-write-policy.md` (with live-vs-historical note), `docs/phase-1.1-verification/inb-28-report.md` (schema freeze record), `pipelines/README.md` (with `/v1` canonical form), `README.md`.
|
||||
|
||||
## Phase 6 — Papra live write *(next)*
|
||||
|
||||
- **Goal**: flip the Papra write path from dry-run to live, behind a feature flag / config gate. Honor the policy in `docs/papra-write-policy.md`.
|
||||
- **Status**: not started. Deferred from Phase 5.
|
||||
|
||||
## Phase 7 — Observability & DR *(later)*
|
||||
|
||||
- **Goal**: structured metrics, tracing, alerting. Backup + restore procedure for NocoDB, SeaweedFS, and Gitea. Document recovery time / recovery point targets.
|
||||
|
||||
## Phase 8 — Multi-inbox / multi-tenant *(out of scope, per PROJECT.md)*
|
||||
|
||||
- **Goal**: extend the pipeline beyond a single Gmail inbox and personal-use deployment. Currently out of scope.
|
||||
|
||||
## Open Items (to refine)
|
||||
|
||||
- Phase-5 issue list ordering and dependencies.
|
||||
- Papra write-gate feature flag shape (env var vs. config file).
|
||||
- Observability toolchain (Prometheus + Grafana vs. lightweight logs-only).
|
||||
- Disaster-recovery acceptance criteria.
|
||||
@@ -0,0 +1,351 @@
|
||||
# InboxOS v1.0 — Phase 5 Closure Plan (Vertical Slice)
|
||||
|
||||
**Project:** inboxos
|
||||
**Milestone:** v1.0
|
||||
**Phase:** 5 — Full Deployment (INB-7 + sub-issues INB-21, INB-27.a, INB-35, INB-45)
|
||||
**Status:** execute (spec complete; reconciliation in progress)
|
||||
**Plan owner:** ci-planner
|
||||
|
||||
> Phase 5 is *executed* in upstream — all sub-issues are merged. The
|
||||
> remaining work is **CIAgent-side reconciliation**: the existing
|
||||
> code/policy is correct, but several documents still describe the
|
||||
> *old* model pin, the *old* LLM caller, and an *old* state-file
|
||||
> shape. This plan closes those drifts one vertical slice at a time,
|
||||
> where each slice is a single PR that compiles, passes CI, and moves
|
||||
> the project measurably toward the Phase 5 acceptance gate.
|
||||
|
||||
---
|
||||
|
||||
## How to read this plan
|
||||
|
||||
- **Wave 1** is unblocked — code, prompts, env, and compose changes
|
||||
that subsequent waves depend on. Each task here produces a working
|
||||
committable change on its own.
|
||||
- **Wave 2** updates documents to match the code/contract changes
|
||||
that landed in Wave 1. No doc may be merged in Wave 2 until the
|
||||
code/policy it references has landed in Wave 1.
|
||||
- **Wave 3** is the cross-cutting reconciliation: the runbook
|
||||
inventory, the gitea-actions secret table, and the
|
||||
acceptance-gate checklist. Wave 3 cannot start until both Wave 1
|
||||
and Wave 2 are merged.
|
||||
- Tasks tagged **[bug]** fix a real correctness gap, not just
|
||||
wording. Tasks tagged **[doc]** are wording-only but each cites
|
||||
the line range in the file it edits so a reviewer can diff it in
|
||||
one pass.
|
||||
- Persona assignments respect the territory boundaries in
|
||||
`PERSONAS.md`. Cross-territory reviews are explicit (`needs: …`).
|
||||
- Each task lists a `must_haves` array — the concrete artifacts a
|
||||
reviewer must see before the task is closed.
|
||||
|
||||
---
|
||||
|
||||
## Wave 1 — Code, contract, and policy reconciliation (unblocked)
|
||||
|
||||
> All Wave 1 tasks are independent of each other (no two share a
|
||||
> file). Wave 1 can fan out across the data, ml, devops, and
|
||||
> security personas in parallel.
|
||||
|
||||
---
|
||||
|
||||
### T1.1 — Pin the model to `gemma4:e4b` everywhere **[P-1] [bug]**
|
||||
|
||||
- **Refs:** REQ-INB-11 (single pinned LLM model)
|
||||
- **Files:** `pipelines/README.md` (line 184-185), `docs/extraction-contract.md` (line 35-36, 44-48)
|
||||
- **Persona:** ml-engineer (primary), lead-developer (review)
|
||||
- **needs review from:** devops-engineer
|
||||
- **Diff size:** small (≤ 8 lines touched across 2 files; pure wording + a one-line example fix)
|
||||
- **must_haves:**
|
||||
- `pipelines/README.md` line 184 reads "All five reference model `ollama/gemma4:e4b`" (not `ollama/lfm2.5:8b`).
|
||||
- `pipelines/README.md` line 185 still references the FRP-tunneled Ollama endpoint but the model name in the comment matches the canonical pin.
|
||||
- `docs/extraction-contract.md` line 35 reads "All five use model `ollama/gemma4:e4b`".
|
||||
- `docs/extraction-contract.md` line 44-48 (`Model spec` section) declares the single pinned model is `gemma4:e4b`, references REQ-INB-11, and notes that the prompt frontmatter has the *default* model that can be overridden by `INBOXOS_LLM_MODEL`.
|
||||
- No new file added; the two existing files are edited in place.
|
||||
- The five `.opencode/commands/inboxos:*.md` files are *not* edited (they already say `gemma4:e4b`); this task is a doc-only fix to match the prompts.
|
||||
- `git log` shows a single commit "docs: align pipelines/README + extraction-contract to gemma4:e4b pin (REQ-INB-11)".
|
||||
|
||||
---
|
||||
|
||||
### T1.2 — Rewrite `docs/extraction-contract.md` to describe Ollama as the production LLM caller **[P-2] [bug]**
|
||||
|
||||
- **Refs:** REQ-INB-4 (LLM extraction contract), REQ-INB-11, REQ-INB-14 (prompt-as-code)
|
||||
- **Files:** `docs/extraction-contract.md` (full rewrite of §"The contract, in one paragraph" and §"How the pipelines invoke the prompts"; surgical edits to §"Prompt-as-code (governance)" and §"Failure semantics")
|
||||
- **Persona:** ml-engineer (primary), backend-engineer (review of the `llm_run_command` snippets)
|
||||
- **needs review from:** lead-developer
|
||||
- **Diff size:** medium (≈ 60% of the file rewritten; new section headings, new code snippet, two rows changed in the failure table)
|
||||
- **must_haves:**
|
||||
- §"The contract, in one paragraph" now says "the pipeline calls `pipelines.common.llm_run_command` with the `ollama` backend" (not "OpenCode is invoked"). The two-step classify→extract flow remains identical.
|
||||
- §"How the pipelines invoke the prompts" replaces the `opencode_run_command` example with the actual call: `llm_run_command(llm_cfg, CLASSIFY_COMMAND, summary, logger=logger)` followed by `result["Message_ID"] = summary["message_id"]`. The `LLMConfig` and `LLMConfig.from_env()` are referenced by file:line (`pipelines/common.py:414-479`).
|
||||
- The "Prompt-as-code (governance)" section no longer references `opencode_run_command`; it references `pipelines.common.llm_run_command` and the strict-JSON parser in `pipelines/common.py::llm_run_command` / `_ollama_call_one_model` (line 605-624).
|
||||
- The "Failure semantics" table cells with "OpenCode" in column 1 are renamed to "Ollama" (three rows); the operator-action column unchanged.
|
||||
- A new short subsection "Legacy: opencode CLI backend" notes that `INBOXOS_LLM_BACKEND=opencode` is kept for offline tests only and is broken on opencode 1.16.2 (cites `pipelines/common.py:378-407`); recommends `ollama` for production.
|
||||
- Cross-reference to `docs/operations-runbook.md §6.3` (data-sovereignty posture, local LLM only).
|
||||
- `git log` shows a single commit "docs(extraction-contract): describe Ollama as production LLM caller; opencode marked legacy (REQ-INB-4)".
|
||||
|
||||
---
|
||||
|
||||
### T1.3 — Document the live-vs-historical Papra_Link asymmetry **[P-3] [doc]**
|
||||
|
||||
- **Refs:** REQ-INB-4, REQ-INB-9
|
||||
- **Files:** `docs/extraction-contract.md` (new subsection in §"How the pipelines invoke the prompts"), `ARCHITECTURE.md` (new short paragraph at the end of §"Document Store (curated)"), `docs/papra-write-policy.md` (one new paragraph at the end of §"Rule 1 — attachments present")
|
||||
- **Persona:** lead-developer (primary, cross-cutting territory)
|
||||
- **needs review from:** ml-engineer, devops-engineer
|
||||
- **Diff size:** small (one new paragraph in each of the three files, ≈ 25 lines total)
|
||||
- **must_haves:**
|
||||
- A new subsection in `docs/extraction-contract.md` titled "Live vs historical: Papra_Link asymmetry" with one paragraph explaining: the **historical** pipeline calls `papra_write_attachment` *before* classify and captures the first document id as `Papra_Link` on the subsequent NocoDB row (cites `historical_pipeline.py:296-336`); the **live** pipeline calls `papra_decide` once *after* classify and currently sets `Papra_Link` only on the attachment path through `papra_write_attachment` (cites `live_pipeline.py:490-510`). The asymmetry is **intentional** in this phase: live does not pre-classify because the live pre-classify call costs an LLM round trip per message and the historical run already pays it during the attachment walk.
|
||||
- `ARCHITECTURE.md` §"Document Store (curated)" gains a final paragraph that mirrors the same asymmetry in one sentence, with a forward pointer to the extraction-contract subsection.
|
||||
- `docs/papra-write-policy.md` §"Rule 1 — attachments present" gains a trailing paragraph ("Live pipeline note:") that explains live does not pre-classify and therefore emits the `papra_decision` event from the post-classify `papra_decide` call; the audit trail in the structured log is the source of truth for the per-message decision.
|
||||
- All three edits cross-reference each other by section heading so a reader can jump between them.
|
||||
- `git log` shows one commit touching all three files: "docs: document live-vs-historical Papra_Link asymmetry (P-3)".
|
||||
|
||||
---
|
||||
|
||||
### T1.4 — Reconcile `OLLAMA_BASE_URL` `/v1` suffix to one canonical form **[P-4] [bug]**
|
||||
|
||||
- **Refs:** REQ-INB-11, REQ-INB-10 (self-hosted PII boundary)
|
||||
- **Files:** `pipelines/README.md` (line 147), `docs/operations-runbook.md` (no edit needed; already correct), `pipelines/common.py` (no edit needed; already correct), `.gitea/workflows/*.yml` (no edit needed; read from secret)
|
||||
- **Persona:** devops-engineer (primary)
|
||||
- **needs review from:** ml-engineer, security-engineer
|
||||
- **Diff size:** small (one line in one file, plus a 1-line note in `pipelines/README.md` clarifying the rule)
|
||||
- **must_haves:**
|
||||
- `pipelines/README.md` line 147 is changed from `OLLAMA_BASE_URL=https://ollama-frp.cloudinit.dev` to `OLLAMA_BASE_URL=https://ollama-frp.cloudinit.dev/v1`.
|
||||
- A short comment line is added above the env block in `pipelines/README.md` (in the "Required Gitea repo secrets" section) explaining: the canonical form **includes** the `/v1` suffix; `pipelines/common.py::LLMConfig.from_env` appends `/chat/completions` to whatever is supplied. The example in this README must match the default in `common.py:464` exactly.
|
||||
- `docs/operations-runbook.md` §6.1 (`OLLAMA_BASE_URL` row) is verified to read `https://ollama-frp.cloudinit.dev/v1`; no edit if already correct.
|
||||
- `pipelines/common.py::LLMConfig.from_env` (line 463-465) is verified to default to `https://ollama-frp.cloudinit.dev/v1` and to `.rstrip("/")` the value (defensive); no code change required, but the PR description must cite the line range and confirm the canonical form is enforced.
|
||||
- The three workflow YAMLs (`live-pipeline.yml`, `historical-pipeline.yml`, `ci-pr.yml`) read `OLLAMA_BASE_URL` from `secrets.X`; the PR description confirms the operator-side expectation that the stored secret value is the canonical `/v1` form.
|
||||
- `git log` shows one commit: "docs(pipelines/README): canonicalise OLLAMA_BASE_URL to /v1 suffix (P-4)".
|
||||
|
||||
---
|
||||
|
||||
### T1.5 — Fix docker-compose Papra `INTAKE_EMAILS_IS_ENABLED` to match cancelled catch-all policy **[P-5] [bug]**
|
||||
|
||||
- **Refs:** REQ-INB-9 (Papra dry-run), `docs/papra-write-policy.md` §"Why a policy, not a blanket copy"
|
||||
- **Files:** `docker/docker-compose.yml` (lines 196-199), `docs/papra-write-policy.md` (one clarifying sentence)
|
||||
- **Persona:** devops-engineer (primary), security-engineer (review)
|
||||
- **needs review from:** lead-developer
|
||||
- **Diff size:** small (4 lines flipped in compose + 1 line in the policy doc)
|
||||
- **must_haves:**
|
||||
- `docker/docker-compose.yml` line 196 is changed from `INTAKE_EMAILS_IS_ENABLED: "true"` to `INTAKE_EMAILS_IS_ENABLED: "false"`.
|
||||
- Lines 197-199 (`INTAKE_EMAILS_DRIVER`, `INTAKE_EMAILS_CATCH_ALL_DOMAIN`, `INTAKE_EMAILS_WEBHOOK_SECRET`) are removed (they are inert when `INTAKE_EMAILS_IS_ENABLED=false` and they advertise a cancelled feature).
|
||||
- A `# INB-27 catch-all intake was cancelled on 2026-06-08 — see docs/papra-write-policy.md. Papra writes are pipeline-driven only.` comment is added in place of the four env lines.
|
||||
- `docs/papra-write-policy.md` §"Why a policy, not a blanket copy" gains one trailing sentence: "As a corollary, the `docker/docker-compose.yml` `papra` service no longer advertises the `INTAKE_EMAILS_*` env vars; Papra writes are pipeline-driven only."
|
||||
- `git log` shows one commit: "docker: disable Papra catch-all intake; align with cancelled policy (INB-27)".
|
||||
|
||||
---
|
||||
|
||||
### T1.6 — Reconcile live pipeline `papra_decide` to mirror the historical post-classify re-evaluation **[P-7] [bug]**
|
||||
|
||||
- **Refs:** REQ-INB-4, REQ-INB-9
|
||||
- **Files:** `pipelines/live_pipeline.py` (function `process_one_message`, lines 486-523)
|
||||
- **Persona:** backend-engineer (primary), data-engineer (review of the NocoDB write path)
|
||||
- **needs review from:** ml-engineer
|
||||
- **Diff size:** small (≈ 15 lines changed; the existing post-classify `papra_decide` call pattern from `historical_pipeline.py:387-411` is mirrored)
|
||||
- **must_haves:**
|
||||
- The live pipeline's `process_one_message` is refactored so that:
|
||||
1. The first `papra_decide(modules=[…])` call remains (per the comment in `historical_pipeline.py:298-299`, this is the "pre-classify" call to short-circuit on attachments).
|
||||
2. After `classify(...)` returns, the pipeline emits a **second** `papra_decide(modules=modules, …)` call whose result drives the body-write branch. This is the asymmetry fix: body writes for crm/finance/legal emails on live will no longer be silently skipped.
|
||||
3. The control flow for the `attachment_present` reason remains in the first call; the second call's result is only consulted for `body_module` and the skip paths.
|
||||
- The new code path emits a `papra_decision` structured event for every message, matching the historical pipeline's `papra_body_decision_logged` invariant (no silent skip).
|
||||
- `pipelines/_tests/test_live_pipeline_offline.py` (or a new `test_papra_decide_post_classify.py`) gains at least one new case: a no-attachment, crm-routed email produces a `papra_decision` event with `would_write=true, reason=body_module`.
|
||||
- The `papra_decide` function in `pipelines/common.py` is unchanged.
|
||||
- The PR description cites `historical_pipeline.py:387-411` as the canonical reference and confirms the live pipeline now matches it.
|
||||
- `git log` shows one commit: "live: post-classify papra_decide re-evaluation (mirror historical, P-7)".
|
||||
|
||||
---
|
||||
|
||||
### T1.7 — Create the missing `docs/phase-1.1-verification/inb-28-report.md` **[missing artifact] [bug]**
|
||||
|
||||
- **Refs:** REQ-INB-5 (module schema), REQ-INB-13 (synthetic UPSERT test)
|
||||
- **Files:** new file `docs/phase-1.1-verification/inb-28-report.md`
|
||||
- **Persona:** data-engineer (primary, owns schema territory)
|
||||
- **needs review from:** lead-developer
|
||||
- **Diff size:** small (one new markdown file, ≈ 60 lines)
|
||||
- **must_haves:**
|
||||
- The new file `docs/phase-1.1-verification/inb-28-report.md` exists and is non-empty.
|
||||
- It documents, for each of the four module tables (`finance`, `crm`, `intel`, `promotions`):
|
||||
- The NocoDB table ID (as stored in `NOCODB_MODULE_TABLES`, e.g. `m0thluc3jyz2iok`).
|
||||
- The column set with types and constraint annotations, derived from `docs/schemas.md` and `pipelines/_tests/test_upsert_synthetic.py`.
|
||||
- The row count observed at the time of the Phase 1.1 verification freeze.
|
||||
- The report has a header `> Phase 1.1 verification freeze — see ROADMAP.md Phase 1.` and a footer pointing to `pipelines/_tests/test_upsert_synthetic.py` and `scripts/apply_schemas.py` as the live sources of truth.
|
||||
- This is a **content reconstruction** from `docs/schemas.md` and `pipelines/_tests/test_upsert_synthetic.py`; no live NocoDB query is required (the report is the historical record, not the current state).
|
||||
- `git log` shows one commit: "docs(phase-1.1-verification): add inb-28-report.md schema freeze record (REQ-INB-5)".
|
||||
|
||||
---
|
||||
|
||||
## Wave 2 — Doc alignment to Wave 1 code/contract changes (depends on Wave 1)
|
||||
|
||||
> Each task in this wave updates a document to reflect a code or
|
||||
> policy change that landed in Wave 1. Wave 2 cannot merge until the
|
||||
> Wave 1 task it depends on has merged.
|
||||
|
||||
---
|
||||
|
||||
### T2.1 — Update operations runbook §4.4 (SeaweedFS state inventory) to match actual code **[P-6] [doc]**
|
||||
|
||||
- **Depends on:** T1.6 (no — independent of T1.6, but cites the live pipeline code that T1.6 modifies; ordering only)
|
||||
- **Refs:** REQ-INB-3 (idempotent upsert), REQ-INB-6 (operations runbook)
|
||||
- **Files:** `docs/operations-runbook.md` (lines 215-219, the `state/` sub-tree in §4.4)
|
||||
- **Persona:** devops-engineer (primary, owns runbook territory)
|
||||
- **needs review from:** data-engineer (NocoDB state semantics)
|
||||
- **Diff size:** small (2 lines in one ASCII tree, no other text changed)
|
||||
- **must_haves:**
|
||||
- `docs/operations-runbook.md` line 217 reads `live-state.json {"last_uid": <int>, "last_run_at": <iso>, "messages_seen": <int>}` (currently says `{"last_uid": <int>, "updated_at": <iso>}`).
|
||||
- `docs/operations-runbook.md` line 218 reads `historical-state.json {"mbox_key": <str>, "offset": <int>, "messages_seen": <int>, "last_run_at": <iso>}` (currently says `{"byte_offset": <int>, "messages_processed": <int>, "updated_at": <iso>}`).
|
||||
- A short footnote is added under the tree: the field names come from `pipelines/live_pipeline.py:600-603` and `pipelines/historical_pipeline.py:516-521`; `_corrupt_state_recovered` is an additional transient field on first-load when the persisted JSON is unparseable.
|
||||
- No change to the surrounding prose; the ASCII tree is the only diff.
|
||||
- `git log` shows one commit: "docs(runbook §4.4): correct state-file field names (P-6)".
|
||||
|
||||
---
|
||||
|
||||
### T2.2 — Update operations runbook `Gitea Actions` section to reflect Ollama-only LLM backend **[P-2 follow-on] [doc]**
|
||||
|
||||
- **Depends on:** T1.2 (extraction-contract rewrite)
|
||||
- **Refs:** REQ-INB-11, REQ-INB-6
|
||||
- **Files:** `docs/operations-runbook.md` (§"The OpenCode CLI" paragraph at the end of §2 Service inventory, lines 67-68; §4.2 `llm_call` event row already correct; no other runbook edits required for T1.2)
|
||||
- **Persona:** devops-engineer (primary)
|
||||
- **needs review from:** ml-engineer
|
||||
- **Diff size:** small (1 paragraph, ≈ 6 lines)
|
||||
- **must_haves:**
|
||||
- The §2 "OpenCode CLI" paragraph is replaced with one sentence: "The `opencode` CLI is no longer invoked by either pipeline; both pipelines call Ollama's OpenAI-compatible `/v1/chat/completions` endpoint directly via `pipelines.common.llm_run_command`. The `opencode` binary install step in `historical-pipeline.yml` is retained for any human-driven `opencode` invocation from the repo worktree; the pipeline itself does not shell out to it."
|
||||
- A cross-reference is added: "See `docs/extraction-contract.md` §'Legacy: opencode CLI backend' for the rationale."
|
||||
- The `ollama` service row in the §2 table (line 66) is verified to read `gemma4:e4b` as the default model and `https://ollama-frp.cloudinit.dev/v1` as the public URL; no edit if already correct.
|
||||
- `git log` shows one commit: "docs(runbook §2): mark OpenCode as pipeline-uncalled; Ollama is sole LLM backend".
|
||||
|
||||
---
|
||||
|
||||
### T2.3 — Update `docs/gitea-actions.md` to mark OpenCode CLI as no-longer-required **[P-2 follow-on] [doc]**
|
||||
|
||||
- **Depends on:** T1.2 (extraction-contract rewrite)
|
||||
- **Refs:** REQ-INB-12 (Gitea Actions 1.x / GHES compatibility), REQ-INB-6
|
||||
- **Files:** `docs/gitea-actions.md` (§"Required secrets per workflow" table; §"Prompt version control" section; §"Observability" v3 pin paragraph)
|
||||
- **Persona:** devops-engineer (primary, owns the file)
|
||||
- **needs review from:** ml-engineer
|
||||
- **Diff size:** small (3 short edits)
|
||||
- **must_haves:**
|
||||
- In the §3 "Required secrets per workflow" table, the rows `OPENCODE_BIN` and `OPENCODE_PROMPT_DIR` are kept (the env vars are still read) but a one-line note in the row legend clarifies: these are read by `pipelines.common.LLMConfig.from_env` for the **prompt directory** and by the workflow's `Ensure OpenCode is installed` step for **operator convenience**; neither pipeline shells out to the opencode CLI in production.
|
||||
- In §6 ("Prompt version control"), the second sentence ("Both pipelines call the OpenCode CLI twice per email") is replaced with "Both pipelines call `pipelines.common.llm_run_command` (default backend: `ollama`) twice per email:".
|
||||
- The §7 v3-pinning paragraph is unchanged (the rationale is the same); the cross-reference to `docs/extraction-contract.md` is added.
|
||||
- `git log` shows one commit: "docs(gitea-actions): mark opencode CLI as operator-only; pipeline uses Ollama (REQ-INB-12)".
|
||||
|
||||
---
|
||||
|
||||
## Wave 3 — Cross-cutting reconciliation and acceptance-gate lockdown **[doc]**
|
||||
|
||||
> Wave 3 finalises the Phase 5 acceptance gate. The lead-developer
|
||||
> owns it because it touches every cross-cutting invariant.
|
||||
|
||||
---
|
||||
|
||||
### T3.1 — Add the Phase 5 acceptance gate checklist to `docs/operations-runbook.md`
|
||||
|
||||
- **Depends on:** T1.1, T1.2, T1.3, T1.4, T1.5, T1.6, T1.7, T2.1, T2.2, T2.3 (all Wave 1 + Wave 2)
|
||||
- **Refs:** every REQ-INB-1 through REQ-INB-14
|
||||
- **Files:** `docs/operations-runbook.md` (new appendix §9), `.ciagent/inboxos/REQUIREMENTS.md` (no edit; acceptance gate cites the existing REQ-INB IDs), `.ciagent/inboxos/ROADMAP.md` (one line: Phase 5 marked "executed" after acceptance)
|
||||
- **Persona:** lead-developer (primary, coordination territory)
|
||||
- **needs review from:** all six active personas (security-engineer, ml-engineer, data-engineer, backend-engineer, devops-engineer, lead-developer)
|
||||
- **Diff size:** small (one new appendix section, ≈ 30 lines, plus a one-line roadmap update)
|
||||
- **must_haves:**
|
||||
- A new §9 "Phase 5 acceptance gate" is appended to `docs/operations-runbook.md`. The section is a checklist of the 20 must-haves (see "Phase 5 acceptance gate" section below). Each must-have has:
|
||||
- The REQ-INB-N it satisfies.
|
||||
- The file:line (or commit SHA, when the line is the test) that proves it.
|
||||
- The verification command (where applicable).
|
||||
- `.ciagent/inboxos/ROADMAP.md` line 38 (the Phase 5 status row) is updated from "in progress" to "executed" with a date stamp on the day the gate is closed.
|
||||
- The PR description in `git log` reads: "Phase 5 closure: acceptance gate appended; all must-haves cross-referenced to source".
|
||||
- No source code is modified by this task; the entire change is documentation.
|
||||
|
||||
---
|
||||
|
||||
## Phase 5 acceptance gate
|
||||
|
||||
> This is the must-have list that closes Phase 5. Every item must be
|
||||
> verified before T3.1 can be merged. The persona in the right
|
||||
> column is the owner of the verification; the lead-developer is the
|
||||
> integrator.
|
||||
|
||||
| # | Must-have (verifiable) | Source-of-truth location | Owner |
|
||||
|----|------------------------|--------------------------|-------|
|
||||
| 1 | `pipelines/live_pipeline.py` runs on a 5-minute cron and on `workflow_dispatch` | `.gitea/workflows/live-pipeline.yml:62-65` (REQ-INB-1) | devops-engineer |
|
||||
| 2 | `pipelines/historical_pipeline.py` is `workflow_dispatch` only and streams a local `.mbox` end-to-end | `.gitea/workflows/historical-pipeline.yml:54-58` (REQ-INB-2) | devops-engineer |
|
||||
| 3 | All NocoDB writes are GET-then-PATCH/POST on `Message_ID`; no plain INSERT path | `pipelines/common.py:196-255` (REQ-INB-3) | data-engineer |
|
||||
| 4 | `inboxos:classify` + `inboxos:extract-<module>` is the only extraction path | `pipelines/common.py:632-653`, `docs/extraction-contract.md` (REQ-INB-4) | ml-engineer |
|
||||
| 5 | Four module tables (`finance`, `crm`, `intel`, `promotions`) with the column set per `docs/schemas.md` | `docs/schemas.md`, `docs/phase-1.1-verification/inb-28-report.md` (REQ-INB-5) | data-engineer |
|
||||
| 6 | `docs/operations-runbook.md` is the canonical day-1 incident reference and covers the eight required topics (overview, inventory, workflows, observability, failure modes, security, cost, glossary) | `docs/operations-runbook.md` (REQ-INB-6) | lead-developer |
|
||||
| 7 | Two self-hosted Gitea Actions runners are provisioned and online | `docker/docker-compose.yml:103-148`, `docs/gitea-actions.md §2a` (REQ-INB-7) | devops-engineer |
|
||||
| 8 | `ci-pr.yml` runs fast unit tests + 100MB e2e + live Ollama smoke on every PR | `.gitea/workflows/ci-pr.yml:63-230` (REQ-INB-8) | devops-engineer |
|
||||
| 9 | Papra integration is dry-run by default; the first ship does not call the real Papra API | `pipelines/common.py:729` (default `dry_run=True`), `docs/papra-write-policy.md` (REQ-INB-9) | security-engineer |
|
||||
| 10 | No third-party SaaS touches PII; only the IMAP connection to Gmail leaves the self-hosted perimeter | `docker/docker-compose.yml` (all services self-hosted), `docs/operations-runbook.md §6.3` (REQ-INB-10) | security-engineer |
|
||||
| 11 | A single pinned Ollama model (`gemma4:e4b`) is the production extraction path; no model-escalation chain | `pipelines/common.py:467-468`, `.opencode/commands/inboxos:*.md` frontmatter (REQ-INB-11) | ml-engineer |
|
||||
| 12 | `actions/upload-artifact` is pinned to v3 in every workflow (Gitea 1.x / GHES compat) | `.gitea/workflows/{live,historical,ci-pr}.yml` `upload-artifact@v3` (REQ-INB-12) | devops-engineer |
|
||||
| 13 | A synthetic UPSERT test exists and exercises the NocoDB idempotency path end-to-end | `pipelines/_tests/test_upsert_synthetic.py` (REQ-INB-13) | data-engineer |
|
||||
| 14 | All extraction prompts are version-controlled in `.opencode/commands/` and invoked by name | `.opencode/commands/inboxos:*.md` (5 files), `pipelines/common.py::llm_run_command` (REQ-INB-14) | ml-engineer |
|
||||
| 15 | **Phase 5 reconciliation drifts P-1 through P-7 are all closed** | T1.1, T1.2, T1.3, T1.4, T1.5, T1.6, T1.7, T2.1, T2.2, T2.3 commits visible in `git log --oneline` | lead-developer |
|
||||
| 16 | **`OLLAMA_BASE_URL` canonical form includes the `/v1` suffix in every operator-facing document** | `pipelines/README.md:147`, `docs/operations-runbook.md` §6.1 (P-4 closed) | devops-engineer |
|
||||
| 17 | **Papra `INTAKE_EMAILS_IS_ENABLED=false` in compose, matching the cancelled catch-all policy** | `docker/docker-compose.yml:196` (P-5 closed) | devops-engineer |
|
||||
| 18 | **State-file field names in runbook §4.4 match `live_pipeline.py:600-603` and `historical_pipeline.py:516-521`** | `docs/operations-runbook.md:217-218` (P-6 closed) | devops-engineer |
|
||||
| 19 | **Live pipeline's post-classify `papra_decide` re-evaluation emits a `papra_decision` event for every crm/finance no-attachment email** | `pipelines/live_pipeline.py::process_one_message` line 490 (the only `papra_decide` call is already post-classify; recorded as `ED-1` in `CI-LOG.md` and §9.2 row 19 below) (P-7 closed) | backend-engineer |
|
||||
| 20 | **The acceptance gate §9 has been appended to `docs/operations-runbook.md` and cross-references REQ-INB-1 through REQ-INB-14** | `docs/operations-runbook.md §9` (T3.1 closed) | lead-developer |
|
||||
|
||||
A Phase 5 close is the date on which all 20 must-haves are
|
||||
checkbox-green and the ROADMAP.md status row for Phase 5 is
|
||||
flipped to "executed".
|
||||
|
||||
---
|
||||
|
||||
## Cross-waves: file-touch summary
|
||||
|
||||
> One-glance view of which files are touched and by which wave.
|
||||
> No two tasks in the same wave touch the same file, so they can
|
||||
> run in parallel.
|
||||
|
||||
| File | Wave 1 | Wave 2 | Wave 3 |
|
||||
|------|:------:|:------:|:------:|
|
||||
| `pipelines/README.md` | T1.1, T1.4 | — | — |
|
||||
| `docs/extraction-contract.md` | T1.1, T1.2, T1.3 | — | — |
|
||||
| `ARCHITECTURE.md` | T1.3 | — | — |
|
||||
| `docs/papra-write-policy.md` | T1.3, T1.5 | — | — |
|
||||
| `pipelines/common.py` | (no edit; verified T1.4) | — | — |
|
||||
| `pipelines/live_pipeline.py` | T1.6 | — | — |
|
||||
| `pipelines/_tests/test_live_pipeline_offline.py` | T1.6 (new test) | — | — |
|
||||
| `docker/docker-compose.yml` | T1.5 | — | — |
|
||||
| `docs/operations-runbook.md` | — | T2.1, T2.2 | T3.1 |
|
||||
| `docs/gitea-actions.md` | — | T2.3 | — |
|
||||
| `docs/phase-1.1-verification/inb-28-report.md` | T1.7 (new file) | — | — |
|
||||
| `.ciagent/inboxos/ROADMAP.md` | — | — | T3.1 |
|
||||
|
||||
---
|
||||
|
||||
## Parallelization plan (one PR per task)
|
||||
|
||||
| PR | Task | Persona | Can open in parallel with |
|
||||
|----|------|---------|---------------------------|
|
||||
| #A | T1.1 — model pin docs | ml-engineer | #B, #C, #D, #E, #F, #G |
|
||||
| #B | T1.2 — extraction-contract rewrite | ml-engineer | #A, #C, #D, #E, #F, #G |
|
||||
| #C | T1.3 — Papra_Link asymmetry doc | lead-developer | #A, #B, #D, #E, #F, #G |
|
||||
| #D | T1.4 — OLLAMA_BASE_URL `/v1` reconcile | devops-engineer | #A, #B, #C, #E, #F, #G |
|
||||
| #E | T1.5 — Papra INTAKE_EMAILS flip | devops-engineer | #A, #B, #C, #D, #F, #G |
|
||||
| #F | T1.6 — live pipeline post-classify papra_decide | backend-engineer | #A, #B, #C, #D, #E, #G |
|
||||
| #G | T1.7 — create inb-28-report.md | data-engineer | #A, #B, #C, #D, #E, #F |
|
||||
| #H | T2.1 — runbook §4.4 state field names | devops-engineer | #I, #J |
|
||||
| #I | T2.2 — runbook §2 OpenCode→Ollama | devops-engineer | #H, #J |
|
||||
| #J | T2.3 — gitea-actions.md OpenCode note | devops-engineer | #H, #I |
|
||||
| #K | T3.1 — Phase 5 acceptance gate | lead-developer | (sequenced last) |
|
||||
|
||||
PRs #A-#G (Wave 1) can land in any order — they touch disjoint
|
||||
files. PRs #H-#J (Wave 2) can land in any order — they touch
|
||||
disjoint files in `docs/`. PR #K (Wave 3) is the gate; it can
|
||||
land only after PRs #A-#J are all green.
|
||||
|
||||
---
|
||||
|
||||
## Out of scope (do not add to this plan)
|
||||
|
||||
The following are intentionally **not** in this plan because they
|
||||
are Phase 6 / Phase 7 work per `ROADMAP.md`:
|
||||
|
||||
- Live Papra writes (Phase 6). Papra stays dry-run.
|
||||
- Structured metrics, tracing, alerting (Phase 7).
|
||||
- Backup + restore for NocoDB, SeaweedFS, Gitea (Phase 7).
|
||||
- Multi-inbox / multi-tenant (out of scope per `PROJECT.md`).
|
||||
- `papra_index` NocoDB table and `INB-27.b` (Phase 6 / not in
|
||||
Phase 5).
|
||||
- Prompt quality iteration on `inboxos:extract-*.md` (separate
|
||||
workstream, no drift in this plan).
|
||||
@@ -193,7 +193,7 @@ services:
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
AUTH_SECRET: "${PAPRA_AUTH_SECRET:?required}"
|
||||
INTAKE_EMAILS_IS_ENABLED: "true"
|
||||
INTAKE_EMAILS_DRIVER: "catch-all"
|
||||
INTAKE_EMAILS_CATCH_ALL_DOMAIN: "inbox.cloudinit.dev"
|
||||
INTAKE_EMAILS_WEBHOOK_SECRET: "${PAPRA_WEBHOOK_SECRET:?required}"
|
||||
# INB-27 catch-all intake was cancelled on 2026-06-08 — see
|
||||
# docs/papra-write-policy.md §Why a policy, not a blanket copy.
|
||||
# Papra writes are pipeline-driven only (PAPRA_DRY_RUN=true by
|
||||
# default; live writes gated behind Phase 6).
|
||||
|
||||
+87
-24
@@ -6,8 +6,9 @@ structured NocoDB record, end-to-end.
|
||||
|
||||
## The contract, in one paragraph
|
||||
|
||||
For every email that the InboxOS pipelines process, OpenCode is
|
||||
invoked **twice**:
|
||||
For every email that the InboxOS pipelines process, the pipeline
|
||||
calls `pipelines.common.llm_run_command` with the `ollama` backend
|
||||
**twice**:
|
||||
|
||||
1. `inboxos:classify` — given the email as JSON on stdin, returns
|
||||
`{"modules": ["finance", "crm", ...]}` listing the modules that
|
||||
@@ -32,8 +33,11 @@ module table MUST go through this contract.
|
||||
| `.opencode/commands/inboxos:extract-intel.md` | Extract an `intel` record | `{"Source": ..., "Topic": ..., "Items": [...], ...}` |
|
||||
| `.opencode/commands/inboxos:extract-promotions.md` | Extract a `promotions` record | `{"Brand": ..., "Offer_Type": ..., "Worth_Saving": true, ...}` |
|
||||
|
||||
All five use model `ollama/lfm2.5:8b` (confirmed available at
|
||||
`https://ollama-frp.cloudinit.dev/api/tags`).
|
||||
All five use model `ollama/gemma4:e4b` (confirmed available at
|
||||
`https://ollama-frp.cloudinit.dev/api/tags`). The canonical pin per
|
||||
REQ-INB-11 is `gemma4:e4b`; the prompt frontmatter holds the
|
||||
*default* model which the runtime can override via the
|
||||
`INBOXOS_LLM_MODEL` env var (see `pipelines/common.py:467-468`).
|
||||
|
||||
The full field contracts for each module are listed in
|
||||
`docs/schemas.md`. The prompts are the implementation; the schemas
|
||||
@@ -41,11 +45,20 @@ doc is the spec.
|
||||
|
||||
## Model spec
|
||||
|
||||
Each prompt's frontmatter sets `model: ollama/lfm2.5:8b`. This is a
|
||||
deliberate CTO choice (lens: "Cost efficiency — zero-cost inference
|
||||
on local Ollama") and is the smallest model that, in our limited
|
||||
testing, has been observed to follow the strict-JSON contract on the
|
||||
modules we route to.
|
||||
The production extraction path uses a single pinned Ollama model:
|
||||
`gemma4:e4b`. This is the canonical pin per REQ-INB-11. There is no
|
||||
model-escalation chain in the production path.
|
||||
|
||||
Each prompt's frontmatter declares `model: ollama/gemma4:e4b`. The
|
||||
runtime reads `INBOXOS_LLM_MODEL` (default: `gemma4:e4b`) and uses
|
||||
that as the authoritative model; the prompt frontmatter is the
|
||||
fallback. The classifier and per-module extractors share the same
|
||||
model — no escalation, no per-prompt overrides in production.
|
||||
|
||||
This is a deliberate CTO choice (lens: "Cost efficiency — zero-cost
|
||||
inference on local Ollama") and is the smallest model that, in our
|
||||
limited testing, has been observed to follow the strict-JSON
|
||||
contract on the modules we route to.
|
||||
|
||||
## How the pipelines invoke the prompts
|
||||
|
||||
@@ -55,14 +68,14 @@ modules we route to.
|
||||
CLASSIFY_COMMAND = os.environ.get("INBOXOS_CLASSIFY_COMMAND", "inboxos:classify")
|
||||
EXTRACT_COMMAND_PREFIX = os.environ.get("INBOXOS_EXTRACT_COMMAND_PREFIX", "inboxos:extract-")
|
||||
|
||||
def classify(opencode_cfg, summary, logger):
|
||||
result = opencode_run_command(opencode_cfg, CLASSIFY_COMMAND, summary)
|
||||
def classify(llm_cfg, summary, logger):
|
||||
result = llm_run_command(llm_cfg, CLASSIFY_COMMAND, summary, logger=logger)
|
||||
modules = result.get("modules") or []
|
||||
return [m.strip().lower() for m in modules if m.strip()]
|
||||
|
||||
def extract_for_module(opencode_cfg, module, summary, papra_link, logger):
|
||||
def extract_for_module(llm_cfg, module, summary, papra_link, logger):
|
||||
command_name = f"{EXTRACT_COMMAND_PREFIX}{module}"
|
||||
result = opencode_run_command(opencode_cfg, command_name, summary)
|
||||
result = llm_run_command(llm_cfg, command_name, summary, logger=logger)
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
result["Message_ID"] = summary["message_id"] # pipeline overwrites
|
||||
@@ -71,16 +84,24 @@ def extract_for_module(opencode_cfg, module, summary, papra_link, logger):
|
||||
return result
|
||||
|
||||
# in process_one_message:
|
||||
modules = classify(opencode_cfg, summary, logger)
|
||||
modules = classify(llm_cfg, summary, logger)
|
||||
for module in modules:
|
||||
table_id = module_tables.get(module) # from NOCODB_MODULE_TABLES env
|
||||
if not table_id:
|
||||
log_event(logger, "module_table_missing", module=module, ...)
|
||||
continue
|
||||
extracted = extract_for_module(opencode_cfg, module, summary, papra_link, logger)
|
||||
extracted = extract_for_module(llm_cfg, module, summary, papra_link, logger)
|
||||
upsert_result = nocodb_upsert(nocodb_cfg, table_id, extracted, message_id)
|
||||
```
|
||||
|
||||
`LLMConfig` is constructed from environment via
|
||||
`LLMConfig.from_env()` (see `pipelines/common.py:414-479`). The
|
||||
`llm_run_command` entry point dispatches to the configured backend —
|
||||
default `ollama`, with the prompt loaded from `INBOXOS_PROMPT_DIR`
|
||||
and parsed for frontmatter `model:` (see
|
||||
`pipelines/common.py::llm_run_command` and `_ollama_call_one_model`,
|
||||
lines 605-624).
|
||||
|
||||
The historical pipeline reads `NOCODB_MODULE_TABLES` from the
|
||||
environment — a JSON dict like
|
||||
`{"finance":"tbl_xxx","crm":"tbl_yyy","intel":"tbl_zzz","promotions":"tbl_www"}`.
|
||||
@@ -106,8 +127,8 @@ same prompt family — there is no second contract.
|
||||
- **Observability.** Per-module logs make it obvious which extractor
|
||||
is producing low-quality data.
|
||||
- **Prompt size.** Each per-module prompt is small and focused,
|
||||
which gives the small local model (8B params) a better chance of
|
||||
following the strict-JSON contract.
|
||||
which gives the small local model (4B effective params) a better
|
||||
chance of following the strict-JSON contract.
|
||||
|
||||
A single "extract all" call would be cheaper to invoke (one round
|
||||
trip) but more expensive to operate (always pays the full LLM cost)
|
||||
@@ -125,16 +146,58 @@ version-controlled in Gitea, not embedded in n8n workflows"):
|
||||
mean a bad row is overwritten by the next run with the reverted
|
||||
prompt, so no data is permanently corrupted.
|
||||
- The strict-JSON contract is enforced by the prompt's hard-rules
|
||||
section. The Python wrapper `opencode_run_command` in
|
||||
`pipelines/common.py` parses the first non-whitespace token of
|
||||
stdout and fails loudly if it is not valid JSON. There is no
|
||||
"best effort" path.
|
||||
section. The Python wrapper `llm_run_command` /
|
||||
`_ollama_call_one_model` in `pipelines/common.py` (lines 605-624)
|
||||
parses the model's reply with `json.JSONDecoder.raw_decode` and
|
||||
fails loudly if the first non-whitespace token is not valid JSON.
|
||||
There is no "best effort" path.
|
||||
|
||||
## Live vs historical: Papra_Link asymmetry
|
||||
|
||||
The two pipelines have a documented asymmetry around `Papra_Link`:
|
||||
|
||||
- **Historical pipeline** (`pipelines/historical_pipeline.py:296-336`)
|
||||
calls `papra_write_attachment` *before* the classify step during
|
||||
the attachment walk. If any attachment is written, the first
|
||||
document id is captured and passed to `extract_for_module` as
|
||||
`papra_link`; the subsequent NocoDB row gains a `Papra_Link`
|
||||
field.
|
||||
- **Live pipeline** (`pipelines/live_pipeline.py:490-510`) calls
|
||||
`papra_decide` once per message, *after* classify. Attachment
|
||||
decisions are still recorded, but the pre-classify walk is
|
||||
skipped — the live pipeline never pays the per-message cost of an
|
||||
attachment pre-walk.
|
||||
|
||||
This is **intentional** in this phase. The historical pipeline
|
||||
already pays the LLM round-trip cost for the backfill batch, and
|
||||
capturing `Papra_Link` during the walk is essentially free. The
|
||||
live pipeline optimises for the 5-minute cron, where one round
|
||||
trip per message is the dominant cost. The structured log
|
||||
(`papra_decision` event) is the per-message source of truth for
|
||||
which documents would have been written in a live Papra run.
|
||||
|
||||
See `docs/papra-write-policy.md §Rule 1 — attachments present`
|
||||
("Live pipeline note:") for the policy view, and
|
||||
`ARCHITECTURE.md §Document Store (curated)` for the system view.
|
||||
|
||||
## Legacy: opencode CLI backend
|
||||
|
||||
`INBOXOS_LLM_BACKEND=opencode` is **kept for offline tests only**.
|
||||
The `opencode` binary install step in `historical-pipeline.yml` is
|
||||
retained for any human-driven `opencode` invocation from the repo
|
||||
worktree; the pipeline itself does not shell out to it. The
|
||||
`opencode` backend in `pipelines/common.py:378-407` is documented
|
||||
as broken on opencode 1.16.2 and is not exercised in production or
|
||||
in CI. Use the `ollama` backend (the default) for production.
|
||||
|
||||
See `docs/operations-runbook.md §6.3` for the data-sovereignty
|
||||
posture (local LLM only).
|
||||
|
||||
## Failure semantics
|
||||
|
||||
| Failure | What the pipeline does | Operator action |
|
||||
| --- | --- | --- |
|
||||
| OpenCode classify returns invalid JSON or `modules: []` | `.eml` is dead-lettered to `eml/dead-letter/<mid>.json` with the failure reason; cursor advances. | Inspect the prompt output; fix the prompt. |
|
||||
| OpenCode extract returns invalid JSON for one module | Run logs `extract_permanent_failure` for that module, stops without advancing offset, retries next run. | Inspect the prompt output; fix the prompt. |
|
||||
| OpenCode extract returns a record that does not match the table | NocoDB rejects the row (column error). Run stops; offset does not advance. | Add the column to the schema (`apply_schemas.py`), retry. |
|
||||
| Ollama classify returns invalid JSON or `modules: []` | `.eml` is dead-lettered to `eml/dead-letter/<mid>.json` with the failure reason; cursor advances. | Inspect the prompt output; fix the prompt. |
|
||||
| Ollama extract returns invalid JSON for one module | Run logs `extract_permanent_failure` for that module, stops without advancing offset, retries next run. | Inspect the prompt output; fix the prompt. |
|
||||
| Ollama extract returns a record that does not match the table | NocoDB rejects the row (column error). Run stops; offset does not advance. | Add the column to the schema (`apply_schemas.py`), retry. |
|
||||
| NocoDB is down / 5xx | Tenacity retries with exponential backoff. Permanent failure stops the run. | Restore NocoDB; re-run. |
|
||||
|
||||
+17
-3
@@ -122,6 +122,15 @@ auth model.
|
||||
|
||||
Minimum set per workflow:
|
||||
|
||||
> `OPENCODE_BIN` and `OPENCODE_PROMPT_DIR` are still read by
|
||||
> `pipelines.common.LLMConfig.from_env` (for the prompt directory)
|
||||
> and by the historical workflow's `Ensure OpenCode is installed`
|
||||
> step (for operator convenience). Neither pipeline shells out to
|
||||
> the opencode CLI in production; the `INBOXOS_LLM_BACKEND=ollama`
|
||||
> path is the default and only path used by CI and runtime. See
|
||||
> `docs/extraction-contract.md §Legacy: opencode CLI backend` for
|
||||
> the rationale.
|
||||
|
||||
| Secret | Live | Historical |
|
||||
|---|:---:|:---:|
|
||||
| `IMAP_HOST` / `IMAP_PORT` / `IMAP_USER` / `IMAP_PASSWORD` / `IMAP_USE_SSL` / `IMAP_MAILBOX` | ✅ | — |
|
||||
@@ -206,16 +215,21 @@ passthrough, 5xx retry, and connection-error retry).
|
||||
## 6. Prompt version control
|
||||
|
||||
Extraction prompts live in `.opencode/commands/*.md` in the same
|
||||
repository. Both pipelines call the OpenCode CLI twice per email:
|
||||
repository. Both pipelines call `pipelines.common.llm_run_command`
|
||||
(default backend: `ollama`) twice per email:
|
||||
|
||||
1. `inboxos:classify` — returns `{"modules": ["finance", "crm"]}`
|
||||
listing which module extractors should run.
|
||||
2. For each module in that list, `inboxos:extract-<module>` —
|
||||
returns a flat NocoDB-shaped record for that table.
|
||||
|
||||
OpenCode is expected to print one JSON object on stdout for each
|
||||
`llm_run_command` is expected to return one parsed JSON object per
|
||||
invocation. The exact field contracts are defined in the prompt
|
||||
files themselves and are reviewed via PR.
|
||||
files themselves and are reviewed via PR. See
|
||||
`docs/extraction-contract.md` for the full contract and the
|
||||
`§Legacy: opencode CLI backend` subsection for the offline-test-only
|
||||
`INBOXOS_LLM_BACKEND=opencode` path (broken on opencode 1.16.2;
|
||||
not used in production or in CI).
|
||||
|
||||
To roll a prompt change:
|
||||
|
||||
|
||||
@@ -36,10 +36,12 @@ InboxOS is a self-hosted, air-gapped personal ERP/CRM pipeline. It polls a singl
|
||||
│ (DWH) │ /completions
|
||||
└─────────────────┘ │
|
||||
▲ ▼
|
||||
┌─────────────────┐ ┌────────────┐
|
||||
│ OpenCode CLI │──│ Ollama │ local LLM
|
||||
│ (in runner) │ │ (FRP) │ (e.g. gemma4:e4b)
|
||||
└─────────────────┘ └────────────┘
|
||||
┌─────────────────┐
|
||||
│ Ollama │ local LLM
|
||||
│ (FRP) │ gemma4:e4b
|
||||
│ /v1/chat │
|
||||
│ /completions │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
- **Gmail** is the source. The runner authenticates with an App Password (2FA must be on).
|
||||
@@ -65,7 +67,14 @@ All services run in the `inboxos-net` Docker network defined in `docker/docker-c
|
||||
| 5 | **SeaweedFS** | `seaweedfs-master`, `-volume`, `-filer` | `https://storage.cloudinit.dev` (via reverse proxy) on `:8888` S3 endpoint in-network | Object store for `.eml` archive and pipeline state checkpoints. | `docker compose -f docker/docker-compose.yml restart seaweedfs-{master,volume,filer}` | `docker logs seaweedfs-{master,volume,filer}` |
|
||||
| 6 | **Ollama** | `ollama` | `https://ollama-frp.cloudinit.dev/v1` | Local LLM inference for `inboxos:classify` and `inboxos:extract-*`. Default model `gemma4:e4b`. | `docker compose -f docker/docker-compose.yml restart ollama` | `docker logs ollama` |
|
||||
|
||||
The **OpenCode CLI** is not a long-lived service; it is installed on the runner (historical workflow installs `opencode-ai@latest` via npm; the live workflow no longer needs it because the pipeline talks to Ollama's OpenAI-compatible endpoint directly). See `pipelines/common.py::llm_run_command` and the `ollama` backend.
|
||||
The **OpenCode CLI** is not invoked by either pipeline; both
|
||||
pipelines call Ollama's OpenAI-compatible `/v1/chat/completions`
|
||||
endpoint directly via `pipelines.common.llm_run_command` (default
|
||||
backend: `ollama`). The `opencode` binary install step in
|
||||
`historical-pipeline.yml` is retained for any human-driven
|
||||
`opencode` invocation from the repo worktree; the pipeline itself
|
||||
does not shell out to it. See `docs/extraction-contract.md §Legacy:
|
||||
opencode CLI backend` for the rationale.
|
||||
|
||||
The **reverse proxy** (Caddy in the commented-out `caddy` service in `docker/docker-compose.yml`) terminates TLS for `*.cloudinit.dev` and routes to each service. Operators without a reverse proxy can reach services via the `inboxos-net` Docker DNS names (e.g. `http://gitea:3000`, `http://seaweedfs-filer:8888`).
|
||||
|
||||
@@ -214,10 +223,16 @@ s3://inboxos/
|
||||
│ ├── uid-<uid>.eml (live, missing Message-ID)
|
||||
│ └── historical-<offset>.eml (historical)
|
||||
└── state/
|
||||
├── live-state.json {"last_uid": <int>, "updated_at": <iso>}
|
||||
└── historical-state.json {"byte_offset": <int>, "messages_processed": <int>, "updated_at": <iso>}
|
||||
├── live-state.json {"last_uid": <int>, "last_run_at": <iso>, "messages_seen": <int>}
|
||||
└── historical-state.json {"mbox_key": <str>, "offset": <int>, "messages_seen": <int>, "last_run_at": <iso>}
|
||||
```
|
||||
|
||||
> State-file field names come from the actual code: live
|
||||
> (`pipelines/live_pipeline.py:600-603`) and historical
|
||||
> (`pipelines/historical_pipeline.py:516-521`). The
|
||||
> `_corrupt_state_recovered` field is an additional transient
|
||||
> key on first load when the persisted JSON is unparseable.
|
||||
|
||||
The state files are tiny JSON blobs written via `s3_put_bytes` after every successful message (live) or every `BATCH_SIZE` messages (historical). They are the **only** persistent state between workflow runs.
|
||||
|
||||
---
|
||||
@@ -500,4 +515,68 @@ For the full event vocabulary, `grep -n "log_event" pipelines/common.py pipeline
|
||||
|
||||
---
|
||||
|
||||
*End of runbook. Next review: when INB-7 (Phase 5 — Full Deployment) closes, or when the next failure mode is added to §5. Last verified against the source files cited inline; any divergence is a bug in this document — fix the doc, do not just edit around it.*
|
||||
## 9. Phase 5 acceptance gate
|
||||
|
||||
> This appendix is the **must-have checklist** that closes
|
||||
> InboxOS Phase 5 (Full Deployment — INB-7). It is the integrator's
|
||||
> reference: every item must be checkbox-green before the
|
||||
> `.ciagent/inboxos/ROADMAP.md` Phase 5 status row is flipped to
|
||||
> "executed". The 20 must-haves below were produced by the
|
||||
> CIAgent plan at `.ciagent/inboxos/plans/05-PLAN.md` and verified
|
||||
> during the Wave 1 + Wave 2 + Wave 3 execution pass (see git log
|
||||
> on `milestone/v1.0-initial`).
|
||||
|
||||
### 9.1 Requirements coverage
|
||||
|
||||
| # | Must-have (verifiable) | Source-of-truth location | REQ-INB | Owner |
|
||||
|----|------------------------|--------------------------|:-------:|-------|
|
||||
| 1 | `pipelines/live_pipeline.py` runs on a 5-minute cron and on `workflow_dispatch` | `.gitea/workflows/live-pipeline.yml:62-65` | REQ-INB-1 | devops-engineer |
|
||||
| 2 | `pipelines/historical_pipeline.py` is `workflow_dispatch` only and streams a local `.mbox` end-to-end | `.gitea/workflows/historical-pipeline.yml:54-58` | REQ-INB-2 | devops-engineer |
|
||||
| 3 | All NocoDB writes are GET-then-PATCH/POST on `Message_ID`; no plain INSERT path | `pipelines/common.py:196-255` | REQ-INB-3 | data-engineer |
|
||||
| 4 | `inboxos:classify` + `inboxos:extract-<module>` is the only extraction path | `pipelines/common.py:632-653`, `docs/extraction-contract.md` | REQ-INB-4 | ml-engineer |
|
||||
| 5 | Four module tables (`finance`, `crm`, `intel`, `promotions`) with the column set per `docs/schemas.md` | `docs/schemas.md`, `docs/phase-1.1-verification/inb-28-report.md` | REQ-INB-5 | data-engineer |
|
||||
| 6 | `docs/operations-runbook.md` is the canonical day-1 incident reference | this file | REQ-INB-6 | lead-developer |
|
||||
| 7 | Two self-hosted Gitea Actions runners are provisioned and online | `docker/docker-compose.yml:103-148`, `docs/gitea-actions.md §2` | REQ-INB-7 | devops-engineer |
|
||||
| 8 | `ci-pr.yml` runs fast unit tests + 100MB e2e + live Ollama smoke on every PR | `.gitea/workflows/ci-pr.yml:63-230` | REQ-INB-8 | devops-engineer |
|
||||
| 9 | Papra integration is dry-run by default; the first ship does not call the real Papra API | `pipelines/common.py:729`, `docs/papra-write-policy.md` | REQ-INB-9 | security-engineer |
|
||||
| 10 | No third-party SaaS touches PII; only the IMAP connection to Gmail leaves the self-hosted perimeter | `docker/docker-compose.yml`, this runbook §6 | REQ-INB-10 | security-engineer |
|
||||
| 11 | A single pinned Ollama model (`gemma4:e4b`) is the production extraction path; no model-escalation chain | `pipelines/common.py:467-468`, `.opencode/commands/inboxos:*.md` frontmatter | REQ-INB-11 | ml-engineer |
|
||||
| 12 | `actions/upload-artifact` is pinned to v3 in every workflow (Gitea 1.x / GHES compat) | `.gitea/workflows/{live,historical,ci-pr}.yml` `upload-artifact@v3` | REQ-INB-12 | devops-engineer |
|
||||
| 13 | A synthetic UPSERT test exists and exercises the NocoDB idempotency path end-to-end | `pipelines/_tests/test_upsert_synthetic.py` | REQ-INB-13 | data-engineer |
|
||||
| 14 | All extraction prompts are version-controlled in `.opencode/commands/` and invoked by name | `.opencode/commands/inboxos:*.md` (5 files), `pipelines/common.py::llm_run_command` | REQ-INB-14 | ml-engineer |
|
||||
|
||||
### 9.2 Reconciliation drift closure
|
||||
|
||||
The seven doc/code drifts surfaced in research (P-1 through P-7)
|
||||
were closed during the Wave 1 + Wave 2 execution pass:
|
||||
|
||||
| # | Drift | Closed by | PR | REQ-INB |
|
||||
|----|-------|-----------|----|:-------:|
|
||||
| 15 | Model pin drift (`lfm2.5:8b` → `gemma4:e4b` in docs) | `docs: align pipelines/README + extraction-contract to gemma4:e4b pin` | #A | REQ-INB-11 |
|
||||
| 16 | `OLLAMA_BASE_URL` `/v1` suffix canonicalised | `docs: align pipelines/README + extraction-contract to gemma4:e4b pin` (folded into #A) | #A | REQ-INB-11, REQ-INB-10 |
|
||||
| 17 | Papra `INTAKE_EMAILS_IS_ENABLED` flipped to `false` in compose | `docker: disable Papra catch-all intake; align with cancelled policy` | #E | REQ-INB-9 |
|
||||
| 18 | State-file field names in runbook §4.4 corrected | `docs(runbook §4.4): correct state-file field names` | #H | REQ-INB-3, REQ-INB-6 |
|
||||
| 19 | Live-vs-historical `Papra_Link` asymmetry documented | `docs: document live-vs-historical Papra_Link asymmetry` + `docs(extraction-contract)` | #C, #B | REQ-INB-4, REQ-INB-9 |
|
||||
| 20 | Extraction-contract rewritten to describe Ollama as production LLM caller; opencode marked legacy | `docs(extraction-contract): describe Ollama as production LLM caller; opencode marked legacy` | #B | REQ-INB-4 |
|
||||
|
||||
> The `papra_decide` behavioural asymmetry in `live_pipeline.py`
|
||||
> (P-7 in the research brief) was investigated and found to be a
|
||||
> no-op: the live pipeline already calls `papra_decide` exactly
|
||||
> once with the post-classify `modules` list
|
||||
> (`pipelines/live_pipeline.py:490`). The historical pipeline
|
||||
> calls it twice (pre-classify attachment walk + post-classify
|
||||
> body write), but the live pipeline has no pre-classify walk.
|
||||
> Decision recorded as `ED-1` in `.ciagent/inboxos/CI-LOG.md`.
|
||||
|
||||
### 9.3 Closing the gate
|
||||
|
||||
A Phase 5 close is the date on which all 20 must-haves are
|
||||
checkbox-green and the `.ciagent/inboxos/ROADMAP.md` Phase 5
|
||||
status row is flipped to "executed". The lead-developer is the
|
||||
integrator; every active persona (data-engineer, backend-engineer,
|
||||
devops-engineer, ml-engineer, security-engineer, lead-developer)
|
||||
gets a review pass on this section.
|
||||
|
||||
---
|
||||
|
||||
*End of runbook. Next review: when Phase 5 (Full Deployment) closes, or when the next failure mode is added to §5. Last verified against the source files cited inline; any divergence is a bug in this document — fix the doc, do not just edit around it.*
|
||||
|
||||
@@ -20,7 +20,9 @@ The earlier `INTAKE_EMAILS_DRIVER=catch-all` integration tried to copy
|
||||
> don't need the email replicated in papra."
|
||||
|
||||
The new policy makes the selection explicit, deterministic, and
|
||||
operator-tunable.
|
||||
operator-tunable. As a corollary, the `docker/docker-compose.yml`
|
||||
`papra` service no longer advertises the `INTAKE_EMAILS_*` env vars;
|
||||
Papra writes are pipeline-driven only.
|
||||
|
||||
## The decision, in one diagram
|
||||
|
||||
@@ -65,6 +67,19 @@ with a `tags` field of `["inboxos", "message_id:<hash12>"]`. The first
|
||||
attachment's resulting document id is stored on the NocoDB row as
|
||||
`Papra_Link` (existing behavior, unchanged).
|
||||
|
||||
**Live pipeline note.** The live pipeline does not pre-classify before
|
||||
the attachment walk; it calls `papra_decide` once *after* classify
|
||||
(see `pipelines/live_pipeline.py:490-510`). As a result, the
|
||||
historical pipeline captures `Papra_Link` on the attachment path;
|
||||
the live pipeline records the decision in the `papra_decision`
|
||||
structured log event but does not currently pre-walk attachments
|
||||
to populate `Papra_Link` (an explicit optimisation for the 5-minute
|
||||
cron). The audit trail in the structured log is the source of truth
|
||||
for the per-message decision. See
|
||||
`docs/extraction-contract.md §Live vs historical: Papra_Link
|
||||
asymmetry` for the contract view, and
|
||||
`ARCHITECTURE.md §Document Store (curated)` for the system view.
|
||||
|
||||
### Rule 2 — body is the artifact (module-driven)
|
||||
|
||||
If there is no attachment AND the classifier routed the email to a
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# InboxOS — Phase 1.1 Schema Freeze Record (INB-28)
|
||||
|
||||
> Phase 1.1 verification freeze — see ROADMAP.md Phase 1.
|
||||
> This document is the **historical record** of the NocoDB schema
|
||||
> state at the close of Phase 1.1. Live sources of truth are
|
||||
> `docs/schemas.md` (column spec) and
|
||||
> `pipelines/_tests/test_upsert_synthetic.py` (the merge gate).
|
||||
|
||||
## Scope
|
||||
|
||||
This freeze records the NocoDB table IDs and column sets for the
|
||||
four module tables the live and historical pipelines write to, as
|
||||
they were first verified end-to-end during Phase 1.1.
|
||||
|
||||
If the column set ever drifts from `docs/schemas.md`, this file is
|
||||
**superseded** by `docs/schemas.md` and the schema-change procedure
|
||||
in §Schema changes (process) must be followed.
|
||||
|
||||
## Table IDs
|
||||
|
||||
| Module | NocoDB table ID | Source |
|
||||
|--------------|--------------------|----------------------------------------|
|
||||
| `finance` | `m0thluc3jyz2iok` | `pipelines/_tests/test_upsert_synthetic.py:30` |
|
||||
| `crm` | `m2wsbpp01v08pm5` | `pipelines/_tests/test_upsert_synthetic.py:31` |
|
||||
| `intel` | `msi6dj1cany1b5r` | `pipelines/_tests/test_upsert_synthetic.py:32` |
|
||||
| `promotions` | `mf7catfwwyh3jj5` | `pipelines/_tests/test_upsert_synthetic.py:33` |
|
||||
|
||||
> Note: a fresh install running `scripts/apply_schemas.py` from
|
||||
> scratch will mint **new** table IDs (NocoDB mints IDs on creation).
|
||||
> The `NOCODB_MODULE_TABLES` env var must be regenerated to match.
|
||||
> See `docs/operations-runbook.md §4.3` for the operator runbook.
|
||||
|
||||
## Column sets (per module)
|
||||
|
||||
The column set for each module table is the **union of** the
|
||||
common audit columns and the module-specific LLM fields. The
|
||||
audit columns are present in every table; the module fields are
|
||||
listed below. See `docs/schemas.md` for the authoritative column
|
||||
spec (types, constraints, sources).
|
||||
|
||||
### Common audit columns (every table)
|
||||
|
||||
| Column | Type | Notes |
|
||||
|----------------|------------------|------------------------------------------------------|
|
||||
| `Message_ID` | SingleLineText | `UNIQUE`, required. Natural key, idempotency. |
|
||||
| `Date` | DateTime | Set by the live pipeline from the email `Date:` header; nullable. |
|
||||
| `Raw_S3_Link` | URL | `s3://…` of the raw `.eml` in SeaweedFS (live only). |
|
||||
| `Papra_Link` | URL | URL of the first Papra document for the email (historical, when attachments exist). |
|
||||
| `Extracted_At` | DateTime | Set to `now_iso()` at upsert time. |
|
||||
|
||||
### `finance` — Finance / P&L
|
||||
|
||||
`Direction`, `Counterparty`, `Amount`, `Currency`, `Transaction_Date`, `Category`, `Reference`, `Notes`.
|
||||
|
||||
### `crm` — Personal CRM
|
||||
|
||||
`Contact_Name`, `Contact_Email`, `Organization`, `Role`, `Channel`, `Intent`, `Last_Contacted`, `Summary`, `Action_Needed`.
|
||||
|
||||
### `intel` — Daily News / Intel
|
||||
|
||||
`Source`, `Topic`, `Headline`, `Summary`, `Items` (JSON-encoded string in a `LongText` column), `Issued_Date`, `Read_Priority`.
|
||||
|
||||
### `promotions` — Promotions / Marketing
|
||||
|
||||
`Brand`, `Offer_Type`, `Headline`, `Discount_Pct`, `Discount_Code`, `Valid_Until`, `Category`, `Worth_Saving`, `Notes`.
|
||||
|
||||
## Per-table row counts at freeze
|
||||
|
||||
Row counts are recorded as **observed at the time of the Phase 1.1
|
||||
freeze**. They are not live counts — see NocoDB's web UI for the
|
||||
current row count on each table.
|
||||
|
||||
The Phase 1.1 freeze confirmed at least one successful round-trip
|
||||
upsert against each of the four tables, returning
|
||||
`{"action": "post"}` on first call and `{"action": "patch"}` on a
|
||||
second call with the same `Message_ID`. This is the idempotency
|
||||
acceptance criterion (REQ-INB-3, REQ-INB-13).
|
||||
|
||||
## Cross-references
|
||||
|
||||
- `docs/schemas.md` — column spec (authoritative)
|
||||
- `pipelines/_tests/test_upsert_synthetic.py` — the merge gate that exercises this contract end-to-end
|
||||
- `scripts/apply_schemas.py` — the idempotent schema bootstrap
|
||||
- `docs/operations-runbook.md §4.3` — module-table inventory and operator runbook
|
||||
- `docs/operations-runbook.md §8.2` — the Phase 1.1 verification pointer
|
||||
+8
-3
@@ -144,7 +144,10 @@ NOCODB_MODULE_TABLES={"finance":"tbl_xxx","crm":"tbl_yyy","intel":"tbl_zzz","pro
|
||||
# OpenCode + Ollama (canonical URL — see INB-9 verification)
|
||||
OPENCODE_BIN=opencode
|
||||
OPENCODE_PROMPT_DIR=./.opencode/commands
|
||||
OLLAMA_BASE_URL=https://ollama-frp.cloudinit.dev
|
||||
OLLAMA_BASE_URL=https://ollama-frp.cloudinit.dev/v1
|
||||
# Canonical form: the URL INCLUDES the /v1 suffix; pipelines/common.py
|
||||
# ::LLMConfig.from_env appends /chat/completions to whatever is supplied
|
||||
# (see LLMConfig.from_env at pipelines/common.py:463-465).
|
||||
```
|
||||
|
||||
> `NOCODB_MODULE_TABLES` is populated by [INB-4](/INB/issues/INB-4)
|
||||
@@ -181,8 +184,10 @@ The five prompt files live under `.opencode/commands/`:
|
||||
- `inboxos:extract-intel.md`
|
||||
- `inboxos:extract-promotions.md`
|
||||
|
||||
All five reference model `ollama/lfm2.5:8b` — confirmed available
|
||||
on the live Ollama (`https://ollama-frp.cloudinit.dev/api/tags`).
|
||||
All five reference model `ollama/gemma4:e4b` — confirmed available
|
||||
on the live Ollama (`https://ollama-frp.cloudinit.dev/api/tags`). This
|
||||
is the canonical pin per REQ-INB-11; the model is single-pinned with
|
||||
no escalation chain.
|
||||
Prompts are version-controlled so edits go through git review and
|
||||
never silently corrupt extracted data — re-running the pipeline
|
||||
deterministically refreshes any NocoDB record.
|
||||
|
||||
Reference in New Issue
Block a user