164 lines
7.8 KiB
YAML
164 lines
7.8 KiB
YAML
# ── InboxOS — Live Pipeline ─────────────────────────────
|
|
# Net-new email processing. Triggered on a cron schedule (every
|
|
# 5 minutes) and on workflow_dispatch for manual runs.
|
|
#
|
|
# Responsibilities (in order):
|
|
# 1. Poll IMAP for new messages with UID > last-seen-UID.
|
|
# 2. Back up the raw .eml to SeaweedFS at
|
|
# eml/live/<yyyy>/<mm>/<dd>/<Message-ID>.eml
|
|
# 3. Call `inboxos:classify` to pick which extraction modules apply.
|
|
# 4. For each applicable module, call `inboxos:extract-<module>`
|
|
# and UPSERT the result into the configured NocoDB table.
|
|
#
|
|
# Idempotency:
|
|
# - Per-message checkpoint is flushed to SeaweedFS after every
|
|
# successful message (state/live-state.json). A crash never
|
|
# replays a successful UID.
|
|
# - All NocoDB writes go through common.nocodb_upsert which looks
|
|
# up Message_ID and PATCHes if present, POSTs otherwise.
|
|
#
|
|
# Required repository secrets (set in Gitea → repo → Settings → Actions → Secrets):
|
|
# IMAP_HOST, IMAP_PORT, IMAP_USER, IMAP_PASSWORD, IMAP_USE_SSL
|
|
# IMAP_MAILBOX (default INBOX)
|
|
# SEAWEEDFS_S3_ENDPOINT, SEAWEEDFS_S3_ACCESS_KEY, SEAWEEDFS_S3_SECRET_KEY
|
|
# SEAWEEDFS_S3_BUCKET (e.g. "inboxos")
|
|
# NOCODB_BASE_URL, NOCODB_API_TOKEN, NOCODB_PROJECT_ID
|
|
# NOCODB_MODULE_TABLES (JSON map: {"finance":"tbl_xxx","crm":"tbl_yyy",...})
|
|
# OLLAMA_BASE_URL (e.g. https://ollama-frp.cloudinit.dev/v1) ← FRP-tunneled Ollama
|
|
# OLLAMA_API_KEY (optional, only if your Ollama is auth-gated)
|
|
# INBOXOS_LLM_BACKEND (default "ollama"; legacy "opencode" available)
|
|
# INBOXOS_LLM_MODEL (default "gemma4:e4b"; classify + everything not
|
|
# routed by the extract prefix)
|
|
# INBOXOS_LLM_MODEL_EXTRACT (default "gemma4:e4b"; used for
|
|
# inboxos:extract-*. Set to a larger model
|
|
# to route extracts only; classify stays
|
|
# cheap.)
|
|
# INBOXOS_LLM_MAX_TOKENS (default "4096"; the previous 1024 ran out
|
|
# of room for the 11-field extract schema on
|
|
# rich emails — INB-21)
|
|
# INBOXOS_LLM_TIMEOUT (default "120" seconds)
|
|
#
|
|
# Papra write-through (INB-27) — selective, policy-driven. The default
|
|
# is dry-run. To enable real writes, set PAPRA_DRY_RUN=false on the
|
|
# repo (vars) after reviewing the dry-run decision log.
|
|
# PAPRA_BASE_URL (default unset; the pipeline silently skips when unset)
|
|
# PAPRA_API_TOKEN (default unset)
|
|
# PAPRA_ORGANIZATION_ID (default unset)
|
|
# PAPRA_DRY_RUN (default "true"; set to "false" to enable real writes)
|
|
# PAPRA_BODY_WRITE_MODULES (default "crm,finance,legal"; comma-separated)
|
|
# PAPRA_METADATA_ONLY_MODULES (default "intel,promotions"; comma-separated)
|
|
#
|
|
# Backend note: the live pipeline no longer shells out to the opencode
|
|
# CLI. It uses `pipelines.common.llm_run_command` to call Ollama's
|
|
# OpenAI-compatible /v1/chat/completions endpoint directly, with the
|
|
# prompt body loaded from `OPENCODE_PROMPT_DIR`. The `ollama` backend
|
|
# in the runner is configured via env vars (OLLAMA_BASE_URL,
|
|
# INBOXOS_LLM_BACKEND, INBOXOS_LLM_MODEL) — no per-job
|
|
# `~/.config/opencode/opencode.json` is required.
|
|
|
|
name: InboxOS — Live Pipeline
|
|
|
|
on:
|
|
schedule:
|
|
# Every 60 minutes. Adjust if you want a slower cadence.
|
|
- cron: "0 * * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
since_uid:
|
|
description: "Override: only fetch IMAP UIDs greater than this number (leave blank to use persisted state)"
|
|
required: false
|
|
type: string
|
|
max_emails:
|
|
description: "Override max emails per run (default 50)"
|
|
required: false
|
|
type: string
|
|
default: "50"
|
|
|
|
concurrency:
|
|
group: live-pipeline
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
process-new-emails:
|
|
name: Poll IMAP, extract, upsert
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
env:
|
|
PYTHONUNBUFFERED: "1"
|
|
steps:
|
|
# v1.4 REQ-INB-56: secret-redaction mask for the 9 secret env vars
|
|
- name: Secret mask (REQ-INB-56)
|
|
run: |
|
|
for v in IMAP_PASSWORD NOCODB_API_TOKEN NOCODB_API_TOKEN_READONLY GITEA_USER GITEA_PASS SEAWEEDFS_S3_ACCESS_KEY SEAWEEDFS_S3_SECRET_KEY PAPRA_API_TOKEN OLLAMA_API_KEY; do
|
|
val="${!v:-}"
|
|
if [ -n "$val" ]; then
|
|
echo "::add-mask::$val"
|
|
fi
|
|
done
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Install Python deps
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install \
|
|
imapclient \
|
|
boto3 \
|
|
requests \
|
|
tenacity
|
|
|
|
- name: Run live pipeline
|
|
env:
|
|
IMAP_HOST: ${{ secrets.IMAP_HOST }}
|
|
IMAP_PORT: ${{ secrets.IMAP_PORT }}
|
|
IMAP_USER: ${{ secrets.IMAP_USER }}
|
|
IMAP_PASSWORD: ${{ secrets.IMAP_PASSWORD }}
|
|
IMAP_USE_SSL: ${{ secrets.IMAP_USE_SSL }}
|
|
IMAP_MAILBOX: ${{ secrets.IMAP_MAILBOX }}
|
|
SEAWEEDFS_S3_ENDPOINT: ${{ secrets.SEAWEEDFS_S3_ENDPOINT }}
|
|
SEAWEEDFS_S3_ACCESS_KEY: ${{ secrets.SEAWEEDFS_S3_ACCESS_KEY }}
|
|
SEAWEEDFS_S3_SECRET_KEY: ${{ secrets.SEAWEEDFS_S3_SECRET_KEY }}
|
|
SEAWEEDFS_S3_BUCKET: ${{ secrets.SEAWEEDFS_S3_BUCKET }}
|
|
NOCODB_BASE_URL: ${{ secrets.NOCODB_BASE_URL }}
|
|
NOCODB_API_TOKEN: ${{ secrets.NOCODB_API_TOKEN }}
|
|
NOCODB_PROJECT_ID: ${{ secrets.NOCODB_PROJECT_ID }}
|
|
NOCODB_MODULE_TABLES: ${{ secrets.NOCODB_MODULE_TABLES }}
|
|
OPENCODE_PROMPT_DIR: ${{ vars.OPENCODE_PROMPT_DIR || secrets.OPENCODE_PROMPT_DIR || './.opencode/commands' }}
|
|
OLLAMA_BASE_URL: ${{ secrets.OLLAMA_BASE_URL }}
|
|
OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
|
|
INBOXOS_LLM_BACKEND: ${{ vars.INBOXOS_LLM_BACKEND || secrets.INBOXOS_LLM_BACKEND || 'ollama' }}
|
|
INBOXOS_LLM_MODEL: ${{ vars.INBOXOS_LLM_MODEL || secrets.INBOXOS_LLM_MODEL || 'gemma4:e4b' }}
|
|
INBOXOS_LLM_MODEL_EXTRACT: ${{ vars.INBOXOS_LLM_MODEL_EXTRACT || secrets.INBOXOS_LLM_MODEL_EXTRACT || 'gemma4:e4b' }}
|
|
INBOXOS_LLM_MAX_TOKENS: ${{ vars.INBOXOS_LLM_MAX_TOKENS || secrets.INBOXOS_LLM_MAX_TOKENS || '4096' }}
|
|
INBOXOS_LLM_TIMEOUT: ${{ vars.INBOXOS_LLM_TIMEOUT || secrets.INBOXOS_LLM_TIMEOUT || '120' }}
|
|
# Papra write-through (INB-27). Dry-run by default; flip to
|
|
# "false" via vars/secrets only after the CEO has reviewed
|
|
# the dry-run decision log on a real cycle.
|
|
PAPRA_BASE_URL: ${{ vars.PAPRA_BASE_URL || secrets.PAPRA_BASE_URL || '' }}
|
|
PAPRA_API_TOKEN: ${{ vars.PAPRA_API_TOKEN || secrets.PAPRA_API_TOKEN || '' }}
|
|
PAPRA_ORGANIZATION_ID: ${{ vars.PAPRA_ORGANIZATION_ID || secrets.PAPRA_ORGANIZATION_ID || '' }}
|
|
PAPRA_DRY_RUN: ${{ vars.PAPRA_DRY_RUN || secrets.PAPRA_DRY_RUN || 'true' }}
|
|
PAPRA_BODY_WRITE_MODULES: ${{ vars.PAPRA_BODY_WRITE_MODULES || secrets.PAPRA_BODY_WRITE_MODULES || 'crm,finance,legal' }}
|
|
PAPRA_METADATA_ONLY_MODULES: ${{ vars.PAPRA_METADATA_ONLY_MODULES || secrets.PAPRA_METADATA_ONLY_MODULES || 'intel,promotions' }}
|
|
SINCE_UID_OVERRIDE: ${{ inputs.since_uid }}
|
|
MAX_EMAILS_OVERRIDE: ${{ inputs.max_emails }}
|
|
run: |
|
|
python pipelines/live_pipeline.py
|
|
|
|
- name: Upload run log as artifact
|
|
if: always()
|
|
# Pin to v3: actions/upload-artifact@v4 uses GitHub GHES-only
|
|
# endpoints that Gitea Actions does not implement (ref:
|
|
# GHESNotSupportedError on the 2026-06-07 live run #674).
|
|
# v3 talks the older v1 Gitea-compatible artifact API and works.
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: live-pipeline-log
|
|
path: pipelines/_artifacts/live-run-*.log
|
|
if-no-files-found: ignore
|