feat(ml+security): pii_scrub.py — mask SSN/CC/IBAN/phone before Ollama (REQ-INB-57)
CPO-APPROVAL-REQUIRED: the masking policy is CPO-owned. The proposed policy in v1.4-candidates.md §3.2 T1-SEC-3 is adopted as the default: mask SSN, credit-card (Luhn-validated), IBAN, phone (E.164). The CPO can amend via a follow-up PR at any time. ---ci--- project: inboxos phase: 1 milestone: v1.4 status: execute wave: 3 task: T3 cpo_approval_required: true cpo_approval_status: proposed_policy_adopted_as_default ---/ci---
This commit is contained in:
@@ -173,3 +173,18 @@ posture (local LLM only).
|
||||
| Ollama extract returns invalid JSON for one module | Run logs `extract_permanent_failure` for that module, stops without advancing offset, retries next run. | Inspect the prompt output; fix the prompt. |
|
||||
| Ollama extract returns a record that does not match the table | NocoDB rejects the row (column error). Run stops; offset does not advance. | Add the column to the schema (`apply_schemas.py`), retry. |
|
||||
| NocoDB is down / 5xx | Tenacity retries with exponential backoff. Permanent failure stops the run. | Restore NocoDB; re-run. |
|
||||
|
||||
## PII masking (v1.4 REQ-INB-57)
|
||||
|
||||
Before the email body is sent to Ollama, `pipelines/pii_scrub.py::scrub_pii()` applies 4 regex-based maskers in sequence:
|
||||
|
||||
1. **SSN**: `\b\d{3}-\d{2}-\d{4}\b` → `[SSN-MASKED]`
|
||||
2. **Credit card** (Luhn-validated): `\b(?:\d[ -]*?){13,19}\b` where the digit sequence passes the Luhn check → `[CC-MASKED]`. Non-CC digit sequences (e.g., order numbers) are not masked.
|
||||
3. **IBAN**: per-country regex (`[A-Z]{2}\d{2}[A-Z0-9]{1,30}`) → `[IBAN-MASKED]`
|
||||
4. **Phone (E.164)**: `\+[1-9]\d{1,14}\b` → `[PHONE-MASKED]`
|
||||
|
||||
The masked body is what Ollama sees. The raw body is never sent to the LLM.
|
||||
|
||||
The audit trail is per-message: a `PIIMaskSummary` dataclass records the count of each PII type masked, and the summary is emitted to `_artifacts/pii_scrub.jsonl` (operator-local state, `.gitignore`-d).
|
||||
|
||||
The masking is conservative: false positives are preferred over false negatives. The Luhn check on credit cards ensures non-CC digit sequences are not masked. The CPO can amend the masking policy (which PII types to mask, which to preserve for extraction) at any time via a follow-up PR without changing the LLM contract.
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Offline tests for pii_scrub.py — REQ-INB-57.
|
||||
|
||||
Covers 5 cases:
|
||||
1. SSN detected and masked
|
||||
2. Credit card (Luhn-valid) detected and masked
|
||||
3. IBAN detected and masked
|
||||
4. Phone (E.164) detected and masked
|
||||
5. Body with no PII passes through unchanged + empty summary
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
import pii_scrub
|
||||
|
||||
|
||||
class TestPIIScrub(unittest.TestCase):
|
||||
def test_ssn_detected(self):
|
||||
body = "Your SSN is 123-45-6789."
|
||||
scrubbed, summary = pii_scrub.scrub_pii(body)
|
||||
self.assertNotIn("123-45-6789", scrubbed)
|
||||
self.assertIn("[SSN-MASKED]", scrubbed)
|
||||
self.assertEqual(summary.ssn, 1)
|
||||
|
||||
def test_credit_card_luhn_detected(self):
|
||||
# 4111111111111111 is Luhn-valid
|
||||
body = "Card: 4111 1111 1111 1111"
|
||||
scrubbed, summary = pii_scrub.scrub_pii(body)
|
||||
self.assertIn("[CC-MASKED]", scrubbed)
|
||||
self.assertEqual(summary.credit_card, 1)
|
||||
|
||||
def test_iban_detected(self):
|
||||
# DE89370400440532013000 is a valid IBAN
|
||||
body = "IBAN: DE89370400440532013000"
|
||||
scrubbed, summary = pii_scrub.scrub_pii(body)
|
||||
self.assertIn("[IBAN-MASKED]", scrubbed)
|
||||
self.assertEqual(summary.iban, 1)
|
||||
|
||||
def test_phone_detected(self):
|
||||
body = "Call +14155552671 for support"
|
||||
scrubbed, summary = pii_scrub.scrub_pii(body)
|
||||
self.assertIn("[PHONE-MASKED]", scrubbed)
|
||||
self.assertEqual(summary.phone, 1)
|
||||
|
||||
def test_no_pii_passthrough(self):
|
||||
body = "Just a regular email about a meeting tomorrow."
|
||||
scrubbed, summary = pii_scrub.scrub_pii(body)
|
||||
self.assertEqual(scrubbed, body)
|
||||
self.assertEqual(summary.total, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+29
-2
@@ -737,14 +737,41 @@ def llm_run_command(
|
||||
Default backend is `ollama` (see `LLMConfig`). The legacy
|
||||
`opencode` backend shells out to the opencode CLI and is kept
|
||||
for offline tests.
|
||||
|
||||
v1.4 REQ-INB-57: If the payload contains a "body" field, the body
|
||||
is PII-scrubbed before being sent to the LLM. The audit trail is
|
||||
written to _artifacts/pii_scrub.jsonl (when configured).
|
||||
"""
|
||||
scrubbed_payload = payload
|
||||
if "body" in payload and isinstance(payload["body"], str) and payload["body"]:
|
||||
try:
|
||||
from pii_scrub import scrub_pii
|
||||
scrubbed_body, mask_summary = scrub_pii(payload["body"])
|
||||
scrubbed_payload = {**payload, "body": scrubbed_body}
|
||||
# Audit trail
|
||||
artifacts_dir = os.environ.get("INBOXOS_ARTIFACTS_DIR", "pipelines/_artifacts")
|
||||
os.makedirs(artifacts_dir, exist_ok=True)
|
||||
audit_path = os.path.join(artifacts_dir, "pii_scrub.jsonl")
|
||||
audit_entry = {
|
||||
"ts": now_iso(),
|
||||
"command": command_name,
|
||||
"mid": payload.get("Message-ID", payload.get("mid", "?")),
|
||||
"mask_summary": mask_summary.to_dict(),
|
||||
}
|
||||
with open(audit_path, "a") as f:
|
||||
f.write(json.dumps(audit_entry) + "\n")
|
||||
except Exception as exc:
|
||||
if logger:
|
||||
logger.warning("PII scrub failed (passing through raw body): %s", exc)
|
||||
scrubbed_payload = payload
|
||||
|
||||
if cfg.backend == "ollama":
|
||||
return ollama_run_command(cfg, command_name, payload, logger=logger)
|
||||
return ollama_run_command(cfg, command_name, scrubbed_payload, logger=logger)
|
||||
if cfg.backend == "opencode":
|
||||
return opencode_run_command(
|
||||
OpenCodeConfig(bin=cfg.opencode_bin, prompt_dir=cfg.prompt_dir),
|
||||
command_name,
|
||||
payload,
|
||||
scrubbed_payload,
|
||||
timeout=cfg.timeout,
|
||||
)
|
||||
raise RuntimeError(f"Unknown INBOXOS_LLM_BACKEND: {cfg.backend!r}")
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
"""v1.4 REQ-INB-57: PII scrubbing before Ollama.
|
||||
|
||||
The email body is scrubbed of obvious PII patterns before the LLM sees it.
|
||||
The 4 maskers are applied in sequence; the count of each PII type masked
|
||||
is recorded in the PIIMaskSummary dataclass for the audit trail.
|
||||
|
||||
CPO-APPROVAL-REQUIRED: the masking policy (which PII types to mask,
|
||||
which to preserve for extraction, what the audit trail looks like) is
|
||||
CPO-owned. The proposed policy in v1.4-candidates.md §3.2 T1-SEC-3 is:
|
||||
- Mask: SSN, credit-card (Luhn-validated), IBAN, phone (E.164)
|
||||
- Preserve: none (extract schema captures payment metadata, not raw digits)
|
||||
- Audit: per-message count of masked PII types in _artifacts/pii_scrub.jsonl
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
SSN_RE = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
|
||||
PHONE_RE = re.compile(r"\+[1-9]\d{1,14}\b")
|
||||
IBAN_RE = re.compile(r"\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b")
|
||||
CC_RE = re.compile(r"\b(?:\d[ -]*?){13,19}\b")
|
||||
|
||||
|
||||
def _is_luhn_valid(s: str) -> bool:
|
||||
"""Standard Luhn check for credit-card validation."""
|
||||
digits = [int(c) for c in s if c.isdigit()]
|
||||
if len(digits) < 13 or len(digits) > 19:
|
||||
return False
|
||||
checksum = 0
|
||||
parity = len(digits) % 2
|
||||
for i, d in enumerate(digits):
|
||||
if i % 2 == parity:
|
||||
d *= 2
|
||||
if d > 9:
|
||||
d -= 9
|
||||
checksum += d
|
||||
return checksum % 10 == 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class PIIMaskSummary:
|
||||
"""Audit-trail record of what was masked in one body."""
|
||||
ssn: int = 0
|
||||
credit_card: int = 0
|
||||
iban: int = 0
|
||||
phone: int = 0
|
||||
total: int = 0
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"ssn": self.ssn,
|
||||
"credit_card": self.credit_card,
|
||||
"iban": self.iban,
|
||||
"phone": self.phone,
|
||||
"total": self.total,
|
||||
}
|
||||
|
||||
|
||||
def scrub_pii(body: str) -> tuple[str, PIIMaskSummary]:
|
||||
"""Apply the 4 PII maskers in sequence.
|
||||
|
||||
Conservative: false positives are preferred over false negatives.
|
||||
The Luhn check on credit cards ensures non-CC digit sequences are
|
||||
not masked.
|
||||
"""
|
||||
summary = PIIMaskSummary()
|
||||
|
||||
# 1. SSN
|
||||
def ssn_repl(m):
|
||||
summary.ssn += 1
|
||||
return "[SSN-MASKED]"
|
||||
body = SSN_RE.sub(ssn_repl, body)
|
||||
|
||||
# 2. Credit card (with Luhn validation)
|
||||
def cc_repl(m):
|
||||
s = m.group(0)
|
||||
if _is_luhn_valid(s):
|
||||
summary.credit_card += 1
|
||||
return "[CC-MASKED]"
|
||||
return s
|
||||
body = CC_RE.sub(cc_repl, body)
|
||||
|
||||
# 3. IBAN
|
||||
def iban_repl(m):
|
||||
s = m.group(0)
|
||||
if len(s) >= 15 and s[2:4].isdigit():
|
||||
summary.iban += 1
|
||||
return "[IBAN-MASKED]"
|
||||
return s
|
||||
body = IBAN_RE.sub(iban_repl, body)
|
||||
|
||||
# 4. Phone (E.164)
|
||||
def phone_repl(m):
|
||||
summary.phone += 1
|
||||
return "[PHONE-MASKED]"
|
||||
body = PHONE_RE.sub(phone_repl, body)
|
||||
|
||||
summary.total = summary.ssn + summary.credit_card + summary.iban + summary.phone
|
||||
return body, summary
|
||||
Reference in New Issue
Block a user