feat(P10b): drift detection (R-018/R-019/R-020, REQ-103..113)
internal/drift/drift.go: Detector (Watch via iter.Seq2, Aggregate,
Remediate with cooldown-on-success, Acknowledge), Config with tiered
cadence (critical 5s + Path units, standard 30s, default 60s).
internal/cli/drift.go: orca drift {show,watch,acknowledge,remediate,
config}. internal/emitter/drift_path.go: systemd Path+service unit
emitter (User=orca, ProtectSystem=strict). scripts/orca-drift-notify.sh
(sha256 event JSON), orca-remediate.sh (cooldown-on-success, transient
retry). Pre-flight gate (R-020, --force + per-ns scoping). orca
system user (REQ-111), NFS detection (D-233), orca job restart for
EnvironmentFile drift (D-235).
---ci---
project: orca
phase: 10b
milestone: v0.11
status: execute
---/ci---
This commit is contained in:
Jon Chery
@@ -0,0 +1,391 @@
|
||||
// Package cli: drift.go implements the `orca drift` subcommand family
|
||||
// (P10b, v0.11; R-018/R-019/R-020, REQ-104). Subcommands:
|
||||
//
|
||||
// orca drift show current drift state (table)
|
||||
// orca drift watch [--interval=2s] [--paths=...] [--json]
|
||||
// stream drift events (iter.Seq2, D-017)
|
||||
// ctrl-c cancels via signal.NotifyContext (D-023)
|
||||
// orca drift show [--peer <host>] detailed drift events for a peer
|
||||
// orca drift acknowledge <peer> <path>
|
||||
// record operator acknowledgment
|
||||
// orca drift remediate <peer> <path> [--force]
|
||||
// trigger manual remediation
|
||||
// orca drift config show show current drift config
|
||||
// orca drift config validate validate config
|
||||
//
|
||||
// Plus the `orca job restart <name>` command for EnvironmentFile drift
|
||||
// (REQ-113, D-235) — restarts an allocation to pick up env-file drift.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/drift"
|
||||
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||
)
|
||||
|
||||
var (
|
||||
driftWatchInterval time.Duration
|
||||
driftWatchPaths []string
|
||||
driftShowPeer string
|
||||
driftConfigPath string
|
||||
driftRemediateForce bool
|
||||
driftAckPeer string
|
||||
driftAckPath string
|
||||
driftRemediatePeer string
|
||||
driftRemediatePath string
|
||||
driftWatchPollOverride time.Duration
|
||||
)
|
||||
|
||||
// driftTransport is the SSH-push surface the drift CLI needs. It
|
||||
// mirrors drift.Transport; tests substitute a mock.
|
||||
type driftTransport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error)
|
||||
ReadFile(ctx context.Context, peer string, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
// driftTransportOverride is the package-level test seam.
|
||||
var driftTransportOverride driftTransport
|
||||
|
||||
func driftTransportFromCtx() (driftTransport, error) {
|
||||
if driftTransportOverride != nil {
|
||||
return driftTransportOverride, nil
|
||||
}
|
||||
keyPath := certpaths.SSHKeyPath()
|
||||
khPath := certpaths.KnownHostsPath()
|
||||
return sshpush.NewTransport(keyPath, khPath), nil
|
||||
}
|
||||
|
||||
// driftDetectorOverride is the package-level test seam for the
|
||||
// Detector itself. When non-nil it replaces the production detector
|
||||
// (which wraps a driftTransport). Tests set it and restore nil.
|
||||
var driftDetectorOverride drift.Detector
|
||||
|
||||
func driftDetector() (drift.Detector, error) {
|
||||
if driftDetectorOverride != nil {
|
||||
return driftDetectorOverride, nil
|
||||
}
|
||||
t, err := driftTransportFromCtx()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return drift.NewDefaultDetector(t), nil
|
||||
}
|
||||
|
||||
var driftCmd = &cobra.Command{
|
||||
Use: "drift",
|
||||
Short: "Detect and remediate control-plane drift (P10b)",
|
||||
Long: `Orca's drift detector is a BACKSTOP (R-019): the primary
|
||||
consistency mechanism is systemd / Traefik / step-ca / Syncthing
|
||||
themselves. The detector polls the lead's aggregated drift state
|
||||
(drift-events-aggregated.json) and can trigger orca-remediate.sh for
|
||||
auto-remediable paths. Pre-flight drift blocks txn apply (R-020).`,
|
||||
}
|
||||
|
||||
var driftShowCmd = &cobra.Command{
|
||||
Use: "show",
|
||||
Short: "Show current drift state (table format)",
|
||||
Long: `Show the aggregated drift events from the lead. Use --peer to filter to a single peer.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
events, err := d.Aggregate(cmd.Context(), driftShowPeer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("aggregate: %w", err)
|
||||
}
|
||||
if driftShowPeer != "" {
|
||||
var filtered []drift.Event
|
||||
for _, e := range events {
|
||||
if e.Host == driftShowPeer {
|
||||
filtered = append(filtered, e)
|
||||
}
|
||||
}
|
||||
events = filtered
|
||||
}
|
||||
if jsonOutput {
|
||||
return printJSON(events)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if len(events) == 0 {
|
||||
fmt.Fprintln(out, "No drift events.")
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "%-20s %-20s %-40s %-10s %-10s\n", "EVENT-ID", "HOST", "PATH", "STATUS", "CONFIRMED")
|
||||
for _, e := range events {
|
||||
path := e.Path
|
||||
if len(path) > 40 {
|
||||
path = "..." + path[len(path)-37:]
|
||||
}
|
||||
confirmed := "no"
|
||||
if e.DriftConfirmed {
|
||||
confirmed = "yes"
|
||||
}
|
||||
fmt.Fprintf(out, "%-20s %-20s %-40s %-10s %-10s\n", e.EventID, e.Host, path, e.Status, confirmed)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftWatchCmd = &cobra.Command{
|
||||
Use: "watch",
|
||||
Short: "Stream drift events (ctrl-c to cancel)",
|
||||
Long: `Stream drift events from the lead's aggregated state. Default
|
||||
poll is 2s; override with --interval. Use --paths=<glob1>,<glob2> to
|
||||
filter. Uses iter.Seq2 (D-017) and signal.NotifyContext for ctrl-c
|
||||
(D-023).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
interval := driftWatchInterval
|
||||
if interval <= 0 {
|
||||
interval = 2 * time.Second
|
||||
}
|
||||
var specs []drift.PathSpec
|
||||
for _, p := range driftWatchPaths {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
specs = append(specs, drift.PathSpec{Pattern: p, Interval: interval})
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
for e, err := range d.Watch(ctx, specs) {
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "watch error: %v\n", err)
|
||||
continue
|
||||
}
|
||||
if jsonOutput {
|
||||
line, _ := json.Marshal(e)
|
||||
fmt.Fprintln(out, string(line))
|
||||
} else {
|
||||
fmt.Fprintf(out, "%s [%s] %s %s %s confirmed=%t\n", e.TS.Format(time.RFC3339), e.EventID, e.Host, e.Path, e.Status, e.DriftConfirmed)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftAckCmd = &cobra.Command{
|
||||
Use: "acknowledge <peer> <path>",
|
||||
Short: "Record operator acknowledgment of drift on a peer",
|
||||
Long: `Record operator acknowledgment for the given path in
|
||||
drift-acknowledgments.json on the lead. Acknowledged drift no longer
|
||||
blocks txn apply for that namespace (R-020).`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
peer := args[0]
|
||||
path := args[1]
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
if err := d.Acknowledge(cmd.Context(), peer, path); err != nil {
|
||||
return fmt.Errorf("acknowledge: %w", err)
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Acknowledged drift on %s for %s", peer, path), map[string]any{
|
||||
"peer": peer, "path": path, "status": "acknowledged",
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftRemediateCmd = &cobra.Command{
|
||||
Use: "remediate <peer> <path>",
|
||||
Short: "Trigger manual remediation of drift on a peer",
|
||||
Long: `Trigger orca-remediate.sh on the lead for the given path.
|
||||
--force bypasses the cooldown window (C4).`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
peer := args[0]
|
||||
path := args[1]
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
if err := d.Remediate(cmd.Context(), peer, path, driftRemediateForce); err != nil {
|
||||
if errors.Is(err, drift.ErrCooldown) {
|
||||
printResult(fmt.Sprintf("✗ Remediation in cooldown for %s on %s (use --force to bypass)", path, peer), map[string]any{
|
||||
"peer": peer, "path": path, "status": "cooldown",
|
||||
})
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("remediate: %w", err)
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Remediated drift on %s for %s", peer, path), map[string]any{
|
||||
"peer": peer, "path": path, "status": "remediated", "force": driftRemediateForce,
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftConfigCmd = &cobra.Command{
|
||||
Use: "config",
|
||||
Short: "Show or validate the drift config",
|
||||
}
|
||||
|
||||
var driftConfigShowCmd = &cobra.Command{
|
||||
Use: "show",
|
||||
Short: "Show the current drift config",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := drift.LoadConfig(driftConfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
if jsonOutput {
|
||||
return printJSON(cfg)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "Polling: enabled=%t default=%s max_peers=%d\n", cfg.Polling.Enabled, cfg.Polling.DefaultInterval, cfg.Polling.MaxConcurrentPeers)
|
||||
fmt.Fprintln(out, "Critical paths:")
|
||||
for _, p := range cfg.Paths.Critical {
|
||||
fmt.Fprintf(out, " [%s] %s (interval=%s, path_unit=%t)\n", p.Tier, p.Pattern, p.Interval, p.SystemdPathUnit)
|
||||
}
|
||||
fmt.Fprintln(out, "Standard paths:")
|
||||
for _, p := range cfg.Paths.Standard {
|
||||
fmt.Fprintf(out, " [%s] %s (interval=%s)\n", p.Tier, p.Pattern, p.Interval)
|
||||
}
|
||||
fmt.Fprintln(out, "Excluded paths:")
|
||||
for _, p := range cfg.Paths.Excluded {
|
||||
fmt.Fprintf(out, " %s\n", p)
|
||||
}
|
||||
fmt.Fprintf(out, "Remediation: auto=%t notify=%t\n", cfg.Remediate.Auto, cfg.Remediate.NotifyOnRemediation)
|
||||
if len(cfg.Remediate.AutoPaths) > 0 {
|
||||
fmt.Fprintln(out, " auto_paths:")
|
||||
for _, p := range cfg.Remediate.AutoPaths {
|
||||
fmt.Fprintf(out, " %s\n", p)
|
||||
}
|
||||
}
|
||||
if len(cfg.Remediate.RequireApproval) > 0 {
|
||||
fmt.Fprintln(out, " require_approval:")
|
||||
for _, p := range cfg.Remediate.RequireApproval {
|
||||
fmt.Fprintf(out, " %s\n", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftConfigValidateCmd = &cobra.Command{
|
||||
Use: "validate",
|
||||
Short: "Validate the drift config",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := drift.LoadConfig(driftConfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
if err := drift.ValidateConfig(cfg); err != nil {
|
||||
printResult(fmt.Sprintf("✗ Config invalid: %v", err), map[string]any{"valid": false, "error": err.Error()})
|
||||
return err
|
||||
}
|
||||
printResult("✓ Config valid", map[string]any{"valid": true})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// jobRestartCmd implements `orca job restart <name>` (REQ-113, D-235):
|
||||
// restart an allocation on its peer to pick up EnvironmentFile drift.
|
||||
var jobRestartCmd = &cobra.Command{
|
||||
Use: "restart <name>",
|
||||
Short: "Restart an allocation to pick up EnvironmentFile drift (REQ-113)",
|
||||
Long: `SSH to the peer running allocation <name> and run
|
||||
systemctl restart orca-alloc-<id>.service. This is the normal
|
||||
allocation lifecycle (NOT file-level remediation) and is triggered
|
||||
when /etc/orca/allocs/<id>/env drifts.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
name := args[0]
|
||||
peer := jobRestartPeer
|
||||
if peer == "" {
|
||||
return fmt.Errorf("--peer is required for job restart")
|
||||
}
|
||||
transport, err := driftTransportFromCtx()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
unit := fmt.Sprintf("orca-alloc-%s.service", name)
|
||||
restartCmd := fmt.Sprintf("systemctl restart %s", shellQuoteDrift(unit))
|
||||
out, err := transport.Exec(cmd.Context(), peer, restartCmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restart %s on %s: %w (output: %s)", unit, peer, err, string(out))
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Restarted %s on %s", unit, peer), map[string]any{
|
||||
"unit": unit, "peer": peer, "status": "restarted", "output": string(out),
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var jobRestartPeer string
|
||||
|
||||
func shellQuoteDrift(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||
}
|
||||
|
||||
func init() {
|
||||
driftWatchCmd.Flags().DurationVar(&driftWatchInterval, "interval", 2*time.Second, "poll interval (default 2s)")
|
||||
driftWatchCmd.Flags().StringSliceVar(&driftWatchPaths, "paths", nil, "comma-separated glob patterns to watch (default: all)")
|
||||
driftShowCmd.Flags().StringVar(&driftShowPeer, "peer", "", "filter to a single peer host")
|
||||
driftRemediateCmd.Flags().BoolVar(&driftRemediateForce, "force", false, "bypass the cooldown window (C4)")
|
||||
driftConfigCmd.PersistentFlags().StringVar(&driftConfigPath, "config", "", "path to drift config JSON (default: built-in)")
|
||||
jobRestartCmd.Flags().StringVar(&jobRestartPeer, "peer", "", "peer address (host:port) running the allocation")
|
||||
|
||||
driftCmd.AddCommand(driftShowCmd)
|
||||
driftCmd.AddCommand(driftWatchCmd)
|
||||
driftCmd.AddCommand(driftAckCmd)
|
||||
driftCmd.AddCommand(driftRemediateCmd)
|
||||
driftCmd.AddCommand(driftConfigCmd)
|
||||
driftConfigCmd.AddCommand(driftConfigShowCmd)
|
||||
driftConfigCmd.AddCommand(driftConfigValidateCmd)
|
||||
rootCmd.AddCommand(driftCmd)
|
||||
|
||||
jobCmd.AddCommand(jobRestartCmd)
|
||||
}
|
||||
|
||||
// writeClusterDefaultDriftConfig writes the canonical default drift
|
||||
// config to the cluster state dir so the lead has a reference copy.
|
||||
// Best-effort; caller logs failures.
|
||||
func writeClusterDefaultDriftConfig() error {
|
||||
dir := filepath.Join(os.Getenv("ORCA_HOME"), "cluster", "state")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir cluster state: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, "drift.json")
|
||||
cfg := drift.DefaultConfig()
|
||||
raw, err := json.MarshalIndent(cfg, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal drift config: %w", err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, raw, 0o644); err != nil {
|
||||
return fmt.Errorf("write drift config: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return fmt.Errorf("rename drift config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,631 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/drift"
|
||||
)
|
||||
|
||||
type mockDriftTransport struct {
|
||||
execOut []byte
|
||||
execErr error
|
||||
readOut []byte
|
||||
readErr error
|
||||
writes []mockDriftWrite
|
||||
execFn func(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
execs []string
|
||||
}
|
||||
|
||||
type mockDriftWrite struct {
|
||||
peer string
|
||||
path string
|
||||
content []byte
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func (m *mockDriftTransport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
|
||||
if m.execFn != nil {
|
||||
return m.execFn(ctx, peer, cmd)
|
||||
}
|
||||
m.execs = append(m.execs, cmd)
|
||||
return m.execOut, m.execErr
|
||||
}
|
||||
|
||||
func (m *mockDriftTransport) WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
|
||||
m.writes = append(m.writes, mockDriftWrite{peer, path, content, mode})
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *mockDriftTransport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) {
|
||||
return m.readOut, m.readErr
|
||||
}
|
||||
|
||||
// fakeDetector is a record-replay drift.Detector for CLI tests.
|
||||
type fakeDetector struct {
|
||||
aggEvents []drift.Event
|
||||
aggErr error
|
||||
remediateErr error
|
||||
ackWrites int
|
||||
remediateCalls []remediateCall
|
||||
}
|
||||
|
||||
type remediateCall struct {
|
||||
peer string
|
||||
path string
|
||||
force bool
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Watch(ctx context.Context, paths []drift.PathSpec) iter.Seq2[drift.Event, error] {
|
||||
return func(yield func(drift.Event, error) bool) {
|
||||
for _, e := range f.aggEvents {
|
||||
if !yield(e, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
<-ctx.Done()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Aggregate(ctx context.Context, leadPeer string) ([]drift.Event, error) {
|
||||
return f.aggEvents, f.aggErr
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Remediate(ctx context.Context, leadPeer, path string, force bool) error {
|
||||
f.remediateCalls = append(f.remediateCalls, remediateCall{leadPeer, path, force})
|
||||
return f.remediateErr
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Acknowledge(ctx context.Context, leadPeer, path string) error {
|
||||
f.ackWrites++
|
||||
return nil
|
||||
}
|
||||
|
||||
func setupDriftCLITest(t *testing.T) string {
|
||||
t.Helper()
|
||||
home := t.TempDir()
|
||||
t.Setenv("ORCA_HOME", home)
|
||||
t.Setenv("ORCA_LEAD_STATE_DIR", filepath.Join(home, "state"))
|
||||
return home
|
||||
}
|
||||
|
||||
func TestDriftCmdRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() == "drift" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("drift command not registered on root")
|
||||
}
|
||||
|
||||
func TestDriftSubcommandsRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() != "drift" {
|
||||
continue
|
||||
}
|
||||
want := map[string]bool{
|
||||
"show": false,
|
||||
"watch": false,
|
||||
"acknowledge": false,
|
||||
"remediate": false,
|
||||
"config": false,
|
||||
}
|
||||
for _, sub := range c.Commands() {
|
||||
if _, ok := want[sub.Name()]; ok {
|
||||
want[sub.Name()] = true
|
||||
}
|
||||
}
|
||||
for name, found := range want {
|
||||
if !found {
|
||||
t.Errorf("drift subcommand %q not registered", name)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("drift command not registered")
|
||||
}
|
||||
|
||||
func TestDriftConfigSubcommands(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() != "drift" {
|
||||
continue
|
||||
}
|
||||
for _, sub := range c.Commands() {
|
||||
if sub.Name() != "config" {
|
||||
continue
|
||||
}
|
||||
want := map[string]bool{"show": false, "validate": false}
|
||||
for _, s := range sub.Commands() {
|
||||
if _, ok := want[s.Name()]; ok {
|
||||
want[s.Name()] = true
|
||||
}
|
||||
}
|
||||
for name, found := range want {
|
||||
if !found {
|
||||
t.Errorf("drift config subcommand %q not registered", name)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("drift config not registered")
|
||||
}
|
||||
|
||||
func TestJobRestartRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() != "job" {
|
||||
continue
|
||||
}
|
||||
for _, sub := range c.Commands() {
|
||||
if sub.Name() == "restart" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatal("job restart not registered")
|
||||
}
|
||||
|
||||
func TestDriftShowEmpty(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: nil}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "No drift events") {
|
||||
t.Errorf("empty show output: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftShowTable(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: []drift.Event{
|
||||
{EventID: "E1", Host: "peer1", Path: "/etc/traefik/dynamic/orca.yml", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
for _, want := range []string{"E1", "peer1", "modified"} {
|
||||
if !bytesContains(out, want) {
|
||||
t.Errorf("output missing %q: %s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftShowJSON(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: []drift.Event{
|
||||
{EventID: "E1", Host: "peer1", Path: "/etc/x", Status: drift.StatusCreated, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
_ = rootCmd.PersistentFlags().Set("json", "true")
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show --json: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), `"event_id"`) {
|
||||
t.Errorf("json output missing event_id: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftShowPeerFilter(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: []drift.Event{
|
||||
{EventID: "E1", Host: "peer1", Path: "/a", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
{EventID: "E2", Host: "peer2", Path: "/b", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show", "--peer", "peer1"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show --peer: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
if !bytesContains(out, "E1") {
|
||||
t.Errorf("filtered output should have E1: %s", out)
|
||||
}
|
||||
if bytesContains(out, "E2") {
|
||||
t.Errorf("filtered output should NOT have E2: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftWatchStreamsAndCancels(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
// Use a fakeDetector whose Watch yields one event then blocks on
|
||||
// ctx so the stream terminates when ctrl-c (signal.NotifyContext)
|
||||
// cancels. We simulate the cancel by constructing a fake that yields
|
||||
// then returns when the consumer stops pulling OR ctx is cancelled.
|
||||
fd := &drainingFakeDetector{events: []drift.Event{
|
||||
{EventID: "E1", Host: "p", Path: "/etc/x", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "watch", "--interval", "10ms"})
|
||||
// Inject a context that auto-cancels after the events drain so
|
||||
// the watch loop exits without polluting rootCmd's context (which
|
||||
// is shared across tests). We use PersistentPreRunE's context by
|
||||
// overriding it here and restoring after.
|
||||
origCtx := rootCmd.Context()
|
||||
ctx, cancel := context.WithCancel(origCtx)
|
||||
defer cancel()
|
||||
rootCmd.SetContext(ctx)
|
||||
fd.cancelAfterYield = cancel
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift watch: %v", err)
|
||||
}
|
||||
// Restore rootCmd context for subsequent tests.
|
||||
rootCmd.SetContext(origCtx)
|
||||
if !bytesContains(buf.String(), "E1") {
|
||||
t.Errorf("watch output missing E1: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// drainingFakeDetector yields the events then cancels the provided
|
||||
// cancel func (so the watch loop's signal.NotifyContext ctx is
|
||||
// cancelled and the stream terminates cleanly).
|
||||
type drainingFakeDetector struct {
|
||||
events []drift.Event
|
||||
cancelAfterYield context.CancelFunc
|
||||
remediateCalls []remediateCall
|
||||
ackWrites int
|
||||
}
|
||||
|
||||
func (d *drainingFakeDetector) Watch(ctx context.Context, paths []drift.PathSpec) iter.Seq2[drift.Event, error] {
|
||||
return func(yield func(drift.Event, error) bool) {
|
||||
for _, e := range d.events {
|
||||
if !yield(e, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if d.cancelAfterYield != nil {
|
||||
d.cancelAfterYield()
|
||||
}
|
||||
<-ctx.Done()
|
||||
}
|
||||
}
|
||||
func (d *drainingFakeDetector) Aggregate(ctx context.Context, leadPeer string) ([]drift.Event, error) {
|
||||
return d.events, nil
|
||||
}
|
||||
func (d *drainingFakeDetector) Remediate(ctx context.Context, leadPeer, path string, force bool) error {
|
||||
d.remediateCalls = append(d.remediateCalls, remediateCall{leadPeer, path, force})
|
||||
return nil
|
||||
}
|
||||
func (d *drainingFakeDetector) Acknowledge(ctx context.Context, leadPeer, path string) error {
|
||||
d.ackWrites++
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDriftAcknowledge(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "acknowledge", "peer1", "/etc/traefik/dynamic/orca.yml"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift acknowledge: %v", err)
|
||||
}
|
||||
if fd.ackWrites != 1 {
|
||||
t.Errorf("ackWrites = %d, want 1", fd.ackWrites)
|
||||
}
|
||||
if !bytesContains(buf.String(), "Acknowledged") {
|
||||
t.Errorf("output missing Acknowledged: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftRemediate(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "remediate", "peer1", "/etc/x"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift remediate: %v", err)
|
||||
}
|
||||
if len(fd.remediateCalls) != 1 {
|
||||
t.Fatalf("remediateCalls = %d, want 1", len(fd.remediateCalls))
|
||||
}
|
||||
if fd.remediateCalls[0].peer != "peer1" || fd.remediateCalls[0].path != "/etc/x" {
|
||||
t.Errorf("remediate call: %+v", fd.remediateCalls[0])
|
||||
}
|
||||
if fd.remediateCalls[0].force {
|
||||
t.Errorf("force should be false without --force")
|
||||
}
|
||||
if !bytesContains(buf.String(), "Remediated") {
|
||||
t.Errorf("output missing Remediated: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftRemediateForce(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "remediate", "peer1", "/etc/x", "--force"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift remediate --force: %v", err)
|
||||
}
|
||||
if len(fd.remediateCalls) != 1 {
|
||||
t.Fatalf("remediateCalls = %d, want 1", len(fd.remediateCalls))
|
||||
}
|
||||
if !fd.remediateCalls[0].force {
|
||||
t.Errorf("force should be true with --force")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftRemediateCooldown(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{remediateErr: drift.ErrCooldown}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "remediate", "peer1", "/etc/x"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift remediate cooldown: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "cooldown") {
|
||||
t.Errorf("output should mention cooldown: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftConfigShow(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "config", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift config show: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
for _, want := range []string{"Polling:", "Critical paths:", "Standard paths:", "Excluded paths:", "Remediation:"} {
|
||||
if !bytesContains(out, want) {
|
||||
t.Errorf("config show missing %q: %s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftConfigValidate(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "config", "validate"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift config validate: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "valid") {
|
||||
t.Errorf("output missing 'valid': %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftConfigValidateFails(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
cfgPath := filepath.Join(t.TempDir(), "drift.json")
|
||||
bad := `{"polling":{"enabled":true,"default_interval":60000000000,"max_concurrent_peers":4},"paths":{"critical":[{"tier":"critical","pattern":"","interval":5000000000}]},"remediate":{"auto":true}}`
|
||||
if err := os.WriteFile(cfgPath, []byte(bad), 0o644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "config", "validate", "--config", cfgPath})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
t.Fatal("expected validate error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRestartRequiresPeer(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "restart", "alloc1"})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing --peer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRestartExecs(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("restarted")}
|
||||
driftTransportOverride = mt
|
||||
defer func() { driftTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "restart", "alloc1", "--peer", "peer1:22"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("job restart: %v", err)
|
||||
}
|
||||
if len(mt.execs) != 1 {
|
||||
t.Fatalf("execs = %d, want 1", len(mt.execs))
|
||||
}
|
||||
if !bytesContains(mt.execs[0], "orca-alloc-alloc1.service") {
|
||||
t.Errorf("restart cmd missing: %s", mt.execs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRestartTransientError(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execErr: fmt.Errorf("connection refused")}
|
||||
driftTransportOverride = mt
|
||||
defer func() { driftTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "restart", "alloc1", "--peer", "peer1:22"})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
t.Fatal("expected error for exec failure")
|
||||
}
|
||||
if !errors.Is(err, err) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerSetupCmdRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() == "peer-setup" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("peer-setup command not registered")
|
||||
}
|
||||
|
||||
func TestPeerSetupCreatesUserAndDir(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("ext4")}
|
||||
peerSetupTransportOverride = mt
|
||||
defer func() { peerSetupTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"peer-setup", "peer1:22"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("peer-setup: %v", err)
|
||||
}
|
||||
if len(mt.execs) < 2 {
|
||||
t.Fatalf("execs = %d, want >= 2", len(mt.execs))
|
||||
}
|
||||
useraddSeen := false
|
||||
mkdirSeen := false
|
||||
statSeen := false
|
||||
for _, c := range mt.execs {
|
||||
if bytesContains(c, "useradd -r orca") {
|
||||
useraddSeen = true
|
||||
}
|
||||
if bytesContains(c, "mkdir -p /etc/orca/state/drift-events") {
|
||||
mkdirSeen = true
|
||||
}
|
||||
if bytesContains(c, "stat -f") {
|
||||
statSeen = true
|
||||
}
|
||||
}
|
||||
if !useraddSeen {
|
||||
t.Errorf("useradd not run")
|
||||
}
|
||||
if !mkdirSeen {
|
||||
t.Errorf("mkdir drift-events not run")
|
||||
}
|
||||
if !statSeen {
|
||||
t.Errorf("stat (NFS detect) not run")
|
||||
}
|
||||
if !bytesContains(buf.String(), "nfs=false") {
|
||||
t.Errorf("output should report nfs=false: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerSetupNoOrcaUser(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("ext4")}
|
||||
peerSetupTransportOverride = mt
|
||||
defer func() { peerSetupTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"peer-setup", "peer1:22", "--no-orca-user"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("peer-setup --no-orca-user: %v", err)
|
||||
}
|
||||
useraddSeen := false
|
||||
for _, c := range mt.execs {
|
||||
if bytesContains(c, "useradd -r orca") {
|
||||
useraddSeen = true
|
||||
}
|
||||
}
|
||||
if useraddSeen {
|
||||
t.Errorf("useradd should NOT run with --no-orca-user")
|
||||
}
|
||||
if !bytesContains(buf.String(), "user=false") {
|
||||
t.Errorf("output should report user=false: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerSetupDetectsNFS(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("nfs4")}
|
||||
peerSetupTransportOverride = mt
|
||||
defer func() { peerSetupTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"peer-setup", "peer1:22"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("peer-setup: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "nfs=true") {
|
||||
t.Errorf("output should report nfs=true: %s", buf.String())
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,13 @@ func resetCommandFlags() {
|
||||
txnApplyNamespace = ""
|
||||
txnApplyTimeout = 5 * time.Minute
|
||||
txnApplyLead, txnRollbackLead = "", ""
|
||||
driftWatchInterval = 2 * time.Second
|
||||
driftWatchPaths = nil
|
||||
driftShowPeer = ""
|
||||
driftConfigPath = ""
|
||||
driftRemediateForce = false
|
||||
jobRestartPeer = ""
|
||||
peerSetupNoOrcaUser = false
|
||||
resetNSFlags()
|
||||
// Reset per-command output writers so tests that polluted them
|
||||
// (e.g. daemon tests calling cmd.SetOut(&buf)) don't leak into
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
// Package cli: peer_setup.go implements the orca system-user setup and
|
||||
// NFS detection on peers (P10b-T8/T9, v0.11, REQ-111, REQ-112/D-233).
|
||||
//
|
||||
// `orca node join` now also creates the `orca` system user on the peer
|
||||
// (so the systemd Path-unit services, which run as User=orca, have a
|
||||
// uid to run as). It also detects whether /etc/orca is on an NFS mount
|
||||
// and, when it is, skips emitting Path units for paths under /etc/orca
|
||||
// (falling back to polling for those paths — systemd Path units on NFS
|
||||
// are unreliable because inotify does not fire reliably over NFS).
|
||||
//
|
||||
// The setup is idempotent: re-running on an already-configured peer is
|
||||
// a no-op. A --no-orca-user flag skips user creation (for environments
|
||||
// with existing service accounts).
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var peerSetupNoOrcaUser bool
|
||||
|
||||
// peerSetupTransport is the SSH surface the peer-setup code needs. It
|
||||
// mirrors driftTransport; tests substitute a mock.
|
||||
type peerSetupTransport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
}
|
||||
|
||||
// peerSetupTransportOverride is the test seam.
|
||||
var peerSetupTransportOverride peerSetupTransport
|
||||
|
||||
func peerSetupTransportFromCtx() (peerSetupTransport, error) {
|
||||
if peerSetupTransportOverride != nil {
|
||||
return peerSetupTransportOverride, nil
|
||||
}
|
||||
t, err := driftTransportFromCtx()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// PeerSetupResult records what the peer setup did.
|
||||
type PeerSetupResult struct {
|
||||
UserCreated bool `json:"user_created"`
|
||||
EventsDir string `json:"events_dir"`
|
||||
NFSOnOrca bool `json:"nfs_on_orca"`
|
||||
NFSMsg string `json:"nfs_msg,omitempty"`
|
||||
}
|
||||
|
||||
// setupOrcaUser runs the idempotent `useradd -r orca` and creates the
|
||||
// drift-events directory owned by orca:orca on the peer. Returns the
|
||||
// result; a transient SSH failure returns the error (no partial state).
|
||||
func setupOrcaUser(ctx context.Context, transport peerSetupTransport, peer string) (*PeerSetupResult, error) {
|
||||
if peer == "" {
|
||||
return nil, fmt.Errorf("peer setup: peer is empty")
|
||||
}
|
||||
res := &PeerSetupResult{EventsDir: "/etc/orca/state/drift-events"}
|
||||
|
||||
if !peerSetupNoOrcaUser {
|
||||
useraddCmd := "useradd -r orca -s /usr/sbin/nologin 2>/dev/null || true"
|
||||
if _, err := transport.Exec(ctx, peer, useraddCmd); err != nil {
|
||||
return nil, fmt.Errorf("peer setup: useradd: %w", err)
|
||||
}
|
||||
res.UserCreated = true
|
||||
}
|
||||
|
||||
mkdirCmd := fmt.Sprintf("mkdir -p %s && %s", res.EventsDir, chownDriftEvents(res.EventsDir))
|
||||
if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil {
|
||||
return nil, fmt.Errorf("peer setup: mkdir drift-events: %w", err)
|
||||
}
|
||||
|
||||
nfs, msg := detectNFS(ctx, transport, peer, "/etc/orca")
|
||||
res.NFSOnOrca = nfs
|
||||
res.NFSMsg = msg
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// chownDriftEvents returns the chown command for the drift-events dir.
|
||||
// When --no-orca-user is set the orca user may not exist; chown only
|
||||
// when the user was created.
|
||||
func chownDriftEvents(dir string) string {
|
||||
if peerSetupNoOrcaUser {
|
||||
return "true"
|
||||
}
|
||||
return fmt.Sprintf("chown orca:orca %s 2>/dev/null || true", dir)
|
||||
}
|
||||
|
||||
// detectNFS checks whether the given path is on an NFS mount by running
|
||||
// `stat -f -c %T <path>` on the peer. When the fs type contains "nfs"
|
||||
// it returns (true, msg). Best-effort: a stat failure returns
|
||||
// (false, "stat unavailable").
|
||||
func detectNFS(ctx context.Context, transport peerSetupTransport, peer, path string) (bool, string) {
|
||||
out, err := transport.Exec(ctx, peer, fmt.Sprintf("stat -f -c %%T %s 2>/dev/null || echo unknown", shellQuoteDrift(path)))
|
||||
if err != nil {
|
||||
return false, "stat unavailable: " + err.Error()
|
||||
}
|
||||
fsType := strings.TrimSpace(string(out))
|
||||
if strings.Contains(fsType, "nfs") {
|
||||
return true, fmt.Sprintf("%s is on NFS (%s); skipping Path units for /etc/orca paths", path, fsType)
|
||||
}
|
||||
return false, fsType
|
||||
}
|
||||
|
||||
var peerSetupCmd = &cobra.Command{
|
||||
Use: "peer-setup <peer>",
|
||||
Short: "Create the orca system user + drift-events dir on a peer (REQ-111)",
|
||||
Long: `SSH to <peer> and idempotently create the orca system user
|
||||
(useradd -r orca -s /usr/sbin/nologin) and /etc/orca/state/drift-events/
|
||||
owned by orca:orca. Also detects NFS on /etc/orca (REQ-112/D-233) and
|
||||
logs a warning when /etc/orca is on NFS (Path units are skipped for
|
||||
those paths in that case). Use --no-orca-user to skip user creation
|
||||
(for environments with an existing service account).`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
peer := args[0]
|
||||
transport, err := peerSetupTransportFromCtx()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
res, err := setupOrcaUser(cmd.Context(), transport, peer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Peer %s set up (user=%t, nfs=%t)", peer, res.UserCreated, res.NFSOnOrca), res)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
peerSetupCmd.Flags().BoolVar(&peerSetupNoOrcaUser, "no-orca-user", false, "skip orca system user creation (env has existing service account)")
|
||||
rootCmd.AddCommand(peerSetupCmd)
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
// Package drift implements orca's drift detection subsystem
|
||||
// (P10b, v0.11 milestone; R-018/R-019/R-020, REQ-103..REQ-113).
|
||||
//
|
||||
// The model is poll-based with a systemd Path-unit fast path for
|
||||
// critical files:
|
||||
//
|
||||
// - The Detector interface exposes four operations: Watch (stream
|
||||
// events via iter.Seq2[Event, error], D-017), Aggregate (read the
|
||||
// lead-side aggregated drift state), Remediate (trigger the
|
||||
// peer-side applier via orca-remediate.sh, with cooldown-on-success
|
||||
// per C4), and Acknowledge (record operator ack).
|
||||
// - DefaultDetector implements Detector against the SSH-push
|
||||
// transport (the same *sshpush.Transport used by txn / emitter).
|
||||
// - Config encodes the tiered cadence (critical 5s + systemd Path
|
||||
// units, standard 30s polling, default 60s) plus the remediation
|
||||
// policy (auto paths, require-approval paths, notify-on-remediate).
|
||||
//
|
||||
// The package never logs key material. slog calls carry only metadata.
|
||||
package drift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusModified Status = "modified"
|
||||
StatusDeleted Status = "deleted"
|
||||
StatusCreated Status = "created"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
ActionAutoRemediated Action = "auto_remediated"
|
||||
ActionReported Action = "reported"
|
||||
ActionAcknowledged Action = "acknowledged"
|
||||
)
|
||||
|
||||
type ActionResult string
|
||||
|
||||
const (
|
||||
ActionResultSuccess ActionResult = "success"
|
||||
ActionResultFailed ActionResult = "failed"
|
||||
ActionResultSkipped ActionResult = "skipped"
|
||||
)
|
||||
|
||||
type Tier string
|
||||
|
||||
const (
|
||||
TierCritical Tier = "critical"
|
||||
TierStandard Tier = "standard"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
EventID string `json:"event_id"`
|
||||
TS time.Time `json:"ts"`
|
||||
Host string `json:"host"`
|
||||
Path string `json:"path"`
|
||||
Status Status `json:"status"`
|
||||
NewSHA256 string `json:"new_sha256"`
|
||||
LatestTxn string `json:"latest_txn"`
|
||||
ExpectedSHA256 string `json:"expected_sha256"`
|
||||
DriftConfirmed bool `json:"drift_confirmed"`
|
||||
Action Action `json:"action"`
|
||||
ActionResult ActionResult `json:"action_result"`
|
||||
OperatorID string `json:"operator_id"`
|
||||
}
|
||||
|
||||
type PathSpec struct {
|
||||
Tier Tier `json:"tier"`
|
||||
Pattern string `json:"pattern"`
|
||||
Interval time.Duration `json:"interval"`
|
||||
SystemdPathUnit bool `json:"systemd_path_unit"`
|
||||
}
|
||||
|
||||
type RemediationPolicy struct {
|
||||
Auto bool `json:"auto"`
|
||||
AutoPaths []string `json:"auto_paths"`
|
||||
RequireApproval []string `json:"require_approval"`
|
||||
NotifyOnRemediation bool `json:"notify_on_remediation"`
|
||||
}
|
||||
|
||||
type PollingConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DefaultInterval time.Duration `json:"default_interval"`
|
||||
MaxConcurrentPeers int `json:"max_concurrent_peers"`
|
||||
}
|
||||
|
||||
type PathsConfig struct {
|
||||
Critical []PathSpec `json:"critical"`
|
||||
Standard []PathSpec `json:"standard"`
|
||||
Excluded []string `json:"excluded"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Polling PollingConfig `json:"polling"`
|
||||
Paths PathsConfig `json:"paths"`
|
||||
Remediate RemediationPolicy `json:"remediate"`
|
||||
}
|
||||
|
||||
type Detector interface {
|
||||
Watch(ctx context.Context, paths []PathSpec) iter.Seq2[Event, error]
|
||||
Aggregate(ctx context.Context, leadPeer string) ([]Event, error)
|
||||
Remediate(ctx context.Context, leadPeer, path string, force bool) error
|
||||
Acknowledge(ctx context.Context, leadPeer, path string) error
|
||||
}
|
||||
|
||||
type Transport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error)
|
||||
ReadFile(ctx context.Context, peer string, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
func LeadStateDir() string {
|
||||
if p := os.Getenv("ORCA_LEAD_STATE_DIR"); p != "" {
|
||||
return p
|
||||
}
|
||||
return "/etc/orca/state"
|
||||
}
|
||||
|
||||
func AggregatedPath() string {
|
||||
return filepath.Join(LeadStateDir(), "drift-events-aggregated.json")
|
||||
}
|
||||
|
||||
func AcknowledgmentsPath() string {
|
||||
return filepath.Join(LeadStateDir(), "drift-acknowledgments.json")
|
||||
}
|
||||
|
||||
func RemediatorPath() string {
|
||||
return "/usr/local/sbin/orca-remediate.sh"
|
||||
}
|
||||
|
||||
type DefaultDetector struct {
|
||||
transport Transport
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewDefaultDetector(t Transport) *DefaultDetector {
|
||||
return &DefaultDetector{transport: t, cooldown: 5 * time.Minute}
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) SetCooldown(dur time.Duration) {
|
||||
if dur > 0 {
|
||||
d.cooldown = dur
|
||||
}
|
||||
}
|
||||
|
||||
type aggregatedDoc struct {
|
||||
TS time.Time `json:"ts"`
|
||||
Events []Event `json:"events"`
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Watch(ctx context.Context, paths []PathSpec) iter.Seq2[Event, error] {
|
||||
interval := defaultWatchInterval(paths)
|
||||
if interval <= 0 {
|
||||
interval = 2 * time.Second
|
||||
}
|
||||
return func(yield func(Event, error) bool) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
seen := make(map[string]bool)
|
||||
for {
|
||||
events, err := d.Aggregate(ctx, "")
|
||||
if err != nil {
|
||||
if !yield(Event{}, err) {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
for _, e := range events {
|
||||
if !matchesPaths(e.Path, paths) {
|
||||
continue
|
||||
}
|
||||
key := e.EventID
|
||||
if key == "" {
|
||||
key = e.Host + "|" + e.Path + "|" + e.TS.Format(time.RFC3339Nano)
|
||||
}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
if !yield(e, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Aggregate(ctx context.Context, leadPeer string) ([]Event, error) {
|
||||
var raw []byte
|
||||
var err error
|
||||
if leadPeer == "" {
|
||||
raw, err = os.ReadFile(AggregatedPath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("drift: read aggregated: %w", err)
|
||||
}
|
||||
} else {
|
||||
raw, err = d.transport.ReadFile(ctx, leadPeer, AggregatedPath())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drift: read aggregated from %s: %w", leadPeer, err)
|
||||
}
|
||||
}
|
||||
if len(strings.TrimSpace(string(raw))) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var doc aggregatedDoc
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("drift: parse aggregated: %w", err)
|
||||
}
|
||||
return doc.Events, nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Remediate(ctx context.Context, leadPeer, path string, force bool) error {
|
||||
if leadPeer == "" {
|
||||
return fmt.Errorf("drift: remediate requires a lead peer")
|
||||
}
|
||||
if !force {
|
||||
if d.inCooldown(path) {
|
||||
slog.Info("drift remediate skipped (cooldown)", "path", path, "peer", leadPeer)
|
||||
return ErrCooldown
|
||||
}
|
||||
}
|
||||
cmd := fmt.Sprintf("bash %s %s %s %s", shellQuote(RemediatorPath()), shellQuote(leadPeer), shellQuote(""), shellQuote(path))
|
||||
out, err := d.transport.Exec(ctx, leadPeer, cmd)
|
||||
if err != nil {
|
||||
if isTransientSSH(err) {
|
||||
slog.Warn("drift remediate transient failure (no cooldown)", "path", path, "peer", leadPeer, "error", err)
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("drift: remediate %s on %s: %w (output: %s)", path, leadPeer, err, string(out))
|
||||
}
|
||||
if !force {
|
||||
d.markCooldown(path)
|
||||
}
|
||||
slog.Info("drift remediate ok", "path", path, "peer", leadPeer, "output", string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Acknowledge(ctx context.Context, leadPeer, path string) error {
|
||||
if leadPeer == "" {
|
||||
return fmt.Errorf("drift: acknowledge requires a lead peer")
|
||||
}
|
||||
entry := map[string]any{
|
||||
"ts": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"path": path,
|
||||
"operator": os.Getenv("ORCA_OPERATOR_ID"),
|
||||
}
|
||||
raw, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift: marshal ack: %w", err)
|
||||
}
|
||||
if _, err := d.transport.WriteFileIdempotent(ctx, leadPeer, AcknowledgmentsPath(), append(raw, '\n'), 0o644); err != nil {
|
||||
return fmt.Errorf("drift: write ack: %w", err)
|
||||
}
|
||||
slog.Info("drift acknowledged", "path", path, "peer", leadPeer)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) inCooldown(path string) bool {
|
||||
hash := pathHash(path)
|
||||
stateDir := LeadStateDir()
|
||||
cooldownDir := filepath.Join(stateDir, "remediation-cooldown")
|
||||
stampPath := filepath.Join(cooldownDir, hash)
|
||||
info, err := os.Stat(stampPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return time.Since(info.ModTime()) < d.cooldown
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) markCooldown(path string) {
|
||||
hash := pathHash(path)
|
||||
stateDir := LeadStateDir()
|
||||
cooldownDir := filepath.Join(stateDir, "remediation-cooldown")
|
||||
if err := os.MkdirAll(cooldownDir, 0o755); err != nil {
|
||||
slog.Warn("drift markCooldown mkdir failed", "dir", cooldownDir, "error", err)
|
||||
return
|
||||
}
|
||||
stampPath := filepath.Join(cooldownDir, hash)
|
||||
if err := os.WriteFile(stampPath, []byte(time.Now().UTC().Format(time.RFC3339Nano)), 0o644); err != nil {
|
||||
slog.Warn("drift markCooldown write failed", "path", stampPath, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
ErrCooldown = errors.New("drift: remediation in cooldown")
|
||||
ErrPathExcluded = errors.New("drift: path is excluded")
|
||||
ErrOverlapCritical = errors.New("drift: path appears in both critical and excluded")
|
||||
)
|
||||
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
if path == "" {
|
||||
return DefaultConfig(), nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return DefaultConfig(), nil
|
||||
}
|
||||
return nil, fmt.Errorf("drift: load config %s: %w", path, err)
|
||||
}
|
||||
var cfg Config
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("drift: parse config %s: %w", path, err)
|
||||
}
|
||||
applyConfigDefaults(&cfg)
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func ValidateConfig(cfg *Config) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("drift: nil config")
|
||||
}
|
||||
if cfg.Polling.Enabled && cfg.Polling.DefaultInterval <= 0 {
|
||||
return fmt.Errorf("drift: polling enabled but default_interval is zero")
|
||||
}
|
||||
for _, p := range cfg.Paths.Critical {
|
||||
if p.Pattern == "" {
|
||||
return fmt.Errorf("drift: critical path has empty pattern")
|
||||
}
|
||||
if p.Tier == "" {
|
||||
return fmt.Errorf("drift: critical path %q has empty tier", p.Pattern)
|
||||
}
|
||||
}
|
||||
for _, p := range cfg.Paths.Standard {
|
||||
if p.Pattern == "" {
|
||||
return fmt.Errorf("drift: standard path has empty pattern")
|
||||
}
|
||||
}
|
||||
for _, ex := range cfg.Paths.Excluded {
|
||||
for _, c := range cfg.Paths.Critical {
|
||||
if pathGlobMatch(ex, c.Pattern) {
|
||||
return fmt.Errorf("drift: excluded pattern %q overlaps critical %q: %w", ex, c.Pattern, ErrOverlapCritical)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func DefaultConfig() *Config {
|
||||
cfg := &Config{
|
||||
Polling: PollingConfig{
|
||||
Enabled: true,
|
||||
DefaultInterval: 60 * time.Second,
|
||||
MaxConcurrentPeers: 8,
|
||||
},
|
||||
Paths: PathsConfig{
|
||||
Critical: []PathSpec{
|
||||
{Tier: TierCritical, Pattern: "/etc/traefik/dynamic/orca.yml", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/systemd/system/orca-alloc-*.service", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/nftables.d/orca.nft", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/traefik/dynamic/*", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/systemd/system/orca-alloc-*.service", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/sudoers.d/orca-*", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
},
|
||||
Standard: []PathSpec{
|
||||
{Tier: TierStandard, Pattern: "/etc/orca/actual/*", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/sudoers.d/orca-*", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-collector.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-aggregator.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-pull.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-drift.{service,timer}", Interval: 30 * time.Second},
|
||||
},
|
||||
Excluded: []string{
|
||||
"/etc/orca/credentials/*",
|
||||
"/run/orca/*",
|
||||
"/etc/orca/state/drift-events/*",
|
||||
},
|
||||
},
|
||||
Remediate: RemediationPolicy{
|
||||
Auto: true,
|
||||
AutoPaths: []string{"/etc/traefik/dynamic/*", "/etc/nftables.d/*", "/etc/sudoers.d/orca-*"},
|
||||
RequireApproval: []string{"/etc/systemd/system/orca-alloc-*.service"},
|
||||
NotifyOnRemediation: true,
|
||||
},
|
||||
}
|
||||
applyConfigDefaults(cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func applyConfigDefaults(cfg *Config) {
|
||||
if cfg.Polling.DefaultInterval == 0 {
|
||||
cfg.Polling.DefaultInterval = 60 * time.Second
|
||||
}
|
||||
if cfg.Polling.MaxConcurrentPeers == 0 {
|
||||
cfg.Polling.MaxConcurrentPeers = 8
|
||||
}
|
||||
for i := range cfg.Paths.Critical {
|
||||
if cfg.Paths.Critical[i].Tier == "" {
|
||||
cfg.Paths.Critical[i].Tier = TierCritical
|
||||
}
|
||||
if cfg.Paths.Critical[i].Interval == 0 {
|
||||
cfg.Paths.Critical[i].Interval = 5 * time.Second
|
||||
}
|
||||
}
|
||||
for i := range cfg.Paths.Standard {
|
||||
if cfg.Paths.Standard[i].Tier == "" {
|
||||
cfg.Paths.Standard[i].Tier = TierStandard
|
||||
}
|
||||
if cfg.Paths.Standard[i].Interval == 0 {
|
||||
cfg.Paths.Standard[i].Interval = 30 * time.Second
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NamespaceScope(events []Event, namespace string) []Event {
|
||||
if namespace == "" {
|
||||
var out []Event
|
||||
for _, e := range events {
|
||||
if !e.DriftConfirmed || e.Action == ActionAcknowledged {
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
var out []Event
|
||||
for _, e := range events {
|
||||
if !e.DriftConfirmed || e.Action == ActionAcknowledged {
|
||||
continue
|
||||
}
|
||||
if nsForPath(e.Path) == namespace {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nsForPath(p string) string {
|
||||
const prefix = "/etc/orca/actual/"
|
||||
if !strings.HasPrefix(p, prefix) {
|
||||
return ""
|
||||
}
|
||||
rest := p[len(prefix):]
|
||||
if i := strings.IndexByte(rest, '/'); i >= 0 {
|
||||
return rest[:i]
|
||||
}
|
||||
return rest
|
||||
}
|
||||
|
||||
func matchesPaths(p string, specs []PathSpec) bool {
|
||||
if len(specs) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, s := range specs {
|
||||
if pathGlobMatch(s.Pattern, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func defaultWatchInterval(specs []PathSpec) time.Duration {
|
||||
var min time.Duration
|
||||
for _, s := range specs {
|
||||
if s.Interval > 0 && (min == 0 || s.Interval < min) {
|
||||
min = s.Interval
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
|
||||
func pathGlobMatch(pattern, p string) bool {
|
||||
if pattern == "" {
|
||||
return false
|
||||
}
|
||||
expanded := expandBraces(pattern)
|
||||
for _, alt := range expanded {
|
||||
if globMatchSegment(alt, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func globMatchSegment(pattern, p string) bool {
|
||||
if pattern == "" {
|
||||
return false
|
||||
}
|
||||
if pattern == p {
|
||||
return true
|
||||
}
|
||||
if strings.HasSuffix(pattern, "/*") {
|
||||
prefix := pattern[:len(pattern)-2]
|
||||
if p == prefix {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(p, prefix+"/")
|
||||
}
|
||||
ok, err := filepath.Match(pattern, p)
|
||||
return err == nil && ok
|
||||
}
|
||||
|
||||
func expandBraces(p string) []string {
|
||||
open := strings.IndexByte(p, '{')
|
||||
if open < 0 {
|
||||
return []string{p}
|
||||
}
|
||||
close := strings.IndexByte(p[open:], '}')
|
||||
if close < 0 {
|
||||
return []string{p}
|
||||
}
|
||||
close += open
|
||||
prefix := p[:open]
|
||||
body := p[open+1 : close]
|
||||
suffix := p[close+1:]
|
||||
alternatives := strings.Split(body, ",")
|
||||
var out []string
|
||||
for _, a := range alternatives {
|
||||
for _, tail := range expandBraces(suffix) {
|
||||
out = append(out, prefix+a+tail)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pathHash(p string) string {
|
||||
sum := sha256.Sum256([]byte(p))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func isTransientSSH(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
for _, sub := range []string{"connection refused", "i/o timeout", "EOF", "no such host", "connection reset", "timeout", "deadline exceeded", "temporarily unavailable"} {
|
||||
if strings.Contains(s, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||
}
|
||||
|
||||
func ParseEvent(raw []byte) (Event, error) {
|
||||
var e Event
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return Event{}, fmt.Errorf("drift: parse event: %w", err)
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func MarshalEvent(e Event) ([]byte, error) {
|
||||
return json.MarshalIndent(e, "", " ")
|
||||
}
|
||||
|
||||
var _ Detector = (*DefaultDetector)(nil)
|
||||
@@ -0,0 +1,465 @@
|
||||
package drift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type mockTransport struct {
|
||||
execOut []byte
|
||||
execErr error
|
||||
execFn func(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
readOut []byte
|
||||
readErr error
|
||||
writes []writeEntry
|
||||
}
|
||||
|
||||
type writeEntry struct {
|
||||
peer string
|
||||
path string
|
||||
content []byte
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func (m *mockTransport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
|
||||
if m.execFn != nil {
|
||||
return m.execFn(ctx, peer, cmd)
|
||||
}
|
||||
return m.execOut, m.execErr
|
||||
}
|
||||
|
||||
func (m *mockTransport) WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
|
||||
m.writes = append(m.writes, writeEntry{peer, path, content, mode})
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *mockTransport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) {
|
||||
return m.readOut, m.readErr
|
||||
}
|
||||
|
||||
func setupDriftTestEnv(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ORCA_LEAD_STATE_DIR", dir)
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestParseEvent(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"event_id":"EVT-1",
|
||||
"ts":"2026-01-01T00:00:00Z",
|
||||
"host":"peer1",
|
||||
"path":"/etc/traefik/dynamic/orca.yml",
|
||||
"status":"modified",
|
||||
"new_sha256":"abc",
|
||||
"latest_txn":"T-1234567890abcdef",
|
||||
"expected_sha256":"def",
|
||||
"drift_confirmed":true,
|
||||
"action":"reported",
|
||||
"action_result":"skipped",
|
||||
"operator_id":"op1"
|
||||
}`)
|
||||
e, err := ParseEvent(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseEvent: %v", err)
|
||||
}
|
||||
if e.EventID != "EVT-1" {
|
||||
t.Errorf("EventID = %q, want EVT-1", e.EventID)
|
||||
}
|
||||
if e.Host != "peer1" {
|
||||
t.Errorf("Host = %q, want peer1", e.Host)
|
||||
}
|
||||
if e.Path != "/etc/traefik/dynamic/orca.yml" {
|
||||
t.Errorf("Path = %q", e.Path)
|
||||
}
|
||||
if e.Status != StatusModified {
|
||||
t.Errorf("Status = %q, want modified", e.Status)
|
||||
}
|
||||
if !e.DriftConfirmed {
|
||||
t.Errorf("DriftConfirmed = false, want true")
|
||||
}
|
||||
if e.Action != ActionReported {
|
||||
t.Errorf("Action = %q, want reported", e.Action)
|
||||
}
|
||||
if e.ActionResult != ActionResultSkipped {
|
||||
t.Errorf("ActionResult = %q, want skipped", e.ActionResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEventInvalid(t *testing.T) {
|
||||
_, err := ParseEvent([]byte(`{not json`))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid JSON, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalEventRoundTrip(t *testing.T) {
|
||||
e := Event{EventID: "E1", Host: "h", Path: "/p", Status: StatusCreated, DriftConfirmed: true}
|
||||
raw, err := MarshalEvent(e)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalEvent: %v", err)
|
||||
}
|
||||
out, err := ParseEvent(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseEvent: %v", err)
|
||||
}
|
||||
if out.EventID != e.EventID || out.Host != e.Host || out.Path != e.Path || out.Status != e.Status {
|
||||
t.Errorf("round-trip mismatch: %+v vs %+v", out, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefault(t *testing.T) {
|
||||
cfg, err := LoadConfig("")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig empty: %v", err)
|
||||
}
|
||||
if !cfg.Polling.Enabled {
|
||||
t.Errorf("Polling.Enabled = false, want true")
|
||||
}
|
||||
if cfg.Polling.DefaultInterval != 60*time.Second {
|
||||
t.Errorf("DefaultInterval = %v, want 60s", cfg.Polling.DefaultInterval)
|
||||
}
|
||||
if len(cfg.Paths.Critical) == 0 {
|
||||
t.Errorf("Critical paths empty")
|
||||
}
|
||||
found := false
|
||||
for _, c := range cfg.Paths.Critical {
|
||||
if c.Pattern == "/etc/traefik/dynamic/orca.yml" {
|
||||
found = true
|
||||
if c.Interval != 5*time.Second {
|
||||
t.Errorf("critical interval = %v, want 5s", c.Interval)
|
||||
}
|
||||
if !c.SystemdPathUnit {
|
||||
t.Errorf("SystemdPathUnit = false, want true")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("traefik critical path missing")
|
||||
}
|
||||
found = false
|
||||
for _, s := range cfg.Paths.Standard {
|
||||
if s.Pattern == "/etc/orca/actual/*" {
|
||||
found = true
|
||||
if s.Interval != 30*time.Second {
|
||||
t.Errorf("standard interval = %v, want 30s", s.Interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("/etc/orca/actual/* standard path missing")
|
||||
}
|
||||
if !cfg.Remediate.Auto {
|
||||
t.Errorf("Remediate.Auto = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "drift.json")
|
||||
cfgJSON := `{
|
||||
"polling": {"enabled": true, "default_interval": 90000000000, "max_concurrent_peers": 4},
|
||||
"paths": {
|
||||
"critical": [{"tier":"critical","pattern":"/etc/critical.yml","interval":5000000000,"systemd_path_unit":true}],
|
||||
"standard": [{"tier":"standard","pattern":"/etc/standard.yml","interval":30000000000}],
|
||||
"excluded": ["/run/orca/*"]
|
||||
},
|
||||
"remediate": {"auto": false, "auto_paths": [], "require_approval": ["/etc/critical.yml"], "notify_on_remediation": false}
|
||||
}`
|
||||
if err := os.WriteFile(path, []byte(cfgJSON), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
cfg, err := LoadConfig(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig: %v", err)
|
||||
}
|
||||
if cfg.Polling.MaxConcurrentPeers != 4 {
|
||||
t.Errorf("MaxConcurrentPeers = %d, want 4", cfg.Polling.MaxConcurrentPeers)
|
||||
}
|
||||
if cfg.Polling.DefaultInterval != 90*time.Second {
|
||||
t.Errorf("DefaultInterval = %v, want 90s", cfg.Polling.DefaultInterval)
|
||||
}
|
||||
if cfg.Remediate.Auto {
|
||||
t.Errorf("Auto = true, want false")
|
||||
}
|
||||
if len(cfg.Paths.Critical) != 1 || cfg.Paths.Critical[0].Pattern != "/etc/critical.yml" {
|
||||
t.Errorf("critical = %+v", cfg.Paths.Critical)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigValid(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
if err := ValidateConfig(cfg); err != nil {
|
||||
t.Fatalf("ValidateConfig(default): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigEmptyPattern(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Paths.Critical[0].Pattern = ""
|
||||
if err := ValidateConfig(cfg); err == nil {
|
||||
t.Fatal("expected error for empty critical pattern")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigPollingZero(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Polling.Enabled = true
|
||||
cfg.Polling.DefaultInterval = 0
|
||||
if err := ValidateConfig(cfg); err == nil {
|
||||
t.Fatal("expected error for zero default_interval with polling enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigOverlapCriticalExcluded(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Paths.Excluded = append(cfg.Paths.Excluded, "/etc/traefik/dynamic/orca.yml")
|
||||
err := ValidateConfig(cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected overlap error")
|
||||
}
|
||||
if !errors.Is(err, ErrOverlapCritical) {
|
||||
t.Errorf("expected ErrOverlapCritical, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigNil(t *testing.T) {
|
||||
if err := ValidateConfig(nil); err == nil {
|
||||
t.Fatal("expected error for nil config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateLocalMissing(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
events, err := d.Aggregate(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate missing: %v", err)
|
||||
}
|
||||
if events != nil {
|
||||
t.Errorf("events = %v, want nil", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateLocal(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
events, err := d.Aggregate(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate: %v", err)
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events len = %d, want 1", len(events))
|
||||
}
|
||||
if events[0].EventID != "E1" {
|
||||
t.Errorf("EventID = %q, want E1", events[0].EventID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateRemote(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E2","host":"p","path":"/etc/orca/actual/ns-a/x","status":"created","drift_confirmed":true}]}`
|
||||
mt := &mockTransport{readOut: []byte(doc)}
|
||||
d := NewDefaultDetector(mt)
|
||||
events, err := d.Aggregate(context.Background(), "lead:22")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate remote: %v", err)
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events len = %d, want 1", len(events))
|
||||
}
|
||||
if events[0].EventID != "E2" {
|
||||
t.Errorf("EventID = %q, want E2", events[0].EventID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchCancels(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
d.SetCooldown(50 * time.Millisecond)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
gotEvents := 0
|
||||
for e, err := range d.Watch(ctx, []PathSpec{{Pattern: "/etc/traefik/dynamic/orca.yml", Interval: 10 * time.Millisecond}}) {
|
||||
if err != nil {
|
||||
t.Fatalf("Watch err: %v", err)
|
||||
}
|
||||
gotEvents++
|
||||
_ = e
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
if gotEvents != 1 {
|
||||
t.Errorf("gotEvents = %d, want 1", gotEvents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchStreamsAndDedupes(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
d.SetCooldown(50 * time.Millisecond)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
|
||||
defer cancel()
|
||||
got := 0
|
||||
for _, err := range d.Watch(ctx, nil) {
|
||||
if err != nil {
|
||||
t.Fatalf("Watch err: %v", err)
|
||||
}
|
||||
got++
|
||||
}
|
||||
if got != 1 {
|
||||
t.Errorf("got = %d, want 1 (dedup)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateCooldownOnSuccess(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execOut: []byte("ok")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml", false); err != nil {
|
||||
t.Fatalf("first Remediate: %v", err)
|
||||
}
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml", false); err == nil {
|
||||
t.Fatal("second Remediate should hit cooldown")
|
||||
} else if !errors.Is(err, ErrCooldown) {
|
||||
t.Errorf("expected ErrCooldown, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateForceBypassesCooldown(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execOut: []byte("ok")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", false); err != nil {
|
||||
t.Fatalf("first: %v", err)
|
||||
}
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", true); err != nil {
|
||||
t.Fatalf("force bypass: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateTransientNoCooldown(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execErr: fmt.Errorf("connection refused")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", false); err == nil {
|
||||
t.Fatal("expected transient error")
|
||||
}
|
||||
if d.inCooldown("/etc/p") {
|
||||
t.Errorf("cooldown entered after transient failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateRequiresLead(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
if err := d.Remediate(context.Background(), "", "/etc/p", true); err == nil {
|
||||
t.Fatal("expected error for empty lead")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgeWrites(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{}
|
||||
d := NewDefaultDetector(mt)
|
||||
if err := d.Acknowledge(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml"); err != nil {
|
||||
t.Fatalf("Acknowledge: %v", err)
|
||||
}
|
||||
if len(mt.writes) != 1 {
|
||||
t.Fatalf("writes = %d, want 1", len(mt.writes))
|
||||
}
|
||||
w := mt.writes[0]
|
||||
if w.peer != "lead:22" {
|
||||
t.Errorf("peer = %q, want lead:22", w.peer)
|
||||
}
|
||||
if w.path != AcknowledgmentsPath() {
|
||||
t.Errorf("path = %q, want %q", w.path, AcknowledgmentsPath())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgeRequiresLead(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
if err := d.Acknowledge(context.Background(), "", "/etc/p"); err == nil {
|
||||
t.Fatal("expected error for empty lead")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceScopeClusterWide(t *testing.T) {
|
||||
events := []Event{
|
||||
{EventID: "E1", Path: "/etc/traefik/dynamic/orca.yml", DriftConfirmed: true},
|
||||
{EventID: "E2", Path: "/etc/orca/actual/ns-a/x", DriftConfirmed: true},
|
||||
{EventID: "E3", Path: "/etc/orca/actual/ns-b/y", DriftConfirmed: true, Action: ActionAcknowledged},
|
||||
}
|
||||
out := NamespaceScope(events, "")
|
||||
if len(out) != 2 {
|
||||
t.Errorf("cluster-wide scope len = %d, want 2 (ack excluded)", len(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceScopePerNamespace(t *testing.T) {
|
||||
events := []Event{
|
||||
{EventID: "E1", Path: "/etc/orca/actual/ns-a/x", DriftConfirmed: true},
|
||||
{EventID: "E2", Path: "/etc/orca/actual/ns-b/y", DriftConfirmed: true},
|
||||
{EventID: "E3", Path: "/etc/traefik/dynamic/orca.yml", DriftConfirmed: true},
|
||||
}
|
||||
nsA := NamespaceScope(events, "ns-a")
|
||||
if len(nsA) != 1 || nsA[0].EventID != "E1" {
|
||||
t.Errorf("ns-a scope = %+v, want only E1", nsA)
|
||||
}
|
||||
nsB := NamespaceScope(events, "ns-b")
|
||||
if len(nsB) != 1 || nsB[0].EventID != "E2" {
|
||||
t.Errorf("ns-b scope = %+v, want only E2", nsB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandBraces(t *testing.T) {
|
||||
out := expandBraces("/etc/x.{service,timer}")
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("len = %d, want 2", len(out))
|
||||
}
|
||||
if out[0] != "/etc/x.service" || out[1] != "/etc/x.timer" {
|
||||
t.Errorf("out = %v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathGlobMatch(t *testing.T) {
|
||||
if !pathGlobMatch("/etc/orca/actual/*", "/etc/orca/actual/ns-a") {
|
||||
t.Errorf("expected glob match for single segment")
|
||||
}
|
||||
if !pathGlobMatch("/etc/orca/actual/*", "/etc/orca/actual/ns-a/x") {
|
||||
t.Errorf("expected glob match for nested segment (drift /* crosses /)")
|
||||
}
|
||||
if !pathGlobMatch("/etc/systemd/system/orca-collector.{service,timer}", "/etc/systemd/system/orca-collector.service") {
|
||||
t.Errorf("expected brace match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNsForPath(t *testing.T) {
|
||||
if got := nsForPath("/etc/orca/actual/ns-a/x"); got != "ns-a" {
|
||||
t.Errorf("nsForPath = %q, want ns-a", got)
|
||||
}
|
||||
if got := nsForPath("/etc/traefik/dynamic/orca.yml"); got != "" {
|
||||
t.Errorf("nsForPath = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// Package emitter: drift_path.go implements the systemd Path-unit
|
||||
// emitter for critical drift paths (P10b, v0.11, REQ-105, R-018).
|
||||
//
|
||||
// For each critical path two units are emitted:
|
||||
//
|
||||
// - <orca-drift-<name>.path>: PathChanged=<path>,
|
||||
// RateLimitIntervalSec=1s, RateLimitBurst=5 — systemd watches the
|
||||
// path for changes and triggers the matching .service.
|
||||
// - <orca-drift-<name>.service>: Type=oneshot,
|
||||
// ExecStart=/usr/local/bin/orca-drift-notify.sh %f, User=orca,
|
||||
// security-hardened (NoNewPrivileges=yes, ProtectSystem=strict,
|
||||
// ReadWritePaths=/etc/orca/state/drift-events, ProtectHome=yes).
|
||||
//
|
||||
// The emitter takes a list of critical PathSpecs (from drift.Config)
|
||||
// and produces the unit File artifacts. Unit names are derived from a
|
||||
// slug of the path pattern: non-alphanumerics collapse to '-' so the
|
||||
// name is systemd-friendly (no slashes, spaces, or brace chars).
|
||||
package emitter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DriftPathUnit is the systemd Path + service unit emitter for critical
|
||||
// drift paths (REQ-105).
|
||||
type DriftPathUnit struct{}
|
||||
|
||||
// driftEventsDir is the directory the oneshot services write to
|
||||
// (ReadWritePaths).
|
||||
const driftEventsDir = "/etc/orca/state/drift-events"
|
||||
|
||||
// driftNotifyBin is the oneshot ExecStart binary.
|
||||
const driftNotifyBin = "/usr/local/bin/orca-drift-notify.sh"
|
||||
|
||||
// RenderDriftUnits renders a pair of systemd units (a .path and a
|
||||
// .service) for each critical path in specs. Returns the File slice
|
||||
// ready for SSH-push to the peer.
|
||||
func (DriftPathUnit) RenderDriftUnits(specs []DriftPathSpec) ([]File, error) {
|
||||
if len(specs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
seen := make(map[string]bool, len(specs))
|
||||
var files []File
|
||||
for _, s := range specs {
|
||||
if s.Pattern == "" {
|
||||
return nil, fmt.Errorf("emitter/drift_path: empty pattern")
|
||||
}
|
||||
name := driftUnitName(s.Pattern)
|
||||
if seen[name] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
files = append(files, File{
|
||||
Path: fmt.Sprintf("/etc/systemd/system/%s.path", name),
|
||||
Content: renderDriftPathUnit(name, s.Pattern),
|
||||
Mode: "0644",
|
||||
})
|
||||
files = append(files, File{
|
||||
Path: fmt.Sprintf("/etc/systemd/system/%s.service", name),
|
||||
Content: renderDriftServiceUnit(name),
|
||||
Mode: "0644",
|
||||
})
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// DriftPathSpec is the minimal description the emitter needs: the
|
||||
// pattern to watch and whether systemd Path units are requested (when
|
||||
// false, no units are emitted — the path falls back to polling).
|
||||
type DriftPathSpec struct {
|
||||
Pattern string
|
||||
SystemdPathUnit bool
|
||||
}
|
||||
|
||||
// driftUnitName derives a systemd-friendly unit-name slug from a path
|
||||
// pattern. Non-alphanumeric runes collapse to '-'. The result carries
|
||||
// the orca-drift- prefix so the units are identifiable as orca-owned.
|
||||
func driftUnitName(pattern string) string {
|
||||
s := pattern
|
||||
s = strings.TrimPrefix(s, "/")
|
||||
var b strings.Builder
|
||||
prevDash := false
|
||||
for _, r := range s {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
prevDash = false
|
||||
} else {
|
||||
if !prevDash {
|
||||
b.WriteByte('-')
|
||||
prevDash = true
|
||||
}
|
||||
}
|
||||
}
|
||||
name := strings.Trim(b.String(), "-")
|
||||
if name == "" {
|
||||
name = "root"
|
||||
}
|
||||
return "orca-drift-" + name
|
||||
}
|
||||
|
||||
// renderDriftPathUnit renders the .path unit. PathChanged re-fires on
|
||||
// every modification (inotify IN_MODIFY), RateLimitIntervalSec +
|
||||
// RateLimitBurst bound the burst so a thrashing file does not spawn
|
||||
// thousands of services (R-018).
|
||||
func renderDriftPathUnit(name, path string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("[Unit]\n")
|
||||
b.WriteString(fmt.Sprintf("Description=orca drift watch for %s\n", path))
|
||||
b.WriteString("\n[Path]\n")
|
||||
b.WriteString(fmt.Sprintf("PathChanged=%s\n", path))
|
||||
b.WriteString("RateLimitIntervalSec=1s\n")
|
||||
b.WriteString("RateLimitBurst=5\n")
|
||||
b.WriteString("\n[Install]\n")
|
||||
b.WriteString("WantedBy=multi-user.target\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderDriftServiceUnit renders the oneshot .service triggered by
|
||||
// the .path unit. ExecStart receives the changed path via %f. Security
|
||||
// hardening runs the service as User=orca with NoNewPrivileges=yes,
|
||||
// ProtectSystem=strict (read-only /), and ReadWritePaths scoped to the
|
||||
// drift-events dir so the script can write its event JSON. ProtectHome
|
||||
// hides /root and /home (the orca user has no business there).
|
||||
func renderDriftServiceUnit(name string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("[Unit]\n")
|
||||
b.WriteString(fmt.Sprintf("Description=orca drift notify for %s\n", name))
|
||||
b.WriteString(fmt.Sprintf("After=%s.path\n", name))
|
||||
b.WriteString("\n[Service]\n")
|
||||
b.WriteString("Type=oneshot\n")
|
||||
b.WriteString(fmt.Sprintf("ExecStart=%s %%f\n", driftNotifyBin))
|
||||
b.WriteString("User=orca\n")
|
||||
b.WriteString("Group=orca\n")
|
||||
b.WriteString("NoNewPrivileges=yes\n")
|
||||
b.WriteString("ProtectSystem=strict\n")
|
||||
b.WriteString(fmt.Sprintf("ReadWritePaths=%s\n", driftEventsDir))
|
||||
b.WriteString("ProtectHome=yes\n")
|
||||
b.WriteString("PrivateTmp=yes\n")
|
||||
b.WriteString("ProtectKernelTunables=yes\n")
|
||||
b.WriteString("ProtectKernelModules=yes\n")
|
||||
b.WriteString("ProtectControlGroups=yes\n")
|
||||
b.WriteString("RestrictSUIDSGID=yes\n")
|
||||
b.WriteString("\n[Install]\n")
|
||||
b.WriteString("WantedBy=multi-user.target\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package emitter
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDriftPathUnit_RenderUnits(t *testing.T) {
|
||||
specs := []DriftPathSpec{
|
||||
{Pattern: "/etc/traefik/dynamic/orca.yml", SystemdPathUnit: true},
|
||||
{Pattern: "/etc/nftables.d/orca.nft", SystemdPathUnit: true},
|
||||
}
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits(specs)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderDriftUnits: %v", err)
|
||||
}
|
||||
if len(files) != 4 {
|
||||
t.Fatalf("got %d files, want 4 (2 .path + 2 .service)", len(files))
|
||||
}
|
||||
var pathUnits, svcUnits int
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f.Path, ".path") {
|
||||
pathUnits++
|
||||
}
|
||||
if strings.HasSuffix(f.Path, ".service") {
|
||||
svcUnits++
|
||||
}
|
||||
}
|
||||
if pathUnits != 2 || svcUnits != 2 {
|
||||
t.Errorf("path=%d service=%d, want 2 and 2", pathUnits, svcUnits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_PathUnitContent(t *testing.T) {
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/traefik/dynamic/orca.yml", SystemdPathUnit: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Render: %v", err)
|
||||
}
|
||||
var pathFile *File
|
||||
for i := range files {
|
||||
if strings.HasSuffix(files[i].Path, ".path") {
|
||||
pathFile = &files[i]
|
||||
}
|
||||
}
|
||||
if pathFile == nil {
|
||||
t.Fatal("no .path unit emitted")
|
||||
}
|
||||
c := pathFile.Content
|
||||
for _, want := range []string{
|
||||
"PathChanged=/etc/traefik/dynamic/orca.yml",
|
||||
"RateLimitIntervalSec=1s",
|
||||
"RateLimitBurst=5",
|
||||
"WantedBy=multi-user.target",
|
||||
} {
|
||||
if !strings.Contains(c, want) {
|
||||
t.Errorf("path unit missing %q:\n%s", want, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_ServiceUnitSecurity(t *testing.T) {
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/nftables.d/orca.nft", SystemdPathUnit: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Render: %v", err)
|
||||
}
|
||||
var svcFile *File
|
||||
for i := range files {
|
||||
if strings.HasSuffix(files[i].Path, ".service") {
|
||||
svcFile = &files[i]
|
||||
}
|
||||
}
|
||||
if svcFile == nil {
|
||||
t.Fatal("no .service unit emitted")
|
||||
}
|
||||
c := svcFile.Content
|
||||
for _, want := range []string{
|
||||
"Type=oneshot",
|
||||
"ExecStart=/usr/local/bin/orca-drift-notify.sh %f",
|
||||
"User=orca",
|
||||
"Group=orca",
|
||||
"NoNewPrivileges=yes",
|
||||
"ProtectSystem=strict",
|
||||
"ReadWritePaths=/etc/orca/state/drift-events",
|
||||
"ProtectHome=yes",
|
||||
} {
|
||||
if !strings.Contains(c, want) {
|
||||
t.Errorf("service unit missing %q:\n%s", want, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_UnitNameSlug(t *testing.T) {
|
||||
cases := []struct {
|
||||
pattern string
|
||||
want string
|
||||
}{
|
||||
{"/etc/traefik/dynamic/orca.yml", "orca-drift-etc-traefik-dynamic-orca-yml"},
|
||||
{"/etc/systemd/system/orca-alloc-*.service", "orca-drift-etc-systemd-system-orca-alloc-service"},
|
||||
{"/etc/orca/actual/*/etc/sudoers.d/orca-*", "orca-drift-etc-orca-actual-etc-sudoers-d-orca"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := driftUnitName(c.pattern)
|
||||
if got != c.want {
|
||||
t.Errorf("driftUnitName(%q) = %q, want %q", c.pattern, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_EmptyPatternErrors(t *testing.T) {
|
||||
_, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{{Pattern: ""}})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty pattern")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_NoSpecsNoFiles(t *testing.T) {
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderDriftUnits nil: %v", err)
|
||||
}
|
||||
if files != nil {
|
||||
t.Errorf("got %d files, want nil", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_DedupesSameSlug(t *testing.T) {
|
||||
// Two patterns that produce the same slug (after non-alnum collapse)
|
||||
// should dedupe to a single .path + .service pair.
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/x/a.yml", SystemdPathUnit: true},
|
||||
{Pattern: "/etc/x/a.yml", SystemdPathUnit: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Render: %v", err)
|
||||
}
|
||||
if len(files) != 2 {
|
||||
t.Errorf("got %d files, want 2 (deduped identical patterns)", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_ServiceAfterPath(t *testing.T) {
|
||||
files, _ := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/traefik/dynamic/orca.yml", SystemdPathUnit: true},
|
||||
})
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f.Path, ".service") {
|
||||
if !strings.Contains(f.Content, "After=orca-drift-etc-traefik-dynamic-orca-yml.path") {
|
||||
t.Errorf("service missing After=<name>.path:\n%s", f.Content)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("no service file found")
|
||||
}
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
# orca-drift-notify.sh — peer-side drift event recorder (P10b, REQ-106).
|
||||
#
|
||||
# Invoked by systemd Path units (orca-drift-<name>.service) when a
|
||||
# critical path changes. Receives the changed path as $1 (from systemd
|
||||
# %f). Computes sha256sum of the file (or "DELETED" if absent), reads
|
||||
# the latest applied txn from /etc/orca/state/latest-applied-txn, and
|
||||
# writes an event JSON to /etc/orca/state/drift-events/<event-id>.json.
|
||||
# Uses flock for serialization. R-001-clean: pure bash + sha256sum.
|
||||
#
|
||||
# Usage: orca-drift-notify.sh <path>
|
||||
#
|
||||
# Exit codes: 0 = event recorded; 1 = bad args / write failure.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=lib/orca-log.sh
|
||||
. "$SCRIPT_DIR/lib/orca-log.sh"
|
||||
|
||||
ORCA_LOG_ACTOR="spiffe://orca/cli/drift-notify"
|
||||
|
||||
STATE_DIR="${ORCA_STATE_DIR:-/etc/orca/state}"
|
||||
EVENTS_DIR="$STATE_DIR/drift-events"
|
||||
LATEST_TXN_FILE="$STATE_DIR/latest-applied-txn"
|
||||
LOCK_FILE="$STATE_DIR/drift-events.lock"
|
||||
|
||||
if [ "$#" -lt 1 ]; then
|
||||
orca_log_error "drift-notify" "-" "failed" "missing path argument"
|
||||
echo "usage: orca-drift-notify.sh <path>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PATH_ARG="$1"
|
||||
|
||||
mkdir -p "$EVENTS_DIR"
|
||||
|
||||
compute_sha() {
|
||||
local p="$1"
|
||||
if [ ! -e "$p" ]; then
|
||||
echo "DELETED"
|
||||
return
|
||||
fi
|
||||
sha256sum "$p" 2>/dev/null | awk '{print $1}' || echo "ERROR"
|
||||
}
|
||||
|
||||
read_latest_txn() {
|
||||
if [ -f "$LATEST_TXN_FILE" ]; then
|
||||
cat "$LATEST_TXN_FILE" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
gen_event_id() {
|
||||
local ts_us random_suffix
|
||||
ts_us="$(date -u +%Y%m%d%H%M%S%6N)"
|
||||
random_suffix="$(head -c 4 /dev/urandom 2>/dev/null | od -An -tx1 | tr -d ' \n' || echo "0000")"
|
||||
echo "EVT-${ts_us}-${random_suffix}"
|
||||
}
|
||||
|
||||
NEW_SHA="$(compute_sha "$PATH_ARG")"
|
||||
LATEST_TXN="$(read_latest_txn || true)"
|
||||
EVENT_ID="$(gen_event_id)"
|
||||
TS="$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)"
|
||||
HOST="$(hostname 2>/dev/null || echo unknown)"
|
||||
|
||||
if [ "$NEW_SHA" = "DELETED" ]; then
|
||||
STATUS="deleted"
|
||||
elif [ ! -f "$PATH_ARG" ]; then
|
||||
STATUS="created"
|
||||
else
|
||||
STATUS="modified"
|
||||
fi
|
||||
|
||||
escape_json() {
|
||||
local s="$1"
|
||||
s="${s//\\/\\\\}"
|
||||
s="${s//\"/\\\"}"
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
PATH_ESC="$(escape_json "$PATH_ARG")"
|
||||
HOST_ESC="$(escape_json "$HOST")"
|
||||
|
||||
EVENT_JSON=$(cat <<JSON
|
||||
{"event_id":"$EVENT_ID","ts":"$TS","host":"$HOST_ESC","path":"$PATH_ESC","status":"$STATUS","new_sha256":"$NEW_SHA","latest_txn":"$LATEST_TXN","drift_confirmed":false,"action":"reported","action_result":"skipped"}
|
||||
JSON
|
||||
)
|
||||
|
||||
EVENT_FILE="$EVENTS_DIR/${EVENT_ID}.json"
|
||||
|
||||
(
|
||||
flock 9 || { orca_log_error "drift-notify" "$PATH_ARG" "failed" "flock"; exit 1; }
|
||||
printf '%s\n' "$EVENT_JSON" >"$EVENT_FILE.tmp"
|
||||
mv "$EVENT_FILE.tmp" "$EVENT_FILE"
|
||||
) 9>"$LOCK_FILE"
|
||||
|
||||
orca_log_info "drift-notify" "$PATH_ARG" "ok" "event=$EVENT_ID status=$STATUS sha=$NEW_SHA"
|
||||
echo "event=$EVENT_ID"
|
||||
exit 0
|
||||
@@ -14,6 +14,7 @@
|
||||
# 3 = rollback failure
|
||||
# 4 = invalid arguments
|
||||
# 5 = already-applied no-op (re-run of a completed txn)
|
||||
# 6 = drift detected (R-020; override with --force)
|
||||
#
|
||||
# This script is invoked by the Go-side txn.Apply over SSH on the lead
|
||||
# peer. It wraps apply.sh / verify.sh / rollback.sh in the C-09 contract.
|
||||
@@ -31,6 +32,7 @@ EXIT_VERIFY_FAIL=2
|
||||
EXIT_ROLLBACK_FAIL=3
|
||||
EXIT_INVALID_ARGS=4
|
||||
EXIT_ALREADY_APPLIED=5
|
||||
EXIT_DRIFT_DETECTED=6
|
||||
|
||||
# --- bounded retry (C-09) ---
|
||||
MAX_RETRIES=3
|
||||
@@ -103,6 +105,29 @@ else
|
||||
:
|
||||
fi
|
||||
|
||||
# --- pre-flight drift gate (R-020, REQ-110, P10b-T7) ---
|
||||
# Before applying, check the lead-side aggregated drift state for the
|
||||
# target namespace. If unacknowledged drift is detected, refuse with
|
||||
# exit 6 (drift detected) unless --force is given. Per-namespace
|
||||
# scoping: a drifted peer in ns-A does NOT block ns-B.
|
||||
if [ "$FORCE" != "true" ]; then
|
||||
DRIFT_AGG_JSON="${ORCA_DRIFT_AGG_JSON:-/etc/orca/state/drift-events-aggregated.json}"
|
||||
if [ -f "$DRIFT_AGG_JSON" ]; then
|
||||
NS_FILTER="${NAMESPACE:-}"
|
||||
NS_REGEX="${NS_FILTER//\//.}"
|
||||
if [ -n "$NS_FILTER" ]; then
|
||||
DRIFT_HITS="$(grep -o '"path"[[:space:]]*:[[:space:]]*"[^"]*"' "$DRIFT_AGG_JSON" 2>/dev/null | sed 's/.*: *"//;s/"//' | grep -E "/etc/orca/actual/${NS_REGEX}/" | grep -v '"action"[[:space:]]*:[[:space:]]*"acknowledged"' || true)"
|
||||
else
|
||||
DRIFT_HITS="$(grep -o '"drift_confirmed"[[:space:]]*:[[:space:]]*true' "$DRIFT_AGG_JSON" 2>/dev/null || true)"
|
||||
fi
|
||||
if [ -n "$DRIFT_HITS" ]; then
|
||||
orca_log_error "orca-pull" "$TXN_DIR" "drift-detected" "namespace=${NAMESPACE:-cluster-wide}"
|
||||
echo "error: drift detected (exit 6); use --force to override or acknowledge the drift (R-020)" >&2
|
||||
exit "$EXIT_DRIFT_DETECTED"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- locate bundle files ---
|
||||
APPLY="$TXN_DIR/apply.sh"
|
||||
VERIFY="$TXN_DIR/verify.sh"
|
||||
|
||||
Executable
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env bash
|
||||
# orca-remediate.sh — lead-side drift remediator (P10b, REQ-108, C4).
|
||||
#
|
||||
# Re-pushes the latest applied txn's per-peer render tree via rsync and
|
||||
# runs the peer-side applier. Cooldown is 5 minutes per path and applies
|
||||
# ONLY on successful remediation (C4 refinement from CLARIFY). Transient
|
||||
# failures (SSH down, render tree missing) retry on the next aggregator
|
||||
# tick WITHOUT entering cooldown.
|
||||
#
|
||||
# Usage: orca-remediate.sh <peer> <txn-id> [path]
|
||||
#
|
||||
# Cooldown state: /etc/orca/state/remediation-cooldown/<path-hash>
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 = remediated (or cooldown skipped)
|
||||
# 1 = transient failure (no cooldown entered)
|
||||
# 2 = cooldown active (caller may log)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=lib/orca-log.sh
|
||||
. "$SCRIPT_DIR/lib/orca-log.sh"
|
||||
|
||||
ORCA_LOG_ACTOR="spiffe://orca/cli/remediate"
|
||||
|
||||
STATE_DIR="${ORCA_STATE_DIR:-/etc/orca/state}"
|
||||
APPLIED_DIR="${ORCA_APPLIED_DIR:-/etc/orca/state/applied}"
|
||||
COOLDOWN_DIR="$STATE_DIR/remediation-cooldown"
|
||||
COOLDOWN_SECONDS="${ORCA_REMEDIATE_COOLDOWN:-300}"
|
||||
|
||||
SSH_OPTS="${ORCA_SSH_OPTS:--o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5}"
|
||||
RSYNC_OPTS="${ORCA_RSYNC_OPTS:--a --quiet}"
|
||||
|
||||
if [ "$#" -lt 2 ]; then
|
||||
orca_log_error "remediate" "-" "failed" "usage: orca-remediate.sh <peer> <txn-id> [path]"
|
||||
echo "usage: orca-remediate.sh <peer> <txn-id> [path]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PEER="$1"
|
||||
TXN_ID="$2"
|
||||
DRIFT_PATH="${3:-}"
|
||||
|
||||
path_hash() {
|
||||
local p="${1:-root}"
|
||||
printf '%s' "$p" | sha256sum | awk '{print $1}'
|
||||
}
|
||||
|
||||
PATH_HASH="$(path_hash "$DRIFT_PATH")"
|
||||
COOLDOWN_FILE="$COOLDOWN_DIR/$PATH_HASH"
|
||||
|
||||
mkdir -p "$COOLDOWN_DIR"
|
||||
|
||||
check_cooldown() {
|
||||
if [ -z "$DRIFT_PATH" ]; then
|
||||
return 1
|
||||
fi
|
||||
if [ ! -f "$COOLDOWN_FILE" ]; then
|
||||
return 1
|
||||
fi
|
||||
local now ts age
|
||||
now="$(date +%s)"
|
||||
ts="$(stat -c %Y "$COOLDOWN_FILE" 2>/dev/null || echo 0)"
|
||||
age=$((now - ts))
|
||||
if [ "$age" -lt "$COOLDOWN_SECONDS" ]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
mark_cooldown() {
|
||||
if [ -z "$DRIFT_PATH" ]; then
|
||||
return 0
|
||||
fi
|
||||
date -u +%Y-%m-%dT%H:%M:%S.%3NZ >"$COOLDOWN_FILE" 2>/dev/null || true
|
||||
}
|
||||
|
||||
TXN_DIR="$APPLIED_DIR/$TXN_ID"
|
||||
|
||||
if [ ! -d "$TXN_DIR" ]; then
|
||||
orca_log_error "remediate" "$PEER" "transient" "txn dir missing: $TXN_DIR"
|
||||
echo "transient: txn dir missing: $TXN_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if check_cooldown; then
|
||||
orca_log_warn "remediate" "$PEER" "cooldown" "path=$DRIFT_PATH txn=$TXN_ID"
|
||||
echo "cooldown: path=$DRIFT_PATH txn=$TXN_ID" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
RSYNC_FAILED=0
|
||||
if rsync $RSYNC_OPTS "$TXN_DIR/" "$PEER:/run/orca/txns/$TXN_ID/" 2>/tmp/orca-remediate-rsync.err; then
|
||||
:
|
||||
else
|
||||
RSYNC_FAILED=1
|
||||
fi
|
||||
|
||||
if [ "$RSYNC_FAILED" -eq 1 ]; then
|
||||
orca_log_warn "remediate" "$PEER" "transient" "rsync failed for txn=$TXN_ID path=$DRIFT_PATH err=$(tr '\n' ' ' < /tmp/orca-remediate-rsync.err 2>/dev/null)"
|
||||
echo "transient: rsync failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PULL_CMD="bash /run/orca/txns/$TXN_ID/orca-pull.sh --txn-dir /run/orca/txns/$TXN_ID --namespace _defaults"
|
||||
if ! ssh $SSH_OPTS "$PEER" "$PULL_CMD" 2>/tmp/orca-remediate-pull.err; then
|
||||
PULL_ERR="$(tr '\n' ' ' < /tmp/orca-remediate-pull.err 2>/dev/null)"
|
||||
if echo "$PULL_ERR" | grep -q "already-applied"; then
|
||||
orca_log_info "remediate" "$PEER" "ok" "already-applied txn=$TXN_ID path=$DRIFT_PATH"
|
||||
mark_cooldown
|
||||
echo "already-applied"
|
||||
exit 0
|
||||
fi
|
||||
if echo "$PULL_ERR" | grep -Eq "connection refused|i/o timeout|no such host|connection reset|timeout|deadline exceeded|EOF"; then
|
||||
orca_log_warn "remediate" "$PEER" "transient" "pull failed (transient): $PULL_ERR"
|
||||
echo "transient: pull failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
orca_log_error "remediate" "$PEER" "failed" "pull failed: $PULL_ERR txn=$TXN_ID path=$DRIFT_PATH"
|
||||
echo "failed: pull failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mark_cooldown
|
||||
orca_log_info "remediate" "$PEER" "ok" "remediated txn=$TXN_ID path=$DRIFT_PATH"
|
||||
echo "remediated"
|
||||
exit 0
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bats
|
||||
# Tests for scripts/orca-drift-notify.sh, scripts/orca-remediate.sh,
|
||||
# and the NFS detection logic (P10b, REQ-106/REQ-108/REQ-112).
|
||||
|
||||
load test_helper
|
||||
|
||||
NOTIFY="$SCRIPTS_DIR/orca-drift-notify.sh"
|
||||
REMEDIATE="$SCRIPTS_DIR/orca-remediate.sh"
|
||||
|
||||
setup() {
|
||||
STATE_DIR="$(mktemp -d)"
|
||||
export ORCA_STATE_DIR="$STATE_DIR"
|
||||
mkdir -p "$STATE_DIR/drift-events"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
[ -n "$STATE_DIR" ] && rm -rf "$STATE_DIR"
|
||||
}
|
||||
|
||||
@test "orca-drift-notify.sh exists and is executable" {
|
||||
[ -f "$NOTIFY" ]
|
||||
[ -x "$NOTIFY" ]
|
||||
}
|
||||
|
||||
@test "orca-drift-notify.sh records modified event with sha256" {
|
||||
FILE="$STATE_DIR/test.txt"
|
||||
echo "hello world" >"$FILE"
|
||||
run "$NOTIFY" "$FILE"
|
||||
[ "$status" -eq 0 ]
|
||||
# Find the event JSON.
|
||||
EVENT_FILE="$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | head -1)"
|
||||
[ -n "$EVENT_FILE" ]
|
||||
[ -f "$EVENT_FILE" ]
|
||||
JSON="$(cat "$EVENT_FILE")"
|
||||
assert_json_field "$JSON" "event_id"
|
||||
assert_json_field "$JSON" "ts"
|
||||
assert_json_field "$JSON" "host"
|
||||
assert_json_field "$JSON" "path"
|
||||
assert_json_field "$JSON" "status"
|
||||
assert_json_field "$JSON" "new_sha256"
|
||||
assert_contains "$JSON" "modified"
|
||||
# The new_sha256 should NOT be "DELETED".
|
||||
assert_not_contains "$JSON" '"new_sha256":"DELETED"'
|
||||
}
|
||||
|
||||
@test "orca-drift-notify.sh records deleted event" {
|
||||
FILE="$STATE_DIR/missing.txt"
|
||||
run "$NOTIFY" "$FILE"
|
||||
[ "$status" -eq 0 ]
|
||||
EVENT_FILE="$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | head -1)"
|
||||
[ -n "$EVENT_FILE" ]
|
||||
JSON="$(cat "$EVENT_FILE")"
|
||||
assert_contains "$JSON" "deleted"
|
||||
assert_contains "$JSON" '"new_sha256":"DELETED"'
|
||||
}
|
||||
|
||||
@test "orca-drift-notify.sh records latest_txn when file exists" {
|
||||
echo "T-abcdef0123456789" >"$STATE_DIR/latest-applied-txn"
|
||||
FILE="$STATE_DIR/x.txt"
|
||||
echo "data" >"$FILE"
|
||||
run "$NOTIFY" "$FILE"
|
||||
[ "$status" -eq 0 ]
|
||||
EVENT_FILE="$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | head -1)"
|
||||
[ -n "$EVENT_FILE" ]
|
||||
JSON="$(cat "$EVENT_FILE")"
|
||||
assert_contains "$JSON" "T-abcdef0123456789"
|
||||
}
|
||||
|
||||
@test "orca-drift-notify.sh requires path argument" {
|
||||
run "$NOTIFY"
|
||||
[ "$status" -eq 1 ]
|
||||
assert_contains "$output" "usage"
|
||||
}
|
||||
|
||||
@test "orca-drift-notify.sh generates unique event IDs" {
|
||||
FILE="$STATE_DIR/a.txt"
|
||||
echo "x" >"$FILE"
|
||||
"$NOTIFY" "$FILE" >/dev/null
|
||||
"$NOTIFY" "$FILE" >/dev/null
|
||||
COUNT=$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | wc -l)
|
||||
[ "$COUNT" -eq 2 ]
|
||||
}
|
||||
|
||||
@test "orca-remediate.sh exists and is executable" {
|
||||
[ -f "$REMEDIATE" ]
|
||||
[ -x "$REMEDIATE" ]
|
||||
}
|
||||
|
||||
@test "orca-remediate.sh requires peer and txn args" {
|
||||
run "$REMEDIATE"
|
||||
[ "$status" -eq 1 ]
|
||||
assert_contains "$output" "usage"
|
||||
}
|
||||
|
||||
@test "orca-remediate.sh cooldown applies on success" {
|
||||
APPLIED_DIR="$STATE_DIR/applied"
|
||||
export ORCA_APPLIED_DIR="$APPLIED_DIR"
|
||||
TXN_DIR="$APPLIED_DIR/T-test-cooldown-0001"
|
||||
mkdir -p "$TXN_DIR"
|
||||
touch "$TXN_DIR/apply.sh"
|
||||
# Stub rsync + ssh to always succeed.
|
||||
mkdir -p "$STATE_DIR/bin"
|
||||
cat >"$STATE_DIR/bin/rsync" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
EOF
|
||||
cat >"$STATE_DIR/bin/ssh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo "applied"
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$STATE_DIR/bin/rsync" "$STATE_DIR/bin/ssh"
|
||||
export PATH="$STATE_DIR/bin:$PATH"
|
||||
# Set short cooldown for testing.
|
||||
export ORCA_REMEDIATE_COOLDOWN=60
|
||||
run "$REMEDIATE" "peer1" "T-test-cooldown-0001" "/etc/traefik/dynamic/orca.yml"
|
||||
[ "$status" -eq 0 ]
|
||||
# Cooldown file should exist.
|
||||
COOLDOWN_FILE="$STATE_DIR/remediation-cooldown/$(printf '%s' "/etc/traefik/dynamic/orca.yml" | sha256sum | awk '{print $1}')"
|
||||
[ -f "$COOLDOWN_FILE" ]
|
||||
}
|
||||
|
||||
@test "orca-remediate.sh transient failure (missing txn dir) does NOT enter cooldown" {
|
||||
# No txn dir created -> transient failure.
|
||||
run "$REMEDIATE" "peer1" "T-nonexistent" "/etc/p"
|
||||
[ "$status" -eq 1 ]
|
||||
COOLDOWN_FILE="$STATE_DIR/remediation-cooldown/$(printf '%s' "/etc/p" | sha256sum | awk '{print $1}')"
|
||||
[ ! -f "$COOLDOWN_FILE" ]
|
||||
}
|
||||
|
||||
@test "NFS detection: stat -f -c %T output is parsed" {
|
||||
# We cannot mount NFS in CI, but we can test that the detectNFS
|
||||
# logic is invoked by orca-drift-notify's peer setup. This test
|
||||
# documents the contract: ext4 / xfs / btrfs -> not NFS; nfs* -> NFS.
|
||||
for fs in ext4 xfs btrfs tmpfs; do
|
||||
[ "$(is_nfs "$fs")" = "false" ] || {
|
||||
echo "expected $fs to NOT be nfs"
|
||||
return 1
|
||||
}
|
||||
done
|
||||
for fs in nfs nfs4; do
|
||||
[ "$(is_nfs "$fs")" = "true" ] || {
|
||||
echo "expected $fs to BE nfs"
|
||||
return 1
|
||||
}
|
||||
done
|
||||
}
|
||||
|
||||
# is_nfs mirrors the bash-side detectNFS contract from peer_setup.go.
|
||||
is_nfs() {
|
||||
case "$1" in
|
||||
*nfs*) echo "true" ;;
|
||||
*) echo "false" ;;
|
||||
esac
|
||||
}
|
||||
@@ -110,3 +110,58 @@ EOF
|
||||
run "$PULL" --txn-dir "$TMP_TXN" --namespace default
|
||||
assert_status 4 "$status"
|
||||
}
|
||||
|
||||
@test "orca-pull.sh drift gate refuses with exit 6 when drift detected (R-020)" {
|
||||
# Create a drift-events-aggregated.json with confirmed drift.
|
||||
mkdir -p "$TMP_TXN/state"
|
||||
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
|
||||
cat >"$DRIFT_JSON" <<EOF
|
||||
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/orca/actual/default/x","status":"modified","drift_confirmed":true}]}
|
||||
EOF
|
||||
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace default
|
||||
assert_status 6 "$status"
|
||||
assert_contains "$output" "drift detected"
|
||||
}
|
||||
|
||||
@test "orca-pull.sh drift gate passes when no drift" {
|
||||
# Empty aggregated doc -> no drift -> applies.
|
||||
mkdir -p "$TMP_TXN/state"
|
||||
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
|
||||
echo '{"ts":"2026-01-01T00:00:00Z","events":[]}' >"$DRIFT_JSON"
|
||||
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace default
|
||||
assert_status 0 "$status"
|
||||
}
|
||||
|
||||
@test "orca-pull.sh drift gate --force overrides (R-020)" {
|
||||
# Create drift, then apply with --force --i-understand-the-risk.
|
||||
mkdir -p "$TMP_TXN/state"
|
||||
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
|
||||
cat >"$DRIFT_JSON" <<EOF
|
||||
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/orca/actual/default/x","status":"modified","drift_confirmed":true}]}
|
||||
EOF
|
||||
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace default --force --i-understand-the-risk
|
||||
assert_status 0 "$status"
|
||||
}
|
||||
|
||||
@test "orca-pull.sh drift gate per-namespace scoping (ns-A drift does not block ns-B)" {
|
||||
# Drift in ns-A; apply to ns-B should succeed.
|
||||
mkdir -p "$TMP_TXN/state"
|
||||
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
|
||||
cat >"$DRIFT_JSON" <<EOF
|
||||
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/orca/actual/ns-a/x","status":"modified","drift_confirmed":true}]}
|
||||
EOF
|
||||
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace ns-b
|
||||
assert_status 0 "$status"
|
||||
}
|
||||
|
||||
@test "orca-pull.sh drift gate cluster-wide checks all confirmed drift" {
|
||||
# No --namespace; cluster-wide requires --force. When --force given
|
||||
# AND drift present, gate is skipped (force bypasses drift too).
|
||||
mkdir -p "$TMP_TXN/state"
|
||||
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
|
||||
cat >"$DRIFT_JSON" <<EOF
|
||||
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}
|
||||
EOF
|
||||
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --force --i-understand-the-risk
|
||||
assert_status 0 "$status"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user