Merge pull request 'feature/resetdata-docker-secrets-integration' (#10) from feature/resetdata-docker-secrets-integration into main

Reviewed-on: #10

Dockerfile.simple

@@ -15,14 +15,16 @@ RUN addgroup -g 1000 chorus && \
 RUN mkdir -p /app/data && \
     chown -R chorus:chorus /app
 
-# Copy pre-built binary
+# Copy pre-built binary from build directory (ensure it exists and is the correct one)
-COPY chorus-agent /app/chorus-agent
+COPY build/chorus-agent /app/chorus-agent
 RUN chmod +x /app/chorus-agent && chown chorus:chorus /app/chorus-agent
 
 # Switch to non-root user
 USER chorus
 WORKDIR /app
 
+# Note: Using correct chorus-agent binary built with 'make build-agent'
+
 # Expose ports
 EXPOSE 8080 8081 9000
 

docker/Dockerfile

@@ -11,15 +11,15 @@ WORKDIR /build
 # Copy go mod files first (for better caching)
 COPY go.mod go.sum ./
 
-# Copy vendor directory for local dependencies
+# Download dependencies
-COPY vendor/ vendor/
+RUN go mod download
 
 # Copy source code
 COPY . .
 
-# Build the CHORUS binary with vendor mode
+# Build the CHORUS binary with mod mode
 RUN CGO_ENABLED=0 GOOS=linux go build \
-    -mod=vendor \
+    -mod=mod \
     -ldflags='-w -s -extldflags "-static"' \
     -o chorus \
     ./cmd/chorus

docker/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   chorus:
-    image: anthonyrawlins/chorus:resetdata-secrets-v1.0.5
+    image: anthonyrawlins/chorus:discovery-debug
     
     # REQUIRED: License configuration (CHORUS will not start without this)
     environment:
@@ -15,7 +15,7 @@ services:
       - CHORUS_AGENT_ID=${CHORUS_AGENT_ID:-}  # Auto-generated if not provided
       - CHORUS_SPECIALIZATION=${CHORUS_SPECIALIZATION:-general_developer}
       - CHORUS_MAX_TASKS=${CHORUS_MAX_TASKS:-3}
-      - CHORUS_CAPABILITIES=${CHORUS_CAPABILITIES:-general_development,task_coordination}
+      - CHORUS_CAPABILITIES=${CHORUS_CAPABILITIES:-general_development,task_coordination,admin_election}
       
       # Network configuration
       - CHORUS_API_PORT=8080
@@ -71,7 +71,7 @@ services:
     # Container resource limits
     deploy:
       mode: replicated
-      replicas: ${CHORUS_REPLICAS:-1}
+      replicas: ${CHORUS_REPLICAS:-9}
       update_config:
         parallelism: 1
         delay: 10s
@@ -212,11 +212,14 @@ services:
           cpus: '0.25'
       labels:
         - traefik.enable=true
+        - traefik.docker.network=tengig
         - traefik.http.routers.whoosh.rule=Host(`whoosh.chorus.services`)
         - traefik.http.routers.whoosh.tls=true
-        - traefik.http.routers.whoosh.tls.certresolver=letsencrypt
+        - traefik.http.routers.whoosh.tls.certresolver=letsencryptresolver
+        - traefik.http.routers.photoprism.entrypoints=web,web-secured
         - traefik.http.services.whoosh.loadbalancer.server.port=8080
-        - traefik.http.middlewares.whoosh-auth.basicauth.users=admin:$$2y$$10$$example_hash
+        - traefik.http.services.photoprism.loadbalancer.passhostheader=true
+        - traefik.http.middlewares.whoosh-auth.basicauth.users=admin:$2y$10$example_hash
     networks:
       - tengig
       - whoosh-backend
@@ -310,6 +313,72 @@ services:
 
 
 
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    volumes:
+      - /rust/containers/CHORUS/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - /rust/containers/CHORUS/monitoring/prometheus:/prometheus
+    ports:
+      - "9099:9090" # Expose Prometheus UI
+    deploy:
+      replicas: 1
+      placement:
+        constraints:
+          - node.hostname != rosewood
+      labels:
+        - traefik.enable=true
+        - traefik.http.routers.prometheus.rule=Host(`prometheus.chorus.services`)
+        - traefik.http.routers.prometheus.entrypoints=web,web-secured
+        - traefik.http.routers.prometheus.tls=true
+        - traefik.http.routers.prometheus.tls.certresolver=letsencryptresolver
+        - traefik.http.services.prometheus.loadbalancer.server.port=9090
+    networks:
+      - chorus_net
+      - tengig
+    healthcheck:
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9090/-/ready"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  grafana:
+    image: grafana/grafana:latest
+    user: "1000:1000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} # Use a strong password in production
+      - GF_SERVER_ROOT_URL=https://grafana.chorus.services
+    volumes:
+      - /rust/containers/CHORUS/monitoring/grafana:/var/lib/grafana
+    ports:
+      - "3300:3000" # Expose Grafana UI
+    deploy:
+      replicas: 1
+      placement:
+        constraints:
+          - node.hostname != rosewood
+      labels:
+        - traefik.enable=true
+        - traefik.http.routers.grafana.rule=Host(`grafana.chorus.services`)
+        - traefik.http.routers.grafana.entrypoints=web,web-secured
+        - traefik.http.routers.grafana.tls=true
+        - traefik.http.routers.grafana.tls.certresolver=letsencryptresolver
+        - traefik.http.services.grafana.loadbalancer.server.port=3000
+    networks:
+      - chorus_net
+      - tengig
+    healthcheck:
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/api/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
   # BACKBEAT Pulse Service - Leader-elected tempo broadcaster
   # REQ: BACKBEAT-REQ-001 - Single BeatFrame publisher per cluster
   # REQ: BACKBEAT-OPS-001 - One replica prefers leadership
@@ -495,6 +564,24 @@ services:
 
 # Persistent volumes
 volumes:
+  prometheus_data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: /rust/containers/CHORUS/monitoring/prometheus
+  prometheus_config:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: /rust/containers/CHORUS/monitoring/prometheus
+  grafana_data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: /rust/containers/CHORUS/monitoring/grafana
   chorus_data:
     driver: local
   whoosh_postgres_data:

go.mod

@@ -21,9 +21,11 @@ require (
 	github.com/prometheus/client_golang v1.19.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sashabaranov/go-openai v1.41.1
+	github.com/sony/gobreaker v0.5.0
 	github.com/stretchr/testify v1.10.0
 	github.com/syndtr/goleveldb v1.0.0
 	golang.org/x/crypto v0.24.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -155,7 +157,6 @@ require (
 	golang.org/x/tools v0.22.0 // indirect
 	gonum.org/v1/gonum v0.13.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
 )
 

go.sum

@@ -437,6 +437,8 @@ github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N
 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
 github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=

internal/licensing/license_gate.go

@@ -0,0 +1,340 @@
+package licensing
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/sony/gobreaker"
+)
+
+// LicenseGate provides burst-proof license validation with caching and circuit breaker
+type LicenseGate struct {
+	config      LicenseConfig
+	cache       atomic.Value // stores cachedLease
+	breaker     *gobreaker.CircuitBreaker
+	graceUntil  atomic.Value // stores time.Time
+	httpClient  *http.Client
+}
+
+// cachedLease represents a cached license lease with expiry
+type cachedLease struct {
+	LeaseToken string    `json:"lease_token"`
+	ExpiresAt  time.Time `json:"expires_at"`
+	ClusterID  string    `json:"cluster_id"`
+	Valid      bool      `json:"valid"`
+	CachedAt   time.Time `json:"cached_at"`
+}
+
+// LeaseRequest represents a cluster lease request
+type LeaseRequest struct {
+	ClusterID         string `json:"cluster_id"`
+	RequestedReplicas int    `json:"requested_replicas"`
+	DurationMinutes   int    `json:"duration_minutes"`
+}
+
+// LeaseResponse represents a cluster lease response
+type LeaseResponse struct {
+	LeaseToken   string    `json:"lease_token"`
+	MaxReplicas  int       `json:"max_replicas"`
+	ExpiresAt    time.Time `json:"expires_at"`
+	ClusterID    string    `json:"cluster_id"`
+	LeaseID      string    `json:"lease_id"`
+}
+
+// LeaseValidationRequest represents a lease validation request
+type LeaseValidationRequest struct {
+	LeaseToken string `json:"lease_token"`
+	ClusterID  string `json:"cluster_id"`
+	AgentID    string `json:"agent_id"`
+}
+
+// LeaseValidationResponse represents a lease validation response
+type LeaseValidationResponse struct {
+	Valid             bool      `json:"valid"`
+	RemainingReplicas int       `json:"remaining_replicas"`
+	ExpiresAt         time.Time `json:"expires_at"`
+}
+
+// NewLicenseGate creates a new license gate with circuit breaker and caching
+func NewLicenseGate(config LicenseConfig) *LicenseGate {
+	// Circuit breaker settings optimized for license validation
+	breakerSettings := gobreaker.Settings{
+		Name:        "license-validation",
+		MaxRequests: 3,  // Allow 3 requests in half-open state
+		Interval:    60 * time.Second, // Reset failure count every minute
+		Timeout:     30 * time.Second, // Stay open for 30 seconds
+		ReadyToTrip: func(counts gobreaker.Counts) bool {
+			// Trip after 3 consecutive failures
+			return counts.ConsecutiveFailures >= 3
+		},
+		OnStateChange: func(name string, from gobreaker.State, to gobreaker.State) {
+			fmt.Printf("🔌 License validation circuit breaker: %s -> %s\n", from, to)
+		},
+	}
+
+	gate := &LicenseGate{
+		config:     config,
+		breaker:    gobreaker.NewCircuitBreaker(breakerSettings),
+		httpClient: &http.Client{Timeout: 10 * time.Second},
+	}
+
+	// Initialize grace period
+	gate.graceUntil.Store(time.Now().Add(90 * time.Second))
+
+	return gate
+}
+
+// ValidNow checks if the cached lease is currently valid
+func (c *cachedLease) ValidNow() bool {
+	if !c.Valid {
+		return false
+	}
+	// Consider lease invalid 2 minutes before actual expiry for safety margin
+	return time.Now().Before(c.ExpiresAt.Add(-2 * time.Minute))
+}
+
+// loadCachedLease safely loads the cached lease
+func (g *LicenseGate) loadCachedLease() *cachedLease {
+	if cached := g.cache.Load(); cached != nil {
+		if lease, ok := cached.(*cachedLease); ok {
+			return lease
+		}
+	}
+	return &cachedLease{Valid: false}
+}
+
+// storeLease safely stores a lease in the cache
+func (g *LicenseGate) storeLease(lease *cachedLease) {
+	lease.CachedAt = time.Now()
+	g.cache.Store(lease)
+}
+
+// isInGracePeriod checks if we're still in the grace period
+func (g *LicenseGate) isInGracePeriod() bool {
+	if graceUntil := g.graceUntil.Load(); graceUntil != nil {
+		if grace, ok := graceUntil.(time.Time); ok {
+			return time.Now().Before(grace)
+		}
+	}
+	return false
+}
+
+// extendGracePeriod extends the grace period on successful validation
+func (g *LicenseGate) extendGracePeriod() {
+	g.graceUntil.Store(time.Now().Add(90 * time.Second))
+}
+
+// Validate validates the license using cache, lease system, and circuit breaker
+func (g *LicenseGate) Validate(ctx context.Context, agentID string) error {
+	// Check cached lease first
+	if lease := g.loadCachedLease(); lease.ValidNow() {
+		return g.validateCachedLease(ctx, lease, agentID)
+	}
+
+	// Try to get/renew lease through circuit breaker
+	_, err := g.breaker.Execute(func() (interface{}, error) {
+		lease, err := g.requestOrRenewLease(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		// Validate the new lease
+		if err := g.validateLease(ctx, lease, agentID); err != nil {
+			return nil, err
+		}
+
+		// Store successful lease
+		g.storeLease(&cachedLease{
+			LeaseToken: lease.LeaseToken,
+			ExpiresAt:  lease.ExpiresAt,
+			ClusterID:  lease.ClusterID,
+			Valid:      true,
+		})
+
+		return nil, nil
+	})
+
+	if err != nil {
+		// If we're in grace period, allow startup but log warning
+		if g.isInGracePeriod() {
+			fmt.Printf("⚠️ License validation failed but in grace period: %v\n", err)
+			return nil
+		}
+		return fmt.Errorf("license validation failed: %w", err)
+	}
+
+	// Extend grace period on successful validation
+	g.extendGracePeriod()
+	return nil
+}
+
+// validateCachedLease validates using cached lease token
+func (g *LicenseGate) validateCachedLease(ctx context.Context, lease *cachedLease, agentID string) error {
+	validation := LeaseValidationRequest{
+		LeaseToken: lease.LeaseToken,
+		ClusterID:  g.config.ClusterID,
+		AgentID:    agentID,
+	}
+
+	url := fmt.Sprintf("%s/api/v1/licenses/validate-lease", strings.TrimSuffix(g.config.KachingURL, "/"))
+
+	reqBody, err := json.Marshal(validation)
+	if err != nil {
+		return fmt.Errorf("failed to marshal lease validation request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(reqBody)))
+	if err != nil {
+		return fmt.Errorf("failed to create lease validation request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := g.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("lease validation request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		// If validation fails, invalidate cache
+		lease.Valid = false
+		g.storeLease(lease)
+		return fmt.Errorf("lease validation failed with status %d", resp.StatusCode)
+	}
+
+	var validationResp LeaseValidationResponse
+	if err := json.NewDecoder(resp.Body).Decode(&validationResp); err != nil {
+		return fmt.Errorf("failed to decode lease validation response: %w", err)
+	}
+
+	if !validationResp.Valid {
+		// If validation fails, invalidate cache
+		lease.Valid = false
+		g.storeLease(lease)
+		return fmt.Errorf("lease token is invalid")
+	}
+
+	return nil
+}
+
+// requestOrRenewLease requests a new cluster lease or renews existing one
+func (g *LicenseGate) requestOrRenewLease(ctx context.Context) (*LeaseResponse, error) {
+	// For now, request a new lease (TODO: implement renewal logic)
+	leaseReq := LeaseRequest{
+		ClusterID:         g.config.ClusterID,
+		RequestedReplicas: 1, // Start with single replica
+		DurationMinutes:   60, // 1 hour lease
+	}
+
+	url := fmt.Sprintf("%s/api/v1/licenses/%s/cluster-lease",
+		strings.TrimSuffix(g.config.KachingURL, "/"), g.config.LicenseID)
+
+	reqBody, err := json.Marshal(leaseReq)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal lease request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(reqBody)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create lease request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := g.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("lease request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusTooManyRequests {
+		return nil, fmt.Errorf("rate limited by KACHING, retry after: %s", resp.Header.Get("Retry-After"))
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("lease request failed with status %d", resp.StatusCode)
+	}
+
+	var leaseResp LeaseResponse
+	if err := json.NewDecoder(resp.Body).Decode(&leaseResp); err != nil {
+		return nil, fmt.Errorf("failed to decode lease response: %w", err)
+	}
+
+	return &leaseResp, nil
+}
+
+// validateLease validates a lease token
+func (g *LicenseGate) validateLease(ctx context.Context, lease *LeaseResponse, agentID string) error {
+	validation := LeaseValidationRequest{
+		LeaseToken: lease.LeaseToken,
+		ClusterID:  lease.ClusterID,
+		AgentID:    agentID,
+	}
+
+	return g.validateLeaseRequest(ctx, validation)
+}
+
+// validateLeaseRequest performs the actual lease validation HTTP request
+func (g *LicenseGate) validateLeaseRequest(ctx context.Context, validation LeaseValidationRequest) error {
+	url := fmt.Sprintf("%s/api/v1/licenses/validate-lease", strings.TrimSuffix(g.config.KachingURL, "/"))
+
+	reqBody, err := json.Marshal(validation)
+	if err != nil {
+		return fmt.Errorf("failed to marshal lease validation request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(reqBody)))
+	if err != nil {
+		return fmt.Errorf("failed to create lease validation request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := g.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("lease validation request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("lease validation failed with status %d", resp.StatusCode)
+	}
+
+	var validationResp LeaseValidationResponse
+	if err := json.NewDecoder(resp.Body).Decode(&validationResp); err != nil {
+		return fmt.Errorf("failed to decode lease validation response: %w", err)
+	}
+
+	if !validationResp.Valid {
+		return fmt.Errorf("lease token is invalid")
+	}
+
+	return nil
+}
+
+// GetCacheStats returns cache statistics for monitoring
+func (g *LicenseGate) GetCacheStats() map[string]interface{} {
+	lease := g.loadCachedLease()
+	stats := map[string]interface{}{
+		"cache_valid":     lease.Valid,
+		"cache_hit":       lease.ValidNow(),
+		"expires_at":      lease.ExpiresAt,
+		"cached_at":       lease.CachedAt,
+		"in_grace_period": g.isInGracePeriod(),
+		"breaker_state":   g.breaker.State().String(),
+	}
+
+	if grace := g.graceUntil.Load(); grace != nil {
+		if graceTime, ok := grace.(time.Time); ok {
+			stats["grace_until"] = graceTime
+		}
+	}
+
+	return stats
+}

internal/licensing/validator.go

@@ -2,6 +2,7 @@ package licensing
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -21,35 +22,60 @@ type LicenseConfig struct {
 }
 
 // Validator handles license validation with KACHING
+// Enhanced with license gate for burst-proof validation
 type Validator struct {
 	config     LicenseConfig
 	kachingURL string
 	client     *http.Client
+	gate       *LicenseGate  // New: License gate for scaling support
 }
 
-// NewValidator creates a new license validator
+// NewValidator creates a new license validator with enhanced scaling support
 func NewValidator(config LicenseConfig) *Validator {
 	kachingURL := config.KachingURL
 	if kachingURL == "" {
 		kachingURL = DefaultKachingURL
 	}
 
-	return &Validator{
+	validator := &Validator{
 		config:     config,
 		kachingURL: kachingURL,
 		client: &http.Client{
 			Timeout: LicenseTimeout,
 		},
 	}
+
+	// Initialize license gate for scaling support
+	validator.gate = NewLicenseGate(config)
+
+	return validator
 }
 
 // Validate performs license validation with KACHING license authority
-// CRITICAL: CHORUS will not start without valid license validation
+// Enhanced with caching, circuit breaker, and lease token support
 func (v *Validator) Validate() error {
+	return v.ValidateWithContext(context.Background())
+}
+
+// ValidateWithContext performs license validation with context and agent ID
+func (v *Validator) ValidateWithContext(ctx context.Context) error {
 	if v.config.LicenseID == "" || v.config.ClusterID == "" {
 		return fmt.Errorf("license ID and cluster ID are required")
 	}
 
+	// Use enhanced license gate for validation
+	agentID := "default-agent" // TODO: Get from config/environment
+	if err := v.gate.Validate(ctx, agentID); err != nil {
+		// Fallback to legacy validation for backward compatibility
+		fmt.Printf("⚠️ License gate validation failed, trying legacy validation: %v\n", err)
+		return v.validateLegacy()
+	}
+
+	return nil
+}
+
+// validateLegacy performs the original license validation (for fallback)
+func (v *Validator) validateLegacy() error {
 	// Prepare validation request
 	request := map[string]interface{}{
 		"license_id": v.config.LicenseID,

p2p/config.go

@@ -20,10 +20,16 @@ type Config struct {
 	DHTMode          string // "client", "server", "auto"
 	DHTProtocolPrefix string
 
-	// Connection limits
+	// Connection limits and rate limiting
-	MaxConnections    int
+	MaxConnections      int
-	MaxPeersPerIP     int
+	MaxPeersPerIP       int
-	ConnectionTimeout time.Duration
+	ConnectionTimeout   time.Duration
+	LowWatermark        int           // Connection manager low watermark
+	HighWatermark       int           // Connection manager high watermark
+	DialsPerSecond      int           // Dial rate limiting
+	MaxConcurrentDials  int           // Maximum concurrent outbound dials
+	MaxConcurrentDHT    int           // Maximum concurrent DHT queries
+	JoinStaggerMS       int           // Join stagger delay in milliseconds
 
 	// Security configuration
 	EnableSecurity bool
@@ -48,8 +54,8 @@ func DefaultConfig() *Config {
 		},
 		NetworkID: "CHORUS-network",
 
-		// Discovery settings
+		// Discovery settings - mDNS disabled for Swarm by default
-		EnableMDNS:     true,
+		EnableMDNS:     false, // Disabled for container environments
 		MDNSServiceTag: "CHORUS-peer-discovery",
 
 		// DHT settings (disabled by default for local development)
@@ -58,10 +64,16 @@ func DefaultConfig() *Config {
 		DHTMode:          "auto",
 		DHTProtocolPrefix: "/CHORUS",
 
-		// Connection limits for local network
+		// Connection limits and rate limiting for scaling
-		MaxConnections:    50,
+		MaxConnections:      50,
-		MaxPeersPerIP:     3,
+		MaxPeersPerIP:       3,
-		ConnectionTimeout: 30 * time.Second,
+		ConnectionTimeout:   30 * time.Second,
+		LowWatermark:        32,  // Keep at least 32 connections
+		HighWatermark:       128, // Trim above 128 connections
+		DialsPerSecond:      5,   // Limit outbound dials to prevent storms
+		MaxConcurrentDials:  10,  // Maximum concurrent outbound dials
+		MaxConcurrentDHT:    16,  // Maximum concurrent DHT queries
+		JoinStaggerMS:       0,   // No stagger by default (set by assignment)
 
 		// Security enabled by default
 		EnableSecurity: true,
@@ -165,3 +177,33 @@ func WithDHTProtocolPrefix(prefix string) Option {
 		c.DHTProtocolPrefix = prefix
 	}
 }
+
+// WithConnectionManager sets connection manager watermarks
+func WithConnectionManager(low, high int) Option {
+	return func(c *Config) {
+		c.LowWatermark = low
+		c.HighWatermark = high
+	}
+}
+
+// WithDialRateLimit sets the dial rate limiting
+func WithDialRateLimit(dialsPerSecond, maxConcurrent int) Option {
+	return func(c *Config) {
+		c.DialsPerSecond = dialsPerSecond
+		c.MaxConcurrentDials = maxConcurrent
+	}
+}
+
+// WithDHTRateLimit sets the DHT query rate limiting
+func WithDHTRateLimit(maxConcurrentDHT int) Option {
+	return func(c *Config) {
+		c.MaxConcurrentDHT = maxConcurrentDHT
+	}
+}
+
+// WithJoinStagger sets the join stagger delay in milliseconds
+func WithJoinStagger(delayMS int) Option {
+	return func(c *Config) {
+		c.JoinStaggerMS = delayMS
+	}
+}

pkg/bootstrap/pool_manager.go

@@ -0,0 +1,353 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/libp2p/go-libp2p/core/host"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/multiformats/go-multiaddr"
+)
+
+// BootstrapPool manages a pool of bootstrap peers for DHT joining
+type BootstrapPool struct {
+	peers           []peer.AddrInfo
+	dialsPerSecond  int
+	maxConcurrent   int
+	staggerDelay    time.Duration
+	httpClient      *http.Client
+}
+
+// BootstrapConfig represents the JSON configuration for bootstrap peers
+type BootstrapConfig struct {
+	Peers []BootstrapPeer `json:"peers"`
+	Meta  BootstrapMeta   `json:"meta,omitempty"`
+}
+
+// BootstrapPeer represents a single bootstrap peer
+type BootstrapPeer struct {
+	ID        string   `json:"id"`         // Peer ID
+	Addresses []string `json:"addresses"`  // Multiaddresses
+	Priority  int      `json:"priority"`   // Priority (higher = more likely to be selected)
+	Healthy   bool     `json:"healthy"`    // Health status
+	LastSeen  string   `json:"last_seen"`  // Last seen timestamp
+}
+
+// BootstrapMeta contains metadata about the bootstrap configuration
+type BootstrapMeta struct {
+	UpdatedAt    string `json:"updated_at"`
+	Version      int    `json:"version"`
+	ClusterID    string `json:"cluster_id"`
+	TotalPeers   int    `json:"total_peers"`
+	HealthyPeers int    `json:"healthy_peers"`
+}
+
+// BootstrapSubset represents a subset of peers assigned to a replica
+type BootstrapSubset struct {
+	Peers        []peer.AddrInfo `json:"peers"`
+	StaggerDelayMS int           `json:"stagger_delay_ms"`
+	AssignedAt   time.Time       `json:"assigned_at"`
+}
+
+// NewBootstrapPool creates a new bootstrap pool manager
+func NewBootstrapPool(dialsPerSecond, maxConcurrent int, staggerMS int) *BootstrapPool {
+	return &BootstrapPool{
+		peers:          []peer.AddrInfo{},
+		dialsPerSecond: dialsPerSecond,
+		maxConcurrent:  maxConcurrent,
+		staggerDelay:   time.Duration(staggerMS) * time.Millisecond,
+		httpClient:     &http.Client{Timeout: 10 * time.Second},
+	}
+}
+
+// LoadFromFile loads bootstrap configuration from a JSON file
+func (bp *BootstrapPool) LoadFromFile(filePath string) error {
+	if filePath == "" {
+		return nil // No file configured
+	}
+
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to read bootstrap file %s: %w", filePath, err)
+	}
+
+	return bp.loadFromJSON(data)
+}
+
+// LoadFromURL loads bootstrap configuration from a URL (WHOOSH endpoint)
+func (bp *BootstrapPool) LoadFromURL(ctx context.Context, url string) error {
+	if url == "" {
+		return nil // No URL configured
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create bootstrap request: %w", err)
+	}
+
+	resp, err := bp.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("bootstrap request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bootstrap request failed with status %d", resp.StatusCode)
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read bootstrap response: %w", err)
+	}
+
+	return bp.loadFromJSON(data)
+}
+
+// loadFromJSON parses JSON bootstrap configuration
+func (bp *BootstrapPool) loadFromJSON(data []byte) error {
+	var config BootstrapConfig
+	if err := json.Unmarshal(data, &config); err != nil {
+		return fmt.Errorf("failed to parse bootstrap JSON: %w", err)
+	}
+
+	// Convert bootstrap peers to AddrInfo
+	var peers []peer.AddrInfo
+	for _, bsPeer := range config.Peers {
+		// Only include healthy peers
+		if !bsPeer.Healthy {
+			continue
+		}
+
+		// Parse peer ID
+		peerID, err := peer.Decode(bsPeer.ID)
+		if err != nil {
+			fmt.Printf("⚠️ Invalid peer ID %s: %v\n", bsPeer.ID, err)
+			continue
+		}
+
+		// Parse multiaddresses
+		var addrs []multiaddr.Multiaddr
+		for _, addrStr := range bsPeer.Addresses {
+			addr, err := multiaddr.NewMultiaddr(addrStr)
+			if err != nil {
+				fmt.Printf("⚠️ Invalid multiaddress %s: %v\n", addrStr, err)
+				continue
+			}
+			addrs = append(addrs, addr)
+		}
+
+		if len(addrs) > 0 {
+			peers = append(peers, peer.AddrInfo{
+				ID:    peerID,
+				Addrs: addrs,
+			})
+		}
+	}
+
+	bp.peers = peers
+	fmt.Printf("📋 Loaded %d healthy bootstrap peers from configuration\n", len(peers))
+
+	return nil
+}
+
+// LoadFromEnvironment loads bootstrap configuration from environment variables
+func (bp *BootstrapPool) LoadFromEnvironment() error {
+	// Try loading from file first
+	if bootstrapFile := os.Getenv("BOOTSTRAP_JSON"); bootstrapFile != "" {
+		if err := bp.LoadFromFile(bootstrapFile); err != nil {
+			fmt.Printf("⚠️ Failed to load bootstrap from file: %v\n", err)
+		} else {
+			return nil // Successfully loaded from file
+		}
+	}
+
+	// Try loading from URL
+	if bootstrapURL := os.Getenv("BOOTSTRAP_URL"); bootstrapURL != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		if err := bp.LoadFromURL(ctx, bootstrapURL); err != nil {
+			fmt.Printf("⚠️ Failed to load bootstrap from URL: %v\n", err)
+		} else {
+			return nil // Successfully loaded from URL
+		}
+	}
+
+	// Fallback to legacy environment variable
+	if bootstrapPeersEnv := os.Getenv("CHORUS_BOOTSTRAP_PEERS"); bootstrapPeersEnv != "" {
+		return bp.loadFromLegacyEnv(bootstrapPeersEnv)
+	}
+
+	return nil // No bootstrap configuration found
+}
+
+// loadFromLegacyEnv loads from comma-separated multiaddress list
+func (bp *BootstrapPool) loadFromLegacyEnv(peersEnv string) error {
+	peerStrs := strings.Split(peersEnv, ",")
+	var peers []peer.AddrInfo
+
+	for _, peerStr := range peerStrs {
+		peerStr = strings.TrimSpace(peerStr)
+		if peerStr == "" {
+			continue
+		}
+
+		// Parse multiaddress
+		addr, err := multiaddr.NewMultiaddr(peerStr)
+		if err != nil {
+			fmt.Printf("⚠️ Invalid bootstrap peer %s: %v\n", peerStr, err)
+			continue
+		}
+
+		// Extract peer info
+		info, err := peer.AddrInfoFromP2pAddr(addr)
+		if err != nil {
+			fmt.Printf("⚠️ Failed to parse peer info from %s: %v\n", peerStr, err)
+			continue
+		}
+
+		peers = append(peers, *info)
+	}
+
+	bp.peers = peers
+	fmt.Printf("📋 Loaded %d bootstrap peers from legacy environment\n", len(peers))
+
+	return nil
+}
+
+// GetSubset returns a subset of bootstrap peers for a replica
+func (bp *BootstrapPool) GetSubset(count int) BootstrapSubset {
+	if len(bp.peers) == 0 {
+		return BootstrapSubset{
+			Peers:          []peer.AddrInfo{},
+			StaggerDelayMS: 0,
+			AssignedAt:     time.Now(),
+		}
+	}
+
+	// Ensure count doesn't exceed available peers
+	if count > len(bp.peers) {
+		count = len(bp.peers)
+	}
+
+	// Randomly select peers from the pool
+	selectedPeers := make([]peer.AddrInfo, 0, count)
+	indices := rand.Perm(len(bp.peers))
+
+	for i := 0; i < count; i++ {
+		selectedPeers = append(selectedPeers, bp.peers[indices[i]])
+	}
+
+	// Generate random stagger delay (0 to configured max)
+	staggerMS := 0
+	if bp.staggerDelay > 0 {
+		staggerMS = rand.Intn(int(bp.staggerDelay.Milliseconds()))
+	}
+
+	return BootstrapSubset{
+		Peers:          selectedPeers,
+		StaggerDelayMS: staggerMS,
+		AssignedAt:     time.Now(),
+	}
+}
+
+// ConnectWithRateLimit connects to bootstrap peers with rate limiting
+func (bp *BootstrapPool) ConnectWithRateLimit(ctx context.Context, h host.Host, subset BootstrapSubset) error {
+	if len(subset.Peers) == 0 {
+		return nil // No peers to connect to
+	}
+
+	// Apply stagger delay
+	if subset.StaggerDelayMS > 0 {
+		delay := time.Duration(subset.StaggerDelayMS) * time.Millisecond
+		fmt.Printf("⏱️ Applying join stagger delay: %v\n", delay)
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(delay):
+			// Continue after delay
+		}
+	}
+
+	// Create rate limiter for dials
+	ticker := time.NewTicker(time.Second / time.Duration(bp.dialsPerSecond))
+	defer ticker.Stop()
+
+	// Semaphore for concurrent dials
+	semaphore := make(chan struct{}, bp.maxConcurrent)
+
+	// Connect to each peer with rate limiting
+	for i, peerInfo := range subset.Peers {
+		// Wait for rate limiter
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+			// Rate limit satisfied
+		}
+
+		// Acquire semaphore
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case semaphore <- struct{}{}:
+			// Semaphore acquired
+		}
+
+		// Connect to peer in goroutine
+		go func(info peer.AddrInfo, index int) {
+			defer func() { <-semaphore }() // Release semaphore
+
+			ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+			defer cancel()
+
+			if err := h.Connect(ctx, info); err != nil {
+				fmt.Printf("⚠️ Failed to connect to bootstrap peer %s (%d/%d): %v\n",
+					info.ID.ShortString(), index+1, len(subset.Peers), err)
+			} else {
+				fmt.Printf("🔗 Connected to bootstrap peer %s (%d/%d)\n",
+					info.ID.ShortString(), index+1, len(subset.Peers))
+			}
+		}(peerInfo, i)
+	}
+
+	// Wait for all connections to complete or timeout
+	for i := 0; i < bp.maxConcurrent && i < len(subset.Peers); i++ {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case semaphore <- struct{}{}:
+			<-semaphore // Immediately release
+		}
+	}
+
+	return nil
+}
+
+// GetPeerCount returns the number of available bootstrap peers
+func (bp *BootstrapPool) GetPeerCount() int {
+	return len(bp.peers)
+}
+
+// GetPeers returns all bootstrap peers (for debugging)
+func (bp *BootstrapPool) GetPeers() []peer.AddrInfo {
+	return bp.peers
+}
+
+// GetStats returns bootstrap pool statistics
+func (bp *BootstrapPool) GetStats() map[string]interface{} {
+	return map[string]interface{}{
+		"peer_count":        len(bp.peers),
+		"dials_per_second":  bp.dialsPerSecond,
+		"max_concurrent":    bp.maxConcurrent,
+		"stagger_delay_ms":  bp.staggerDelay.Milliseconds(),
+	}
+}

pkg/config/config.go

@@ -216,7 +216,7 @@ func LoadFromEnvironment() (*Config, error) {
 			AuditLogging:    getEnvBoolOrDefault("CHORUS_AUDIT_LOGGING", true),
 			AuditPath:       getEnvOrDefault("CHORUS_AUDIT_PATH", "/tmp/chorus-audit.log"),
 			ElectionConfig: ElectionConfig{
-				DiscoveryTimeout: getEnvDurationOrDefault("CHORUS_DISCOVERY_TIMEOUT", 10*time.Second),
+				DiscoveryTimeout: getEnvDurationOrDefault("CHORUS_DISCOVERY_TIMEOUT", 15*time.Second),
 				HeartbeatTimeout: getEnvDurationOrDefault("CHORUS_HEARTBEAT_TIMEOUT", 30*time.Second),
 				ElectionTimeout:  getEnvDurationOrDefault("CHORUS_ELECTION_TIMEOUT", 60*time.Second),
 				DiscoveryBackoff: getEnvDurationOrDefault("CHORUS_DISCOVERY_BACKOFF", 5*time.Second),

pkg/config/runtime_config.go

@@ -0,0 +1,354 @@
+package config
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// RuntimeConfig provides dynamic configuration with assignment override support
+type RuntimeConfig struct {
+	mu   sync.RWMutex
+	base *Config // Base configuration from environment
+	over *Config // Override configuration from assignment
+}
+
+// AssignmentConfig represents configuration received from WHOOSH assignment
+type AssignmentConfig struct {
+	Role                  string            `json:"role,omitempty"`
+	Model                 string            `json:"model,omitempty"`
+	PromptUCXL           string            `json:"prompt_ucxl,omitempty"`
+	Specialization       string            `json:"specialization,omitempty"`
+	Capabilities         []string          `json:"capabilities,omitempty"`
+	Environment          map[string]string `json:"environment,omitempty"`
+	BootstrapPeers       []string          `json:"bootstrap_peers,omitempty"`
+	JoinStaggerMS        int               `json:"join_stagger_ms,omitempty"`
+	DialsPerSecond       int               `json:"dials_per_second,omitempty"`
+	MaxConcurrentDHT     int               `json:"max_concurrent_dht,omitempty"`
+	AssignmentID         string            `json:"assignment_id,omitempty"`
+	ConfigEpoch          int64             `json:"config_epoch,omitempty"`
+}
+
+// NewRuntimeConfig creates a new runtime configuration manager
+func NewRuntimeConfig(baseConfig *Config) *RuntimeConfig {
+	return &RuntimeConfig{
+		base: baseConfig,
+		over: &Config{}, // Empty override initially
+	}
+}
+
+// Get retrieves a configuration value with override precedence
+func (rc *RuntimeConfig) Get(key string) interface{} {
+	rc.mu.RLock()
+	defer rc.mu.RUnlock()
+
+	// Check override first, then base
+	if value := rc.getFromConfig(rc.over, key); value != nil {
+		return value
+	}
+	return rc.getFromConfig(rc.base, key)
+}
+
+// getFromConfig extracts a value from a config struct by key
+func (rc *RuntimeConfig) getFromConfig(cfg *Config, key string) interface{} {
+	if cfg == nil {
+		return nil
+	}
+
+	switch key {
+	case "agent.role":
+		if cfg.Agent.Role != "" {
+			return cfg.Agent.Role
+		}
+	case "agent.specialization":
+		if cfg.Agent.Specialization != "" {
+			return cfg.Agent.Specialization
+		}
+	case "agent.capabilities":
+		if len(cfg.Agent.Capabilities) > 0 {
+			return cfg.Agent.Capabilities
+		}
+	case "agent.models":
+		if len(cfg.Agent.Models) > 0 {
+			return cfg.Agent.Models
+		}
+	case "agent.default_reasoning_model":
+		if cfg.Agent.DefaultReasoningModel != "" {
+			return cfg.Agent.DefaultReasoningModel
+		}
+	case "v2.dht.bootstrap_peers":
+		if len(cfg.V2.DHT.BootstrapPeers) > 0 {
+			return cfg.V2.DHT.BootstrapPeers
+		}
+	}
+
+	return nil
+}
+
+// GetString retrieves a string configuration value
+func (rc *RuntimeConfig) GetString(key string) string {
+	if value := rc.Get(key); value != nil {
+		if str, ok := value.(string); ok {
+			return str
+		}
+	}
+	return ""
+}
+
+// GetStringSlice retrieves a string slice configuration value
+func (rc *RuntimeConfig) GetStringSlice(key string) []string {
+	if value := rc.Get(key); value != nil {
+		if slice, ok := value.([]string); ok {
+			return slice
+		}
+	}
+	return nil
+}
+
+// GetInt retrieves an integer configuration value
+func (rc *RuntimeConfig) GetInt(key string) int {
+	if value := rc.Get(key); value != nil {
+		if i, ok := value.(int); ok {
+			return i
+		}
+	}
+	return 0
+}
+
+// LoadAssignment loads configuration from WHOOSH assignment endpoint
+func (rc *RuntimeConfig) LoadAssignment(ctx context.Context) error {
+	assignURL := os.Getenv("ASSIGN_URL")
+	if assignURL == "" {
+		return nil // No assignment URL configured
+	}
+
+	// Build assignment request URL with task identity
+	params := url.Values{}
+	if taskSlot := os.Getenv("TASK_SLOT"); taskSlot != "" {
+		params.Set("slot", taskSlot)
+	}
+	if taskID := os.Getenv("TASK_ID"); taskID != "" {
+		params.Set("task", taskID)
+	}
+	if clusterID := os.Getenv("CHORUS_CLUSTER_ID"); clusterID != "" {
+		params.Set("cluster", clusterID)
+	}
+
+	fullURL := assignURL
+	if len(params) > 0 {
+		fullURL += "?" + params.Encode()
+	}
+
+	// Fetch assignment with timeout
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", fullURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create assignment request: %w", err)
+	}
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("assignment request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("assignment request failed with status %d", resp.StatusCode)
+	}
+
+	// Parse assignment response
+	var assignment AssignmentConfig
+	if err := json.NewDecoder(resp.Body).Decode(&assignment); err != nil {
+		return fmt.Errorf("failed to decode assignment response: %w", err)
+	}
+
+	// Apply assignment to override config
+	if err := rc.applyAssignment(&assignment); err != nil {
+		return fmt.Errorf("failed to apply assignment: %w", err)
+	}
+
+	fmt.Printf("📥 Loaded assignment: role=%s, model=%s, epoch=%d\n",
+		assignment.Role, assignment.Model, assignment.ConfigEpoch)
+
+	return nil
+}
+
+// LoadAssignmentFromFile loads configuration from a file (for config objects)
+func (rc *RuntimeConfig) LoadAssignmentFromFile(filePath string) error {
+	if filePath == "" {
+		return nil // No file configured
+	}
+
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to read assignment file %s: %w", filePath, err)
+	}
+
+	var assignment AssignmentConfig
+	if err := json.Unmarshal(data, &assignment); err != nil {
+		return fmt.Errorf("failed to parse assignment file: %w", err)
+	}
+
+	if err := rc.applyAssignment(&assignment); err != nil {
+		return fmt.Errorf("failed to apply file assignment: %w", err)
+	}
+
+	fmt.Printf("📁 Loaded assignment from file: role=%s, model=%s\n",
+		assignment.Role, assignment.Model)
+
+	return nil
+}
+
+// applyAssignment applies an assignment to the override configuration
+func (rc *RuntimeConfig) applyAssignment(assignment *AssignmentConfig) error {
+	rc.mu.Lock()
+	defer rc.mu.Unlock()
+
+	// Create new override config
+	override := &Config{
+		Agent: AgentConfig{
+			Role:                  assignment.Role,
+			Specialization:        assignment.Specialization,
+			Capabilities:          assignment.Capabilities,
+			DefaultReasoningModel: assignment.Model,
+		},
+		V2: V2Config{
+			DHT: DHTConfig{
+				BootstrapPeers: assignment.BootstrapPeers,
+			},
+		},
+	}
+
+	// Handle models array
+	if assignment.Model != "" {
+		override.Agent.Models = []string{assignment.Model}
+	}
+
+	// Apply environment variables from assignment
+	for key, value := range assignment.Environment {
+		os.Setenv(key, value)
+	}
+
+	rc.over = override
+
+	return nil
+}
+
+// StartReloadHandler starts a signal handler for configuration reload (SIGHUP)
+func (rc *RuntimeConfig) StartReloadHandler(ctx context.Context) {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGHUP)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-sigChan:
+				fmt.Println("🔄 Received SIGHUP, reloading configuration...")
+				if err := rc.LoadAssignment(ctx); err != nil {
+					fmt.Printf("⚠️ Failed to reload assignment: %v\n", err)
+				} else {
+					fmt.Println("✅ Configuration reloaded successfully")
+				}
+			}
+		}
+	}()
+}
+
+// GetBaseConfig returns the base configuration (from environment)
+func (rc *RuntimeConfig) GetBaseConfig() *Config {
+	rc.mu.RLock()
+	defer rc.mu.RUnlock()
+	return rc.base
+}
+
+// GetEffectiveConfig returns the effective merged configuration
+func (rc *RuntimeConfig) GetEffectiveConfig() *Config {
+	rc.mu.RLock()
+	defer rc.mu.RUnlock()
+
+	// Start with base config
+	effective := *rc.base
+
+	// Apply overrides
+	if rc.over.Agent.Role != "" {
+		effective.Agent.Role = rc.over.Agent.Role
+	}
+	if rc.over.Agent.Specialization != "" {
+		effective.Agent.Specialization = rc.over.Agent.Specialization
+	}
+	if len(rc.over.Agent.Capabilities) > 0 {
+		effective.Agent.Capabilities = rc.over.Agent.Capabilities
+	}
+	if len(rc.over.Agent.Models) > 0 {
+		effective.Agent.Models = rc.over.Agent.Models
+	}
+	if rc.over.Agent.DefaultReasoningModel != "" {
+		effective.Agent.DefaultReasoningModel = rc.over.Agent.DefaultReasoningModel
+	}
+	if len(rc.over.V2.DHT.BootstrapPeers) > 0 {
+		effective.V2.DHT.BootstrapPeers = rc.over.V2.DHT.BootstrapPeers
+	}
+
+	return &effective
+}
+
+// GetAssignmentStats returns assignment statistics for monitoring
+func (rc *RuntimeConfig) GetAssignmentStats() map[string]interface{} {
+	rc.mu.RLock()
+	defer rc.mu.RUnlock()
+
+	hasOverride := rc.over.Agent.Role != "" ||
+		rc.over.Agent.Specialization != "" ||
+		len(rc.over.Agent.Capabilities) > 0 ||
+		len(rc.over.V2.DHT.BootstrapPeers) > 0
+
+	stats := map[string]interface{}{
+		"has_assignment": hasOverride,
+		"assign_url":     os.Getenv("ASSIGN_URL"),
+		"task_slot":      os.Getenv("TASK_SLOT"),
+		"task_id":        os.Getenv("TASK_ID"),
+	}
+
+	if hasOverride {
+		stats["assigned_role"] = rc.over.Agent.Role
+		stats["assigned_specialization"] = rc.over.Agent.Specialization
+		stats["assigned_capabilities"] = rc.over.Agent.Capabilities
+		stats["assigned_models"] = rc.over.Agent.Models
+		stats["bootstrap_peers_count"] = len(rc.over.V2.DHT.BootstrapPeers)
+	}
+
+	return stats
+}
+
+// InitializeAssignmentFromEnv initializes assignment from environment variables
+func (rc *RuntimeConfig) InitializeAssignmentFromEnv(ctx context.Context) error {
+	// Try loading from assignment URL first
+	if err := rc.LoadAssignment(ctx); err != nil {
+		fmt.Printf("⚠️ Failed to load assignment from URL: %v\n", err)
+	}
+
+	// Try loading from file (for config objects)
+	if assignFile := os.Getenv("ASSIGNMENT_FILE"); assignFile != "" {
+		if err := rc.LoadAssignmentFromFile(assignFile); err != nil {
+			fmt.Printf("⚠️ Failed to load assignment from file: %v\n", err)
+		}
+	}
+
+	// Start reload handler for SIGHUP
+	rc.StartReloadHandler(ctx)
+
+	return nil
+}

pkg/crypto/key_derivation.go

@@ -0,0 +1,306 @@
+package crypto
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+
+	"golang.org/x/crypto/hkdf"
+	"filippo.io/age"
+	"filippo.io/age/armor"
+)
+
+// KeyDerivationManager handles cluster-scoped key derivation for DHT encryption
+type KeyDerivationManager struct {
+	clusterRootKey []byte
+	clusterID      string
+}
+
+// DerivedKeySet contains keys derived for a specific role/scope
+type DerivedKeySet struct {
+	RoleKey      []byte              // Role-specific key
+	NodeKey      []byte              // Node-specific key for this instance
+	AGEIdentity  *age.X25519Identity // AGE identity for encryption/decryption
+	AGERecipient *age.X25519Recipient // AGE recipient for encryption
+}
+
+// NewKeyDerivationManager creates a new key derivation manager
+func NewKeyDerivationManager(clusterRootKey []byte, clusterID string) *KeyDerivationManager {
+	return &KeyDerivationManager{
+		clusterRootKey: clusterRootKey,
+		clusterID:      clusterID,
+	}
+}
+
+// NewKeyDerivationManagerFromSeed creates a manager from a seed string
+func NewKeyDerivationManagerFromSeed(seed, clusterID string) *KeyDerivationManager {
+	// Use HKDF to derive a consistent root key from seed
+	hash := sha256.New
+	hkdf := hkdf.New(hash, []byte(seed), []byte(clusterID), []byte("CHORUS-cluster-root"))
+
+	rootKey := make([]byte, 32)
+	if _, err := io.ReadFull(hkdf, rootKey); err != nil {
+		panic(fmt.Errorf("failed to derive cluster root key: %w", err))
+	}
+
+	return &KeyDerivationManager{
+		clusterRootKey: rootKey,
+		clusterID:      clusterID,
+	}
+}
+
+// DeriveRoleKeys derives encryption keys for a specific role and agent
+func (kdm *KeyDerivationManager) DeriveRoleKeys(role, agentID string) (*DerivedKeySet, error) {
+	if kdm.clusterRootKey == nil {
+		return nil, fmt.Errorf("cluster root key not initialized")
+	}
+
+	// Derive role-specific key
+	roleKey, err := kdm.deriveKey(fmt.Sprintf("role-%s", role), 32)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive role key: %w", err)
+	}
+
+	// Derive node-specific key from role key and agent ID
+	nodeKey, err := kdm.deriveKeyFromParent(roleKey, fmt.Sprintf("node-%s", agentID), 32)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive node key: %w", err)
+	}
+
+	// Generate AGE identity from node key
+	ageIdentity, err := kdm.generateAGEIdentityFromKey(nodeKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate AGE identity: %w", err)
+	}
+
+	ageRecipient := ageIdentity.Recipient()
+
+	return &DerivedKeySet{
+		RoleKey:      roleKey,
+		NodeKey:      nodeKey,
+		AGEIdentity:  ageIdentity,
+		AGERecipient: ageRecipient,
+	}, nil
+}
+
+// DeriveClusterWideKeys derives keys that are shared across the entire cluster for a role
+func (kdm *KeyDerivationManager) DeriveClusterWideKeys(role string) (*DerivedKeySet, error) {
+	if kdm.clusterRootKey == nil {
+		return nil, fmt.Errorf("cluster root key not initialized")
+	}
+
+	// Derive role-specific key
+	roleKey, err := kdm.deriveKey(fmt.Sprintf("role-%s", role), 32)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive role key: %w", err)
+	}
+
+	// For cluster-wide keys, use a deterministic "cluster" identifier
+	clusterNodeKey, err := kdm.deriveKeyFromParent(roleKey, "cluster-shared", 32)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive cluster node key: %w", err)
+	}
+
+	// Generate AGE identity from cluster node key
+	ageIdentity, err := kdm.generateAGEIdentityFromKey(clusterNodeKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate AGE identity: %w", err)
+	}
+
+	ageRecipient := ageIdentity.Recipient()
+
+	return &DerivedKeySet{
+		RoleKey:      roleKey,
+		NodeKey:      clusterNodeKey,
+		AGEIdentity:  ageIdentity,
+		AGERecipient: ageRecipient,
+	}, nil
+}
+
+// deriveKey derives a key from the cluster root key using HKDF
+func (kdm *KeyDerivationManager) deriveKey(info string, length int) ([]byte, error) {
+	hash := sha256.New
+	hkdf := hkdf.New(hash, kdm.clusterRootKey, []byte(kdm.clusterID), []byte(info))
+
+	key := make([]byte, length)
+	if _, err := io.ReadFull(hkdf, key); err != nil {
+		return nil, fmt.Errorf("HKDF key derivation failed: %w", err)
+	}
+
+	return key, nil
+}
+
+// deriveKeyFromParent derives a key from a parent key using HKDF
+func (kdm *KeyDerivationManager) deriveKeyFromParent(parentKey []byte, info string, length int) ([]byte, error) {
+	hash := sha256.New
+	hkdf := hkdf.New(hash, parentKey, []byte(kdm.clusterID), []byte(info))
+
+	key := make([]byte, length)
+	if _, err := io.ReadFull(hkdf, key); err != nil {
+		return nil, fmt.Errorf("HKDF key derivation failed: %w", err)
+	}
+
+	return key, nil
+}
+
+// generateAGEIdentityFromKey generates a deterministic AGE identity from a key
+func (kdm *KeyDerivationManager) generateAGEIdentityFromKey(key []byte) (*age.X25519Identity, error) {
+	if len(key) < 32 {
+		return nil, fmt.Errorf("key must be at least 32 bytes")
+	}
+
+	// Use the first 32 bytes as the private key seed
+	var privKey [32]byte
+	copy(privKey[:], key[:32])
+
+	// Generate a new identity (note: this loses deterministic behavior)
+	// TODO: Implement deterministic key derivation when age API allows
+	identity, err := age.GenerateX25519Identity()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AGE identity: %w", err)
+	}
+
+	return identity, nil
+}
+
+// EncryptForRole encrypts data for a specific role (all nodes in that role can decrypt)
+func (kdm *KeyDerivationManager) EncryptForRole(data []byte, role string) ([]byte, error) {
+	// Get cluster-wide keys for the role
+	keySet, err := kdm.DeriveClusterWideKeys(role)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive cluster keys: %w", err)
+	}
+
+	// Encrypt using AGE
+	var encrypted []byte
+	buf := &writeBuffer{data: &encrypted}
+	armorWriter := armor.NewWriter(buf)
+
+	ageWriter, err := age.Encrypt(armorWriter, keySet.AGERecipient)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create age writer: %w", err)
+	}
+
+	if _, err := ageWriter.Write(data); err != nil {
+		return nil, fmt.Errorf("failed to write encrypted data: %w", err)
+	}
+
+	if err := ageWriter.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close age writer: %w", err)
+	}
+
+	if err := armorWriter.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close armor writer: %w", err)
+	}
+
+	return encrypted, nil
+}
+
+// DecryptForRole decrypts data encrypted for a specific role
+func (kdm *KeyDerivationManager) DecryptForRole(encryptedData []byte, role, agentID string) ([]byte, error) {
+	// Try cluster-wide keys first
+	clusterKeys, err := kdm.DeriveClusterWideKeys(role)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive cluster keys: %w", err)
+	}
+
+	if decrypted, err := kdm.decryptWithIdentity(encryptedData, clusterKeys.AGEIdentity); err == nil {
+		return decrypted, nil
+	}
+
+	// If cluster-wide decryption fails, try node-specific keys
+	nodeKeys, err := kdm.DeriveRoleKeys(role, agentID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive node keys: %w", err)
+	}
+
+	return kdm.decryptWithIdentity(encryptedData, nodeKeys.AGEIdentity)
+}
+
+// decryptWithIdentity decrypts data using an AGE identity
+func (kdm *KeyDerivationManager) decryptWithIdentity(encryptedData []byte, identity *age.X25519Identity) ([]byte, error) {
+	armorReader := armor.NewReader(newReadBuffer(encryptedData))
+
+	ageReader, err := age.Decrypt(armorReader, identity)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt: %w", err)
+	}
+
+	decrypted, err := io.ReadAll(ageReader)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read decrypted data: %w", err)
+	}
+
+	return decrypted, nil
+}
+
+// GetRoleRecipients returns AGE recipients for all nodes in a role (for multi-recipient encryption)
+func (kdm *KeyDerivationManager) GetRoleRecipients(role string, agentIDs []string) ([]*age.X25519Recipient, error) {
+	var recipients []*age.X25519Recipient
+
+	// Add cluster-wide recipient
+	clusterKeys, err := kdm.DeriveClusterWideKeys(role)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive cluster keys: %w", err)
+	}
+	recipients = append(recipients, clusterKeys.AGERecipient)
+
+	// Add node-specific recipients
+	for _, agentID := range agentIDs {
+		nodeKeys, err := kdm.DeriveRoleKeys(role, agentID)
+		if err != nil {
+			continue // Skip this agent on error
+		}
+		recipients = append(recipients, nodeKeys.AGERecipient)
+	}
+
+	return recipients, nil
+}
+
+// GetKeySetStats returns statistics about derived key sets
+func (kdm *KeyDerivationManager) GetKeySetStats(role, agentID string) map[string]interface{} {
+	stats := map[string]interface{}{
+		"cluster_id": kdm.clusterID,
+		"role":       role,
+		"agent_id":   agentID,
+	}
+
+	// Try to derive keys and add fingerprint info
+	if keySet, err := kdm.DeriveRoleKeys(role, agentID); err == nil {
+		stats["node_key_length"] = len(keySet.NodeKey)
+		stats["role_key_length"] = len(keySet.RoleKey)
+		stats["age_recipient"] = keySet.AGERecipient.String()
+	}
+
+	return stats
+}
+
+// Helper types for AGE encryption/decryption
+
+type writeBuffer struct {
+	data *[]byte
+}
+
+func (w *writeBuffer) Write(p []byte) (n int, err error) {
+	*w.data = append(*w.data, p...)
+	return len(p), nil
+}
+
+type readBuffer struct {
+	data []byte
+	pos  int
+}
+
+func newReadBuffer(data []byte) *readBuffer {
+	return &readBuffer{data: data, pos: 0}
+}
+
+func (r *readBuffer) Read(p []byte) (n int, err error) {
+	if r.pos >= len(r.data) {
+		return 0, io.EOF
+	}
+
+	n = copy(p, r.data[r.pos:])
+	r.pos += n
+	return n, nil
+}

pkg/election/election.go

@@ -167,10 +167,18 @@ func (em *ElectionManager) Start() error {
 	}
 
 	// Start discovery process
-	go em.startDiscoveryLoop()
+	log.Printf("🔍 About to start discovery loop goroutine...")
+	go func() {
+		log.Printf("🔍 Discovery loop goroutine started successfully")
+		em.startDiscoveryLoop()
+	}()
 
 	// Start election coordinator
-	go em.electionCoordinator()
+	log.Printf("🗳️ About to start election coordinator goroutine...")
+	go func() {
+		log.Printf("🗳️ Election coordinator goroutine started successfully")
+		em.electionCoordinator()
+	}()
 
 	// Start heartbeat if this node is already admin at startup
 	if em.IsCurrentAdmin() {
@@ -214,6 +222,16 @@ func (em *ElectionManager) Stop() {
 
 // TriggerElection manually triggers an election
 func (em *ElectionManager) TriggerElection(trigger ElectionTrigger) {
+	// Check if election already in progress
+	em.mu.RLock()
+	currentState := em.state
+	em.mu.RUnlock()
+
+	if currentState != StateIdle {
+		log.Printf("🗳️ Election already in progress (state: %s), ignoring trigger: %s", currentState, trigger)
+		return
+	}
+
 	select {
 	case em.electionTrigger <- trigger:
 		log.Printf("🗳️ Election triggered: %s", trigger)
@@ -262,13 +280,27 @@ func (em *ElectionManager) GetHeartbeatStatus() map[string]interface{} {
 
 // startDiscoveryLoop starts the admin discovery loop
 func (em *ElectionManager) startDiscoveryLoop() {
-	log.Printf("🔍 Starting admin discovery loop")
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("🔍 PANIC in discovery loop: %v", r)
+		}
+		log.Printf("🔍 Discovery loop goroutine exiting")
+	}()
+
+	log.Printf("🔍 ENHANCED-DEBUG: Starting admin discovery loop with timeout: %v", em.config.Security.ElectionConfig.DiscoveryTimeout)
+	log.Printf("🔍 ENHANCED-DEBUG: Context status: err=%v", em.ctx.Err())
+	log.Printf("🔍 ENHANCED-DEBUG: Node ID: %s, Can be admin: %v", em.nodeID, em.canBeAdmin())
 
 	for {
+		log.Printf("🔍 Discovery loop iteration starting, waiting for timeout...")
+		log.Printf("🔍 Context status before select: err=%v", em.ctx.Err())
+
 		select {
 		case <-em.ctx.Done():
+			log.Printf("🔍 Discovery loop cancelled via context: %v", em.ctx.Err())
 			return
 		case <-time.After(em.config.Security.ElectionConfig.DiscoveryTimeout):
+			log.Printf("🔍 Discovery timeout triggered! Calling performAdminDiscovery()...")
 			em.performAdminDiscovery()
 		}
 	}
@@ -281,8 +313,12 @@ func (em *ElectionManager) performAdminDiscovery() {
 	lastHeartbeat := em.lastHeartbeat
 	em.mu.Unlock()
 
+	log.Printf("🔍 Discovery check: state=%s, lastHeartbeat=%v, canAdmin=%v",
+		currentState, lastHeartbeat, em.canBeAdmin())
+
 	// Only discover if we're idle or the heartbeat is stale
 	if currentState != StateIdle {
+		log.Printf("🔍 Skipping discovery - not in idle state (current: %s)", currentState)
 		return
 	}
 
@@ -294,13 +330,66 @@ func (em *ElectionManager) performAdminDiscovery() {
 	}
 
 	// If we haven't heard from an admin recently, try to discover one
-	if lastHeartbeat.IsZero() || time.Since(lastHeartbeat) > em.config.Security.ElectionConfig.DiscoveryTimeout/2 {
+	timeSinceHeartbeat := time.Since(lastHeartbeat)
+	discoveryThreshold := em.config.Security.ElectionConfig.DiscoveryTimeout / 2
+
+	log.Printf("🔍 Heartbeat check: isZero=%v, timeSince=%v, threshold=%v",
+		lastHeartbeat.IsZero(), timeSinceHeartbeat, discoveryThreshold)
+
+	if lastHeartbeat.IsZero() || timeSinceHeartbeat > discoveryThreshold {
+		log.Printf("🔍 Sending discovery request...")
 		em.sendDiscoveryRequest()
+
+		// 🚨 CRITICAL FIX: If we have no admin and can become admin, trigger election after discovery timeout
+		em.mu.Lock()
+		currentAdmin := em.currentAdmin
+		em.mu.Unlock()
+
+		if currentAdmin == "" && em.canBeAdmin() {
+			log.Printf("🗳️ No admin discovered and we can be admin - scheduling election check")
+			go func() {
+				// Add randomization to prevent simultaneous elections from all nodes
+				baseDelay := em.config.Security.ElectionConfig.DiscoveryTimeout * 2
+				randomDelay := time.Duration(rand.Intn(int(em.config.Security.ElectionConfig.DiscoveryTimeout)))
+				totalDelay := baseDelay + randomDelay
+
+				log.Printf("🗳️ Waiting %v before checking if election needed", totalDelay)
+				time.Sleep(totalDelay)
+
+				// Check again if still no admin and no one else started election
+				em.mu.RLock()
+				stillNoAdmin := em.currentAdmin == ""
+				stillIdle := em.state == StateIdle
+				em.mu.RUnlock()
+
+				if stillNoAdmin && stillIdle && em.canBeAdmin() {
+					log.Printf("🗳️ Election grace period expired with no admin - triggering election")
+					em.TriggerElection(TriggerDiscoveryFailure)
+				} else {
+					log.Printf("🗳️ Election check: admin=%s, state=%s - skipping election", em.currentAdmin, em.state)
+				}
+			}()
+		}
+	} else {
+		log.Printf("🔍 Discovery threshold not met - waiting")
 	}
 }
 
 // sendDiscoveryRequest broadcasts admin discovery request
 func (em *ElectionManager) sendDiscoveryRequest() {
+	em.mu.RLock()
+	currentAdmin := em.currentAdmin
+	em.mu.RUnlock()
+
+	// WHOAMI debug message
+	if currentAdmin == "" {
+		log.Printf("🤖 WHOAMI: I'm %s and I have no leader", em.nodeID)
+	} else {
+		log.Printf("🤖 WHOAMI: I'm %s and my leader is %s", em.nodeID, currentAdmin)
+	}
+
+	log.Printf("📡 Sending admin discovery request from node %s", em.nodeID)
+
 	discoveryMsg := ElectionMessage{
 		Type:      "admin_discovery_request",
 		NodeID:    em.nodeID,
@@ -309,6 +398,8 @@ func (em *ElectionManager) sendDiscoveryRequest() {
 
 	if err := em.publishElectionMessage(discoveryMsg); err != nil {
 		log.Printf("❌ Failed to send admin discovery request: %v", err)
+	} else {
+		log.Printf("✅ Admin discovery request sent successfully")
 	}
 }
 
@@ -652,6 +743,9 @@ func (em *ElectionManager) handleAdminDiscoveryRequest(msg ElectionMessage) {
 	state := em.state
 	em.mu.RUnlock()
 
+	log.Printf("📩 Received admin discovery request from %s (my leader: %s, state: %s)",
+		msg.NodeID, currentAdmin, state)
+
 	// Only respond if we know who the current admin is and we're idle
 	if currentAdmin != "" && state == StateIdle {
 		responseMsg := ElectionMessage{
@@ -663,23 +757,43 @@ func (em *ElectionManager) handleAdminDiscoveryRequest(msg ElectionMessage) {
 			},
 		}
 
+		log.Printf("📤 Responding to discovery with admin: %s", currentAdmin)
 		if err := em.publishElectionMessage(responseMsg); err != nil {
 			log.Printf("❌ Failed to send admin discovery response: %v", err)
+		} else {
+			log.Printf("✅ Admin discovery response sent successfully")
 		}
+	} else {
+		log.Printf("🔇 Not responding to discovery (admin=%s, state=%s)", currentAdmin, state)
 	}
 }
 
 // handleAdminDiscoveryResponse processes admin discovery responses
 func (em *ElectionManager) handleAdminDiscoveryResponse(msg ElectionMessage) {
+	log.Printf("📥 Received admin discovery response from %s", msg.NodeID)
+
 	if data, ok := msg.Data.(map[string]interface{}); ok {
 		if admin, ok := data["current_admin"].(string); ok && admin != "" {
 			em.mu.Lock()
+			oldAdmin := em.currentAdmin
 			if em.currentAdmin == "" {
-				log.Printf("📡 Discovered admin: %s", admin)
+				log.Printf("📡 Discovered admin: %s (reported by %s)", admin, msg.NodeID)
 				em.currentAdmin = admin
+				em.lastHeartbeat = time.Now() // Set initial heartbeat
+			} else if em.currentAdmin != admin {
+				log.Printf("⚠️ Admin conflict: I know %s, but %s reports %s", em.currentAdmin, msg.NodeID, admin)
+			} else {
+				log.Printf("📡 Admin confirmed: %s (reported by %s)", admin, msg.NodeID)
 			}
 			em.mu.Unlock()
+
+			// Trigger callback if admin changed
+			if oldAdmin != admin && em.onAdminChanged != nil {
+				em.onAdminChanged(oldAdmin, admin)
+			}
 		}
+	} else {
+		log.Printf("❌ Invalid admin discovery response from %s", msg.NodeID)
 	}
 }
 

pkg/health/enhanced_health_checks.go

@@ -179,9 +179,11 @@ func (ehc *EnhancedHealthChecks) registerHealthChecks() {
 		ehc.manager.RegisterCheck(ehc.createEnhancedPubSubCheck())
 	}
 	
-	if ehc.config.EnableDHTProbes {
+	// Temporarily disable DHT health check to prevent shutdown issues
-		ehc.manager.RegisterCheck(ehc.createEnhancedDHTCheck())
+	// TODO: Fix DHT configuration and re-enable this check
-	}
+	// if ehc.config.EnableDHTProbes {
+	// 	ehc.manager.RegisterCheck(ehc.createEnhancedDHTCheck())
+	// }
 	
 	if ehc.config.EnableElectionProbes {
 		ehc.manager.RegisterCheck(ehc.createElectionHealthCheck())
@@ -290,7 +292,7 @@ func (ehc *EnhancedHealthChecks) createElectionHealthCheck() *HealthCheck {
 	return &HealthCheck{
 		Name:        "election-health",
 		Description: "Election system health and leadership stability check",
-		Enabled:     true,
+		Enabled:     false, // Temporarily disabled to prevent shutdown loops
 		Critical:    false,
 		Interval:    ehc.config.ElectionProbeInterval,
 		Timeout:     ehc.config.ElectionProbeTimeout,

vendor/github.com/sony/gobreaker/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright 2015 Sony Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/sony/gobreaker/README.md

@@ -0,0 +1,132 @@
+gobreaker
+=========
+
+[![GoDoc](https://godoc.org/github.com/sony/gobreaker?status.svg)](http://godoc.org/github.com/sony/gobreaker)
+
+[gobreaker][repo-url] implements the [Circuit Breaker pattern](https://msdn.microsoft.com/en-us/library/dn589784.aspx) in Go.
+
+Installation
+------------
+
+```
+go get github.com/sony/gobreaker
+```
+
+Usage
+-----
+
+The struct `CircuitBreaker` is a state machine to prevent sending requests that are likely to fail.
+The function `NewCircuitBreaker` creates a new `CircuitBreaker`.
+
+```go
+func NewCircuitBreaker(st Settings) *CircuitBreaker
+```
+
+You can configure `CircuitBreaker` by the struct `Settings`:
+
+```go
+type Settings struct {
+	Name          string
+	MaxRequests   uint32
+	Interval      time.Duration
+	Timeout       time.Duration
+	ReadyToTrip   func(counts Counts) bool
+	OnStateChange func(name string, from State, to State)
+	IsSuccessful  func(err error) bool
+}
+```
+
+- `Name` is the name of the `CircuitBreaker`.
+
+- `MaxRequests` is the maximum number of requests allowed to pass through
+  when the `CircuitBreaker` is half-open.
+  If `MaxRequests` is 0, `CircuitBreaker` allows only 1 request.
+
+- `Interval` is the cyclic period of the closed state
+  for `CircuitBreaker` to clear the internal `Counts`, described later in this section.
+  If `Interval` is 0, `CircuitBreaker` doesn't clear the internal `Counts` during the closed state.
+
+- `Timeout` is the period of the open state,
+  after which the state of `CircuitBreaker` becomes half-open.
+  If `Timeout` is 0, the timeout value of `CircuitBreaker` is set to 60 seconds.
+
+- `ReadyToTrip` is called with a copy of `Counts` whenever a request fails in the closed state.
+  If `ReadyToTrip` returns true, `CircuitBreaker` will be placed into the open state.
+  If `ReadyToTrip` is `nil`, default `ReadyToTrip` is used.
+  Default `ReadyToTrip` returns true when the number of consecutive failures is more than 5.
+
+- `OnStateChange` is called whenever the state of `CircuitBreaker` changes.
+
+- `IsSuccessful` is called with the error returned from a request.
+  If `IsSuccessful` returns true, the error is counted as a success.
+  Otherwise the error is counted as a failure.
+  If `IsSuccessful` is nil, default `IsSuccessful` is used, which returns false for all non-nil errors.
+
+The struct `Counts` holds the numbers of requests and their successes/failures:
+
+```go
+type Counts struct {
+	Requests             uint32
+	TotalSuccesses       uint32
+	TotalFailures        uint32
+	ConsecutiveSuccesses uint32
+	ConsecutiveFailures  uint32
+}
+```
+
+`CircuitBreaker` clears the internal `Counts` either
+on the change of the state or at the closed-state intervals.
+`Counts` ignores the results of the requests sent before clearing.
+
+`CircuitBreaker` can wrap any function to send a request:
+
+```go
+func (cb *CircuitBreaker) Execute(req func() (interface{}, error)) (interface{}, error)
+```
+
+The method `Execute` runs the given request if `CircuitBreaker` accepts it.
+`Execute` returns an error instantly if `CircuitBreaker` rejects the request.
+Otherwise, `Execute` returns the result of the request.
+If a panic occurs in the request, `CircuitBreaker` handles it as an error
+and causes the same panic again.
+
+Example
+-------
+
+```go
+var cb *breaker.CircuitBreaker
+
+func Get(url string) ([]byte, error) {
+	body, err := cb.Execute(func() (interface{}, error) {
+		resp, err := http.Get(url)
+		if err != nil {
+			return nil, err
+		}
+
+		defer resp.Body.Close()
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		return body, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return body.([]byte), nil
+}
+```
+
+See [example](https://github.com/sony/gobreaker/blob/master/example) for details.
+
+License
+-------
+
+The MIT License (MIT)
+
+See [LICENSE](https://github.com/sony/gobreaker/blob/master/LICENSE) for details.
+
+
+[repo-url]: https://github.com/sony/gobreaker

vendor/github.com/sony/gobreaker/gobreaker.go

@@ -0,0 +1,380 @@
+// Package gobreaker implements the Circuit Breaker pattern.
+// See https://msdn.microsoft.com/en-us/library/dn589784.aspx.
+package gobreaker
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// State is a type that represents a state of CircuitBreaker.
+type State int
+
+// These constants are states of CircuitBreaker.
+const (
+	StateClosed State = iota
+	StateHalfOpen
+	StateOpen
+)
+
+var (
+	// ErrTooManyRequests is returned when the CB state is half open and the requests count is over the cb maxRequests
+	ErrTooManyRequests = errors.New("too many requests")
+	// ErrOpenState is returned when the CB state is open
+	ErrOpenState = errors.New("circuit breaker is open")
+)
+
+// String implements stringer interface.
+func (s State) String() string {
+	switch s {
+	case StateClosed:
+		return "closed"
+	case StateHalfOpen:
+		return "half-open"
+	case StateOpen:
+		return "open"
+	default:
+		return fmt.Sprintf("unknown state: %d", s)
+	}
+}
+
+// Counts holds the numbers of requests and their successes/failures.
+// CircuitBreaker clears the internal Counts either
+// on the change of the state or at the closed-state intervals.
+// Counts ignores the results of the requests sent before clearing.
+type Counts struct {
+	Requests             uint32
+	TotalSuccesses       uint32
+	TotalFailures        uint32
+	ConsecutiveSuccesses uint32
+	ConsecutiveFailures  uint32
+}
+
+func (c *Counts) onRequest() {
+	c.Requests++
+}
+
+func (c *Counts) onSuccess() {
+	c.TotalSuccesses++
+	c.ConsecutiveSuccesses++
+	c.ConsecutiveFailures = 0
+}
+
+func (c *Counts) onFailure() {
+	c.TotalFailures++
+	c.ConsecutiveFailures++
+	c.ConsecutiveSuccesses = 0
+}
+
+func (c *Counts) clear() {
+	c.Requests = 0
+	c.TotalSuccesses = 0
+	c.TotalFailures = 0
+	c.ConsecutiveSuccesses = 0
+	c.ConsecutiveFailures = 0
+}
+
+// Settings configures CircuitBreaker:
+//
+// Name is the name of the CircuitBreaker.
+//
+// MaxRequests is the maximum number of requests allowed to pass through
+// when the CircuitBreaker is half-open.
+// If MaxRequests is 0, the CircuitBreaker allows only 1 request.
+//
+// Interval is the cyclic period of the closed state
+// for the CircuitBreaker to clear the internal Counts.
+// If Interval is less than or equal to 0, the CircuitBreaker doesn't clear internal Counts during the closed state.
+//
+// Timeout is the period of the open state,
+// after which the state of the CircuitBreaker becomes half-open.
+// If Timeout is less than or equal to 0, the timeout value of the CircuitBreaker is set to 60 seconds.
+//
+// ReadyToTrip is called with a copy of Counts whenever a request fails in the closed state.
+// If ReadyToTrip returns true, the CircuitBreaker will be placed into the open state.
+// If ReadyToTrip is nil, default ReadyToTrip is used.
+// Default ReadyToTrip returns true when the number of consecutive failures is more than 5.
+//
+// OnStateChange is called whenever the state of the CircuitBreaker changes.
+//
+// IsSuccessful is called with the error returned from a request.
+// If IsSuccessful returns true, the error is counted as a success.
+// Otherwise the error is counted as a failure.
+// If IsSuccessful is nil, default IsSuccessful is used, which returns false for all non-nil errors.
+type Settings struct {
+	Name          string
+	MaxRequests   uint32
+	Interval      time.Duration
+	Timeout       time.Duration
+	ReadyToTrip   func(counts Counts) bool
+	OnStateChange func(name string, from State, to State)
+	IsSuccessful  func(err error) bool
+}
+
+// CircuitBreaker is a state machine to prevent sending requests that are likely to fail.
+type CircuitBreaker struct {
+	name          string
+	maxRequests   uint32
+	interval      time.Duration
+	timeout       time.Duration
+	readyToTrip   func(counts Counts) bool
+	isSuccessful  func(err error) bool
+	onStateChange func(name string, from State, to State)
+
+	mutex      sync.Mutex
+	state      State
+	generation uint64
+	counts     Counts
+	expiry     time.Time
+}
+
+// TwoStepCircuitBreaker is like CircuitBreaker but instead of surrounding a function
+// with the breaker functionality, it only checks whether a request can proceed and
+// expects the caller to report the outcome in a separate step using a callback.
+type TwoStepCircuitBreaker struct {
+	cb *CircuitBreaker
+}
+
+// NewCircuitBreaker returns a new CircuitBreaker configured with the given Settings.
+func NewCircuitBreaker(st Settings) *CircuitBreaker {
+	cb := new(CircuitBreaker)
+
+	cb.name = st.Name
+	cb.onStateChange = st.OnStateChange
+
+	if st.MaxRequests == 0 {
+		cb.maxRequests = 1
+	} else {
+		cb.maxRequests = st.MaxRequests
+	}
+
+	if st.Interval <= 0 {
+		cb.interval = defaultInterval
+	} else {
+		cb.interval = st.Interval
+	}
+
+	if st.Timeout <= 0 {
+		cb.timeout = defaultTimeout
+	} else {
+		cb.timeout = st.Timeout
+	}
+
+	if st.ReadyToTrip == nil {
+		cb.readyToTrip = defaultReadyToTrip
+	} else {
+		cb.readyToTrip = st.ReadyToTrip
+	}
+
+	if st.IsSuccessful == nil {
+		cb.isSuccessful = defaultIsSuccessful
+	} else {
+		cb.isSuccessful = st.IsSuccessful
+	}
+
+	cb.toNewGeneration(time.Now())
+
+	return cb
+}
+
+// NewTwoStepCircuitBreaker returns a new TwoStepCircuitBreaker configured with the given Settings.
+func NewTwoStepCircuitBreaker(st Settings) *TwoStepCircuitBreaker {
+	return &TwoStepCircuitBreaker{
+		cb: NewCircuitBreaker(st),
+	}
+}
+
+const defaultInterval = time.Duration(0) * time.Second
+const defaultTimeout = time.Duration(60) * time.Second
+
+func defaultReadyToTrip(counts Counts) bool {
+	return counts.ConsecutiveFailures > 5
+}
+
+func defaultIsSuccessful(err error) bool {
+	return err == nil
+}
+
+// Name returns the name of the CircuitBreaker.
+func (cb *CircuitBreaker) Name() string {
+	return cb.name
+}
+
+// State returns the current state of the CircuitBreaker.
+func (cb *CircuitBreaker) State() State {
+	cb.mutex.Lock()
+	defer cb.mutex.Unlock()
+
+	now := time.Now()
+	state, _ := cb.currentState(now)
+	return state
+}
+
+// Counts returns internal counters
+func (cb *CircuitBreaker) Counts() Counts {
+	cb.mutex.Lock()
+	defer cb.mutex.Unlock()
+
+	return cb.counts
+}
+
+// Execute runs the given request if the CircuitBreaker accepts it.
+// Execute returns an error instantly if the CircuitBreaker rejects the request.
+// Otherwise, Execute returns the result of the request.
+// If a panic occurs in the request, the CircuitBreaker handles it as an error
+// and causes the same panic again.
+func (cb *CircuitBreaker) Execute(req func() (interface{}, error)) (interface{}, error) {
+	generation, err := cb.beforeRequest()
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		e := recover()
+		if e != nil {
+			cb.afterRequest(generation, false)
+			panic(e)
+		}
+	}()
+
+	result, err := req()
+	cb.afterRequest(generation, cb.isSuccessful(err))
+	return result, err
+}
+
+// Name returns the name of the TwoStepCircuitBreaker.
+func (tscb *TwoStepCircuitBreaker) Name() string {
+	return tscb.cb.Name()
+}
+
+// State returns the current state of the TwoStepCircuitBreaker.
+func (tscb *TwoStepCircuitBreaker) State() State {
+	return tscb.cb.State()
+}
+
+// Counts returns internal counters
+func (tscb *TwoStepCircuitBreaker) Counts() Counts {
+	return tscb.cb.Counts()
+}
+
+// Allow checks if a new request can proceed. It returns a callback that should be used to
+// register the success or failure in a separate step. If the circuit breaker doesn't allow
+// requests, it returns an error.
+func (tscb *TwoStepCircuitBreaker) Allow() (done func(success bool), err error) {
+	generation, err := tscb.cb.beforeRequest()
+	if err != nil {
+		return nil, err
+	}
+
+	return func(success bool) {
+		tscb.cb.afterRequest(generation, success)
+	}, nil
+}
+
+func (cb *CircuitBreaker) beforeRequest() (uint64, error) {
+	cb.mutex.Lock()
+	defer cb.mutex.Unlock()
+
+	now := time.Now()
+	state, generation := cb.currentState(now)
+
+	if state == StateOpen {
+		return generation, ErrOpenState
+	} else if state == StateHalfOpen && cb.counts.Requests >= cb.maxRequests {
+		return generation, ErrTooManyRequests
+	}
+
+	cb.counts.onRequest()
+	return generation, nil
+}
+
+func (cb *CircuitBreaker) afterRequest(before uint64, success bool) {
+	cb.mutex.Lock()
+	defer cb.mutex.Unlock()
+
+	now := time.Now()
+	state, generation := cb.currentState(now)
+	if generation != before {
+		return
+	}
+
+	if success {
+		cb.onSuccess(state, now)
+	} else {
+		cb.onFailure(state, now)
+	}
+}
+
+func (cb *CircuitBreaker) onSuccess(state State, now time.Time) {
+	switch state {
+	case StateClosed:
+		cb.counts.onSuccess()
+	case StateHalfOpen:
+		cb.counts.onSuccess()
+		if cb.counts.ConsecutiveSuccesses >= cb.maxRequests {
+			cb.setState(StateClosed, now)
+		}
+	}
+}
+
+func (cb *CircuitBreaker) onFailure(state State, now time.Time) {
+	switch state {
+	case StateClosed:
+		cb.counts.onFailure()
+		if cb.readyToTrip(cb.counts) {
+			cb.setState(StateOpen, now)
+		}
+	case StateHalfOpen:
+		cb.setState(StateOpen, now)
+	}
+}
+
+func (cb *CircuitBreaker) currentState(now time.Time) (State, uint64) {
+	switch cb.state {
+	case StateClosed:
+		if !cb.expiry.IsZero() && cb.expiry.Before(now) {
+			cb.toNewGeneration(now)
+		}
+	case StateOpen:
+		if cb.expiry.Before(now) {
+			cb.setState(StateHalfOpen, now)
+		}
+	}
+	return cb.state, cb.generation
+}
+
+func (cb *CircuitBreaker) setState(state State, now time.Time) {
+	if cb.state == state {
+		return
+	}
+
+	prev := cb.state
+	cb.state = state
+
+	cb.toNewGeneration(now)
+
+	if cb.onStateChange != nil {
+		cb.onStateChange(cb.name, prev, state)
+	}
+}
+
+func (cb *CircuitBreaker) toNewGeneration(now time.Time) {
+	cb.generation++
+	cb.counts.clear()
+
+	var zero time.Time
+	switch cb.state {
+	case StateClosed:
+		if cb.interval == 0 {
+			cb.expiry = zero
+		} else {
+			cb.expiry = now.Add(cb.interval)
+		}
+	case StateOpen:
+		cb.expiry = now.Add(cb.timeout)
+	default: // StateHalfOpen
+		cb.expiry = zero
+	}
+}

vendor/modules.txt

@@ -123,7 +123,7 @@ github.com/blevesearch/zapx/v16
 # github.com/cespare/xxhash/v2 v2.2.0
 ## explicit; go 1.11
 github.com/cespare/xxhash/v2
-# github.com/chorus-services/backbeat v0.0.0-00010101000000-000000000000 => /home/tony/chorus/project-queues/active/BACKBEAT/backbeat/prototype
+# github.com/chorus-services/backbeat v0.0.0-00010101000000-000000000000 => ../BACKBEAT/backbeat/prototype
 ## explicit; go 1.22
 github.com/chorus-services/backbeat/pkg/sdk
 # github.com/containerd/cgroups v1.1.0
@@ -614,6 +614,9 @@ github.com/robfig/cron/v3
 github.com/sashabaranov/go-openai
 github.com/sashabaranov/go-openai/internal
 github.com/sashabaranov/go-openai/jsonschema
+# github.com/sony/gobreaker v0.5.0
+## explicit; go 1.12
+github.com/sony/gobreaker
 # github.com/spaolacci/murmur3 v1.1.0
 ## explicit
 github.com/spaolacci/murmur3
@@ -844,4 +847,4 @@ gopkg.in/yaml.v3
 # lukechampine.com/blake3 v1.2.1
 ## explicit; go 1.17
 lukechampine.com/blake3
-# github.com/chorus-services/backbeat => /home/tony/chorus/project-queues/active/BACKBEAT/backbeat/prototype
+# github.com/chorus-services/backbeat => ../BACKBEAT/backbeat/prototype