feat: Production readiness improvements for WHOOSH council formation

Major security, observability, and configuration improvements: ## Security Hardening - Implemented configurable CORS (no more wildcards) - Added comprehensive auth middleware for admin endpoints - Enhanced webhook HMAC validation - Added input validation and rate limiting - Security headers and CSP policies ## Configuration Management - Made N8N webhook URL configurable (WHOOSH_N8N_BASE_URL) - Replaced all hardcoded endpoints with environment variables - Added feature flags for LLM vs heuristic composition - Gitea fetch hardening with EAGER_FILTER and FULL_RESCAN options ## API Completeness - Implemented GetCouncilComposition function - Added GET /api/v1/councils/{id} endpoint - Council artifacts API (POST/GET /api/v1/councils/{id}/artifacts) - /admin/health/details endpoint with component status - Database lookup for repository URLs (no hardcoded fallbacks) ## Observability & Performance - Added OpenTelemetry distributed tracing with goal/pulse correlation - Performance optimization database indexes - Comprehensive health monitoring - Enhanced logging and error handling ## Infrastructure - Production-ready P2P discovery (replaces mock implementation) - Removed unused Redis configuration - Enhanced Docker Swarm integration - Added migration files for performance indexes ## Code Quality - Comprehensive input validation - Graceful error handling and failsafe fallbacks - Backwards compatibility maintained - Following security best practices 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-09-12 20:34:17 +10:00
parent 56ea52b743
commit 131868bdca
1740 changed files with 575904 additions and 171 deletions
--- a/internal/auth/middleware.go
+++ b/internal/auth/middleware.go
@@ -0,0 +1,192 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/rs/zerolog/log"
+)
+
+type contextKey string
+
+const (
+	UserKey    contextKey = "user"
+	ServiceKey contextKey = "service"
+)
+
+type Middleware struct {
+	jwtSecret     string
+	serviceTokens []string
+}
+
+func NewMiddleware(jwtSecret string, serviceTokens []string) *Middleware {
+	return &Middleware{
+		jwtSecret:     jwtSecret,
+		serviceTokens: serviceTokens,
+	}
+}
+
+// AuthRequired checks for either JWT token or service token
+func (m *Middleware) AuthRequired(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Check Authorization header
+		authHeader := r.Header.Get("Authorization")
+		if authHeader == "" {
+			http.Error(w, "Authorization header required", http.StatusUnauthorized)
+			return
+		}
+
+		// Parse Bearer token
+		parts := strings.SplitN(authHeader, " ", 2)
+		if len(parts) != 2 || parts[0] != "Bearer" {
+			http.Error(w, "Invalid authorization format. Use Bearer token", http.StatusUnauthorized)
+			return
+		}
+
+		token := parts[1]
+
+		// Try service token first (faster check)
+		if m.isValidServiceToken(token) {
+			ctx := context.WithValue(r.Context(), ServiceKey, true)
+			next.ServeHTTP(w, r.WithContext(ctx))
+			return
+		}
+
+		// Try JWT token
+		claims, err := m.validateJWT(token)
+		if err != nil {
+			log.Warn().Err(err).Msg("Invalid JWT token")
+			http.Error(w, "Invalid token", http.StatusUnauthorized)
+			return
+		}
+
+		// Add user info to context
+		ctx := context.WithValue(r.Context(), UserKey, claims)
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+// ServiceTokenRequired checks for valid service token only (for internal services)
+func (m *Middleware) ServiceTokenRequired(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		authHeader := r.Header.Get("Authorization")
+		if authHeader == "" {
+			http.Error(w, "Service authorization required", http.StatusUnauthorized)
+			return
+		}
+
+		parts := strings.SplitN(authHeader, " ", 2)
+		if len(parts) != 2 || parts[0] != "Bearer" {
+			http.Error(w, "Invalid authorization format", http.StatusUnauthorized)
+			return
+		}
+
+		if !m.isValidServiceToken(parts[1]) {
+			http.Error(w, "Invalid service token", http.StatusUnauthorized)
+			return
+		}
+
+		ctx := context.WithValue(r.Context(), ServiceKey, true)
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+// AdminRequired checks for JWT token with admin permissions
+func (m *Middleware) AdminRequired(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		authHeader := r.Header.Get("Authorization")
+		if authHeader == "" {
+			http.Error(w, "Admin authorization required", http.StatusUnauthorized)
+			return
+		}
+
+		parts := strings.SplitN(authHeader, " ", 2)
+		if len(parts) != 2 || parts[0] != "Bearer" {
+			http.Error(w, "Invalid authorization format", http.StatusUnauthorized)
+			return
+		}
+
+		token := parts[1]
+
+		// Service tokens have admin privileges
+		if m.isValidServiceToken(token) {
+			ctx := context.WithValue(r.Context(), ServiceKey, true)
+			next.ServeHTTP(w, r.WithContext(ctx))
+			return
+		}
+
+		// Check JWT for admin role
+		claims, err := m.validateJWT(token)
+		if err != nil {
+			log.Warn().Err(err).Msg("Invalid JWT token for admin access")
+			http.Error(w, "Invalid admin token", http.StatusUnauthorized)
+			return
+		}
+
+		// Check if user has admin role
+		if role, ok := claims["role"].(string); !ok || role != "admin" {
+			http.Error(w, "Admin privileges required", http.StatusForbidden)
+			return
+		}
+
+		ctx := context.WithValue(r.Context(), UserKey, claims)
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func (m *Middleware) isValidServiceToken(token string) bool {
+	for _, serviceToken := range m.serviceTokens {
+		if serviceToken == token {
+			return true
+		}
+	}
+	return false
+}
+
+func (m *Middleware) validateJWT(tokenString string) (jwt.MapClaims, error) {
+	token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
+		// Validate signing method
+		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
+		}
+		return []byte(m.jwtSecret), nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	if !token.Valid {
+		return nil, fmt.Errorf("invalid token")
+	}
+
+	claims, ok := token.Claims.(jwt.MapClaims)
+	if !ok {
+		return nil, fmt.Errorf("invalid claims")
+	}
+
+	// Check expiration
+	if exp, ok := claims["exp"].(float64); ok {
+		if time.Unix(int64(exp), 0).Before(time.Now()) {
+			return nil, fmt.Errorf("token expired")
+		}
+	}
+
+	return claims, nil
+}
+
+// GetUserFromContext retrieves user claims from request context
+func GetUserFromContext(ctx context.Context) (jwt.MapClaims, bool) {
+	claims, ok := ctx.Value(UserKey).(jwt.MapClaims)
+	return claims, ok
+}
+
+// IsServiceRequest checks if request is from a service token
+func IsServiceRequest(ctx context.Context) bool {
+	service, ok := ctx.Value(ServiceKey).(bool)
+	return ok && service
+}
--- a/internal/auth/ratelimit.go
+++ b/internal/auth/ratelimit.go
@@ -0,0 +1,145 @@
+package auth
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+// RateLimiter implements a simple in-memory rate limiter
+type RateLimiter struct {
+	mu       sync.RWMutex
+	buckets  map[string]*bucket
+	requests int
+	window   time.Duration
+	cleanup  time.Duration
+}
+
+type bucket struct {
+	count     int
+	lastReset time.Time
+}
+
+// NewRateLimiter creates a new rate limiter
+func NewRateLimiter(requests int, window time.Duration) *RateLimiter {
+	rl := &RateLimiter{
+		buckets:  make(map[string]*bucket),
+		requests: requests,
+		window:   window,
+		cleanup:  window * 2,
+	}
+	
+	// Start cleanup goroutine
+	go rl.cleanupRoutine()
+	
+	return rl
+}
+
+// Allow checks if a request should be allowed
+func (rl *RateLimiter) Allow(key string) bool {
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+	
+	now := time.Now()
+	
+	// Get or create bucket
+	b, exists := rl.buckets[key]
+	if !exists {
+		rl.buckets[key] = &bucket{
+			count:     1,
+			lastReset: now,
+		}
+		return true
+	}
+	
+	// Check if window has expired
+	if now.Sub(b.lastReset) > rl.window {
+		b.count = 1
+		b.lastReset = now
+		return true
+	}
+	
+	// Check if limit exceeded
+	if b.count >= rl.requests {
+		return false
+	}
+	
+	// Increment counter
+	b.count++
+	return true
+}
+
+// cleanupRoutine periodically removes old buckets
+func (rl *RateLimiter) cleanupRoutine() {
+	ticker := time.NewTicker(rl.cleanup)
+	defer ticker.Stop()
+	
+	for range ticker.C {
+		rl.mu.Lock()
+		now := time.Now()
+		for key, bucket := range rl.buckets {
+			if now.Sub(bucket.lastReset) > rl.cleanup {
+				delete(rl.buckets, key)
+			}
+		}
+		rl.mu.Unlock()
+	}
+}
+
+// RateLimitMiddleware creates a rate limiting middleware
+func (rl *RateLimiter) RateLimitMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Use IP address as the key
+		key := getClientIP(r)
+		
+		if !rl.Allow(key) {
+			log.Warn().
+				Str("client_ip", key).
+				Str("path", r.URL.Path).
+				Msg("Rate limit exceeded")
+			
+			w.Header().Set("X-RateLimit-Limit", fmt.Sprintf("%d", rl.requests))
+			w.Header().Set("X-RateLimit-Window", rl.window.String())
+			w.Header().Set("Retry-After", rl.window.String())
+			
+			http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
+			return
+		}
+		
+		next.ServeHTTP(w, r)
+	})
+}
+
+// getClientIP extracts the real client IP address
+func getClientIP(r *http.Request) string {
+	// Check X-Forwarded-For header (when behind proxy)
+	xff := r.Header.Get("X-Forwarded-For")
+	if xff != "" {
+		// Take the first IP in case of multiple
+		if idx := len(xff); idx > 0 {
+			if commaIdx := 0; commaIdx < idx {
+				for i, char := range xff {
+					if char == ',' {
+						commaIdx = i
+						break
+					}
+				}
+				if commaIdx > 0 {
+					return xff[:commaIdx]
+				}
+			}
+			return xff
+		}
+	}
+	
+	// Check X-Real-IP header
+	if xri := r.Header.Get("X-Real-IP"); xri != "" {
+		return xri
+	}
+	
+	// Fall back to RemoteAddr
+	return r.RemoteAddr
+}