// Package slurp provides contextual intelligence capabilities for CHORUS. // // SLURP (Storage, Logic, Understanding, Retrieval, Processing) implements: // - Hierarchical context resolution with bounded depth traversal // - Decision-hop temporal analysis for tracking conceptual evolution // - Distributed storage with role-based encryption // - Context generation restricted to elected admin nodes // - Real-time context evolution tracking and validation // // Architecture: // - context/ Hierarchical context resolution and caching // - temporal/ Decision-based temporal evolution tracking // - storage/ Distributed encrypted storage layer // - intelligence/ Context generation and analysis (admin-only) // - retrieval/ Context search and query interfaces // // Integration Points: // - pkg/dht Distributed hash table for context storage // - pkg/crypto Role-based encryption for access control // - pkg/election Admin-only operations coordination // - pkg/config Configuration extension for SLURP settings // // This package follows established CHORUS patterns for interfaces, error handling, // and distributed operations while providing native Go implementations of the // contextual intelligence capabilities originally prototyped in Python. package slurp import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "sync" "time" "chorus/pkg/config" "chorus/pkg/crypto" "chorus/pkg/dht" "chorus/pkg/election" slurpContext "chorus/pkg/slurp/context" "chorus/pkg/slurp/storage" "chorus/pkg/slurp/temporal" "chorus/pkg/ucxl" ) const contextStoragePrefix = "slurp:context:" var errContextNotPersisted = errors.New("slurp context not persisted") // SLURP is the main coordinator for contextual intelligence operations. // // It orchestrates the interaction between context resolution, temporal analysis, // distributed storage, and intelligence generation while enforcing security // and access controls through integration with existing CHORUS systems. // // Thread Safety: SLURP is safe for concurrent use across multiple goroutines. // All public methods handle synchronization internally. type SLURP struct { // Configuration and dependencies config *config.Config dht dht.DHT crypto *crypto.AgeCrypto election *election.ElectionManager nodeID string // Roadmap: SEC-SLURP 1.1 persistent storage wiring storagePath string localStorage storage.LocalStorage // Core components contextResolver ContextResolver temporalGraph TemporalGraph temporalSystem *temporal.TemporalGraphSystem temporalStore storage.ContextStore storage DistributedStorage intelligence ContextGenerator retrieval QueryEngine // State management mu sync.RWMutex initialized bool adminMode bool currentAdmin string // SEC-SLURP 1.1: lightweight in-memory context persistence contextsMu sync.RWMutex contextCache map[string]*slurpContext.ContextNode resolvedCache map[string]*slurpContext.ResolvedContext contextBackend storage.ContextStore distributedStorage storage.DistributedStorage cacheManager storage.CacheManager indexManager storage.IndexManager backupManager storage.BackupManager eventNotifier storage.EventNotifier // Background processing ctx context.Context cancel context.CancelFunc backgroundTasks sync.WaitGroup // Performance monitoring metrics *SLURPMetrics // Event handling eventHandlers map[EventType][]EventHandler eventMux sync.RWMutex } // ContextPersister exposes the persistence contract used by leader workflows (SEC-SLURP 1.1). type ContextPersister interface { UpsertContext(ctx context.Context, node *slurpContext.ContextNode) (*slurpContext.ResolvedContext, error) } // SLURPConfig holds SLURP-specific configuration that extends the main CHORUS config type SLURPConfig struct { // Enable/disable SLURP system Enabled bool `yaml:"enabled" json:"enabled"` // Context resolution settings ContextResolution ContextResolutionConfig `yaml:"context_resolution" json:"context_resolution"` // Temporal analysis settings TemporalAnalysis TemporalAnalysisConfig `yaml:"temporal_analysis" json:"temporal_analysis"` // Storage configuration Storage SLURPStorageConfig `yaml:"storage" json:"storage"` // Intelligence/generation settings Intelligence IntelligenceConfig `yaml:"intelligence" json:"intelligence"` // Performance tuning Performance PerformanceConfig `yaml:"performance" json:"performance"` // Security settings Security SLURPSecurityConfig `yaml:"security" json:"security"` } // ContextResolutionConfig configures hierarchical context resolution type ContextResolutionConfig struct { // Bounded traversal settings MaxHierarchyDepth int `yaml:"max_hierarchy_depth" json:"max_hierarchy_depth"` DefaultDepthLimit int `yaml:"default_depth_limit" json:"default_depth_limit"` // Caching configuration CacheTTL time.Duration `yaml:"cache_ttl" json:"cache_ttl"` CacheMaxSize int64 `yaml:"cache_max_size" json:"cache_max_size"` CacheMaxEntries int `yaml:"cache_max_entries" json:"cache_max_entries"` // Resolution behavior RequireStrictMatching bool `yaml:"require_strict_matching" json:"require_strict_matching"` AllowPartialResolution bool `yaml:"allow_partial_resolution" json:"allow_partial_resolution"` MinConfidenceThreshold float64 `yaml:"min_confidence_threshold" json:"min_confidence_threshold"` // Global context settings EnableGlobalContexts bool `yaml:"enable_global_contexts" json:"enable_global_contexts"` GlobalContextTTL time.Duration `yaml:"global_context_ttl" json:"global_context_ttl"` } // TemporalAnalysisConfig configures decision-based temporal evolution tracking type TemporalAnalysisConfig struct { // Decision hop analysis MaxDecisionHops int `yaml:"max_decision_hops" json:"max_decision_hops"` DefaultHopLimit int `yaml:"default_hop_limit" json:"default_hop_limit"` // Temporal navigation EnableNavigation bool `yaml:"enable_navigation" json:"enable_navigation"` MaxNavigationHistory int `yaml:"max_navigation_history" json:"max_navigation_history"` // Staleness detection StalenessThreshold float64 `yaml:"staleness_threshold" json:"staleness_threshold"` StalenessCheckInterval time.Duration `yaml:"staleness_check_interval" json:"staleness_check_interval"` // Decision analysis MinDecisionConfidence float64 `yaml:"min_decision_confidence" json:"min_decision_confidence"` MaxDecisionAge time.Duration `yaml:"max_decision_age" json:"max_decision_age"` // Influence propagation EnableInfluencePropagation bool `yaml:"enable_influence_propagation" json:"enable_influence_propagation"` MaxInfluenceDepth int `yaml:"max_influence_depth" json:"max_influence_depth"` } // SLURPStorageConfig configures distributed storage behavior type SLURPStorageConfig struct { // Storage backend Backend string `yaml:"backend" json:"backend"` // "dht", "hybrid" // Encryption settings DefaultEncryption bool `yaml:"default_encryption" json:"default_encryption"` EncryptionRoles []string `yaml:"encryption_roles" json:"encryption_roles"` // Persistence LocalCacheEnabled bool `yaml:"local_cache_enabled" json:"local_cache_enabled"` LocalCachePath string `yaml:"local_cache_path" json:"local_cache_path"` LocalCacheMaxSize int64 `yaml:"local_cache_max_size" json:"local_cache_max_size"` // Synchronization SyncInterval time.Duration `yaml:"sync_interval" json:"sync_interval"` SyncTimeout time.Duration `yaml:"sync_timeout" json:"sync_timeout"` ConflictResolution string `yaml:"conflict_resolution" json:"conflict_resolution"` // Replication ReplicationFactor int `yaml:"replication_factor" json:"replication_factor"` ConsistencyLevel string `yaml:"consistency_level" json:"consistency_level"` } // IntelligenceConfig configures context generation and analysis type IntelligenceConfig struct { // Generation settings (admin-only) EnableGeneration bool `yaml:"enable_generation" json:"enable_generation"` GenerationTimeout time.Duration `yaml:"generation_timeout" json:"generation_timeout"` GenerationConcurrency int `yaml:"generation_concurrency" json:"generation_concurrency"` // Analysis settings EnableAnalysis bool `yaml:"enable_analysis" json:"enable_analysis"` AnalysisInterval time.Duration `yaml:"analysis_interval" json:"analysis_interval"` // Pattern detection EnablePatternDetection bool `yaml:"enable_pattern_detection" json:"enable_pattern_detection"` PatternMatchThreshold float64 `yaml:"pattern_match_threshold" json:"pattern_match_threshold"` // Quality metrics QualityThreshold float64 `yaml:"quality_threshold" json:"quality_threshold"` EnableQualityMetrics bool `yaml:"enable_quality_metrics" json:"enable_quality_metrics"` // External integrations RAGEndpoint string `yaml:"rag_endpoint" json:"rag_endpoint"` RAGTimeout time.Duration `yaml:"rag_timeout" json:"rag_timeout"` } // PerformanceConfig configures performance optimization settings type PerformanceConfig struct { // Concurrency limits MaxConcurrentResolutions int `yaml:"max_concurrent_resolutions" json:"max_concurrent_resolutions"` MaxConcurrentGenerations int `yaml:"max_concurrent_generations" json:"max_concurrent_generations"` MaxConcurrentQueries int `yaml:"max_concurrent_queries" json:"max_concurrent_queries"` // Timeout settings DefaultRequestTimeout time.Duration `yaml:"default_request_timeout" json:"default_request_timeout"` BackgroundTaskTimeout time.Duration `yaml:"background_task_timeout" json:"background_task_timeout"` HealthCheckTimeout time.Duration `yaml:"health_check_timeout" json:"health_check_timeout"` // Resource limits MaxMemoryUsage int64 `yaml:"max_memory_usage" json:"max_memory_usage"` MaxDiskUsage int64 `yaml:"max_disk_usage" json:"max_disk_usage"` // Batch processing DefaultBatchSize int `yaml:"default_batch_size" json:"default_batch_size"` MaxBatchSize int `yaml:"max_batch_size" json:"max_batch_size"` BatchTimeout time.Duration `yaml:"batch_timeout" json:"batch_timeout"` // Monitoring EnableMetrics bool `yaml:"enable_metrics" json:"enable_metrics"` MetricsCollectionInterval time.Duration `yaml:"metrics_collection_interval" json:"metrics_collection_interval"` } // SLURPSecurityConfig configures security-specific settings type SLURPSecurityConfig struct { // Access control EnforceRoleBasedAccess bool `yaml:"enforce_role_based_access" json:"enforce_role_based_access"` DefaultAccessRoles []string `yaml:"default_access_roles" json:"default_access_roles"` AdminOnlyOperations []string `yaml:"admin_only_operations" json:"admin_only_operations"` // Audit and logging EnableAuditLog bool `yaml:"enable_audit_log" json:"enable_audit_log"` AuditLogPath string `yaml:"audit_log_path" json:"audit_log_path"` LogSensitiveOperations bool `yaml:"log_sensitive_operations" json:"log_sensitive_operations"` // Encryption RequireEncryption bool `yaml:"require_encryption" json:"require_encryption"` EncryptionAlgorithm string `yaml:"encryption_algorithm" json:"encryption_algorithm"` KeyRotationInterval time.Duration `yaml:"key_rotation_interval" json:"key_rotation_interval"` // Rate limiting EnableRateLimiting bool `yaml:"enable_rate_limiting" json:"enable_rate_limiting"` DefaultRateLimit int `yaml:"default_rate_limit" json:"default_rate_limit"` BurstLimit int `yaml:"burst_limit" json:"burst_limit"` } // SLURPMetrics holds performance and operational metrics type SLURPMetrics struct { // Resolution metrics TotalResolutions int64 `json:"total_resolutions"` SuccessfulResolutions int64 `json:"successful_resolutions"` FailedResolutions int64 `json:"failed_resolutions"` AverageResolutionTime time.Duration `json:"average_resolution_time"` CacheHitRate float64 `json:"cache_hit_rate"` CacheHits int64 `json:"cache_hits"` CacheMisses int64 `json:"cache_misses"` PersistenceErrors int64 `json:"persistence_errors"` // Temporal metrics TemporalNodes int64 `json:"temporal_nodes"` DecisionPaths int64 `json:"decision_paths"` InfluenceRelationships int64 `json:"influence_relationships"` StaleContexts int64 `json:"stale_contexts"` // Storage metrics StoredContexts int64 `json:"stored_contexts"` EncryptedContexts int64 `json:"encrypted_contexts"` StorageUtilization float64 `json:"storage_utilization"` SyncOperations int64 `json:"sync_operations"` // Intelligence metrics GenerationRequests int64 `json:"generation_requests"` SuccessfulGenerations int64 `json:"successful_generations"` AnalysisOperations int64 `json:"analysis_operations"` PatternMatches int64 `json:"pattern_matches"` // Performance metrics ActiveConnections int64 `json:"active_connections"` MemoryUsage int64 `json:"memory_usage"` DiskUsage int64 `json:"disk_usage"` // Error metrics AuthenticationErrors int64 `json:"authentication_errors"` AuthorizationErrors int64 `json:"authorization_errors"` TimeoutErrors int64 `json:"timeout_errors"` // Timestamp LastUpdated time.Time `json:"last_updated"` } // EventType represents different types of SLURP events type EventType string const ( EventContextResolved EventType = "context_resolved" EventContextGenerated EventType = "context_generated" EventContextEvolved EventType = "context_evolved" EventDecisionRecorded EventType = "decision_recorded" EventInfluenceAdded EventType = "influence_added" EventAdminChanged EventType = "admin_changed" EventStalenessDetected EventType = "staleness_detected" EventCacheInvalidated EventType = "cache_invalidated" EventStorageSynced EventType = "storage_synced" EventPatternDetected EventType = "pattern_detected" EventErrorOccurred EventType = "error_occurred" ) // EventHandler defines the interface for handling SLURP events type EventHandler func(ctx context.Context, event *SLURPEvent) error // SLURPEvent represents an event within the SLURP system type SLURPEvent struct { Type EventType `json:"type"` Timestamp time.Time `json:"timestamp"` Source string `json:"source"` Data map[string]interface{} `json:"data"` Context map[string]string `json:"context,omitempty"` } // NewSLURP creates a new SLURP instance with the provided dependencies. // // The SLURP system requires integration with existing CHORUS components: // - config: Main CHORUS configuration including SLURP settings // - dhtInstance: Distributed hash table for storage and discovery // - cryptoInstance: Role-based encryption for access control // - electionManager: Admin election coordination for restricted operations // // Returns an initialized but not yet started SLURP instance. func NewSLURP( config *config.Config, dhtInstance dht.DHT, cryptoInstance *crypto.AgeCrypto, electionManager *election.ElectionManager, ) (*SLURP, error) { if config == nil { return nil, fmt.Errorf("config is required") } if dhtInstance == nil { return nil, fmt.Errorf("DHT instance is required") } if cryptoInstance == nil { return nil, fmt.Errorf("crypto instance is required") } if electionManager == nil { return nil, fmt.Errorf("election manager is required") } // Validate SLURP configuration if err := validateSLURPConfig(&config.Slurp); err != nil { return nil, fmt.Errorf("invalid SLURP configuration: %w", err) } ctx, cancel := context.WithCancel(context.Background()) storagePath := defaultStoragePath(config) nodeID := config.Agent.ID if nodeID == "" { nodeID = fmt.Sprintf("slurp-node-%d", time.Now().UnixNano()) } slurp := &SLURP{ config: config, dht: dhtInstance, crypto: cryptoInstance, election: electionManager, nodeID: nodeID, ctx: ctx, cancel: cancel, metrics: &SLURPMetrics{LastUpdated: time.Now()}, eventHandlers: make(map[EventType][]EventHandler), contextCache: make(map[string]*slurpContext.ContextNode), resolvedCache: make(map[string]*slurpContext.ResolvedContext), storagePath: storagePath, } return slurp, nil } // Initialize initializes all SLURP components and prepares for operation. // // This method must be called before using any SLURP functionality. // It sets up the context resolver, temporal graph, storage layer, // intelligence system, and retrieval engine. // // The initialization process includes: // - Loading existing context hierarchies from storage // - Setting up encryption keys and access controls // - Initializing caches and performance optimizations // - Starting background maintenance tasks // // Returns an error if initialization fails. func (s *SLURP) Initialize(ctx context.Context) error { s.mu.Lock() defer s.mu.Unlock() if s.initialized { return fmt.Errorf("SLURP already initialized") } // Check if SLURP is enabled if !s.config.Slurp.Enabled { return fmt.Errorf("SLURP is disabled in configuration") } // Establish runtime context for background operations if ctx != nil { if s.cancel != nil { s.cancel() } s.ctx, s.cancel = context.WithCancel(ctx) } else if s.ctx == nil { s.ctx, s.cancel = context.WithCancel(context.Background()) } // Ensure metrics structure is available if s.metrics == nil { s.metrics = &SLURPMetrics{} } s.metrics.LastUpdated = time.Now() // Initialize in-memory persistence (SEC-SLURP 1.1 bootstrap) s.contextsMu.Lock() if s.contextCache == nil { s.contextCache = make(map[string]*slurpContext.ContextNode) } if s.resolvedCache == nil { s.resolvedCache = make(map[string]*slurpContext.ResolvedContext) } s.contextsMu.Unlock() // Roadmap: SEC-SLURP 1.1 persistent storage bootstrapping if err := s.setupPersistentStorage(); err != nil { return fmt.Errorf("failed to initialize SLURP storage: %w", err) } if err := s.loadPersistedContexts(s.ctx); err != nil { return fmt.Errorf("failed to load persisted contexts: %w", err) } if err := s.initializeTemporalSystem(s.ctx); err != nil { return fmt.Errorf("failed to initialize temporal system: %w", err) } // TODO: Initialize components in dependency order // 1. Initialize storage layer first // 2. Initialize context resolver with storage // 3. Initialize temporal graph with storage // 4. Initialize intelligence with resolver and temporal // 5. Initialize retrieval with all components // 6. Set up event handlers and background tasks // Set admin status based on current election state s.updateAdminStatus() // Set up election event handlers if err := s.setupElectionHandlers(); err != nil { return fmt.Errorf("failed to setup election handlers: %w", err) } // Start background tasks s.startBackgroundTasks() s.initialized = true // Emit initialization event s.emitEvent(EventContextResolved, map[string]interface{}{ "action": "system_initialized", "admin_mode": s.adminMode, "current_admin": s.currentAdmin, }) return nil } // Resolve resolves context for a UCXL address using hierarchical inheritance. // // This is the primary method for context resolution, implementing bounded // hierarchy traversal with caching and role-based access control. // // Parameters: // // ctx: Request context for cancellation and timeouts // ucxlAddress: The UCXL address to resolve context for // // Returns: // // *ResolvedContext: Complete resolved context with metadata // error: Any error during resolution // // The resolution process: // 1. Validates the UCXL address format // 2. Checks cache for existing resolution // 3. Performs bounded hierarchy traversal // 4. Applies global contexts if enabled // 5. Encrypts result based on current role permissions // 6. Caches the result for future requests func (s *SLURP) Resolve(ctx context.Context, ucxlAddress string) (*ResolvedContext, error) { if !s.initialized { return nil, fmt.Errorf("SLURP not initialized") } start := time.Now() parsed, err := ucxl.Parse(ucxlAddress) if err != nil { return nil, fmt.Errorf("invalid UCXL address: %w", err) } key := parsed.String() s.contextsMu.RLock() if resolved, ok := s.resolvedCache[key]; ok { s.contextsMu.RUnlock() s.markCacheHit() s.markResolutionSuccess(time.Since(start)) return convertResolvedForAPI(resolved), nil } s.contextsMu.RUnlock() node := s.getContextNode(key) if node == nil { // Roadmap: SEC-SLURP 1.1 - fallback to persistent storage when caches miss. loadedNode, loadErr := s.loadContextForKey(ctx, key) if loadErr != nil { s.markResolutionFailure() if !errors.Is(loadErr, errContextNotPersisted) { s.markPersistenceError() } if errors.Is(loadErr, errContextNotPersisted) { return nil, fmt.Errorf("context not found for %s", key) } return nil, fmt.Errorf("failed to load context for %s: %w", key, loadErr) } node = loadedNode s.markCacheMiss() } else { s.markCacheMiss() } built := buildResolvedContext(node) s.contextsMu.Lock() s.contextCache[key] = node s.resolvedCache[key] = built s.contextsMu.Unlock() s.markResolutionSuccess(time.Since(start)) return convertResolvedForAPI(built), nil } // ResolveWithDepth resolves context with a specific depth limit. // // This method provides fine-grained control over the hierarchy traversal // depth, allowing callers to optimize performance for specific use cases. func (s *SLURP) ResolveWithDepth(ctx context.Context, ucxlAddress string, maxDepth int) (*ResolvedContext, error) { if !s.initialized { return nil, fmt.Errorf("SLURP not initialized") } if maxDepth < 0 { return nil, fmt.Errorf("maxDepth cannot be negative") } resolved, err := s.Resolve(ctx, ucxlAddress) if err != nil { return nil, err } if resolved != nil { resolved.BoundedDepth = maxDepth } return resolved, nil } // BatchResolve efficiently resolves multiple UCXL addresses in parallel. // // This method is optimized for bulk resolution operations with request // deduplication, shared caching, and controlled concurrency. func (s *SLURP) BatchResolve(ctx context.Context, addresses []string) (map[string]*ResolvedContext, error) { if !s.initialized { return nil, fmt.Errorf("SLURP not initialized") } if len(addresses) == 0 { return make(map[string]*ResolvedContext), nil } results := make(map[string]*ResolvedContext, len(addresses)) var firstErr error for _, addr := range addresses { resolved, err := s.Resolve(ctx, addr) if err != nil { if firstErr == nil { firstErr = err } continue } results[addr] = resolved } return results, firstErr } // GetTemporalEvolution retrieves the temporal evolution history for a context. // // This method provides access to decision-based evolution tracking, // showing how context has changed through different decision points. func (s *SLURP) GetTemporalEvolution(ctx context.Context, ucxlAddress string) ([]*TemporalNode, error) { if !s.initialized { return nil, fmt.Errorf("SLURP not initialized") } if s.temporalGraph == nil { return nil, fmt.Errorf("temporal graph not configured") } parsed, err := ucxl.Parse(ucxlAddress) if err != nil { return nil, fmt.Errorf("invalid UCXL address: %w", err) } return s.temporalGraph.GetEvolutionHistory(ctx, parsed.String()) } // NavigateDecisionHops navigates through the decision graph by hop distance. // // This method implements decision-hop based navigation rather than // chronological time-based navigation, allowing exploration of // conceptually related changes. func (s *SLURP) NavigateDecisionHops(ctx context.Context, ucxlAddress string, hops int, direction NavigationDirection) (*TemporalNode, error) { if !s.initialized { return nil, fmt.Errorf("SLURP not initialized") } if s.temporalGraph == nil { return nil, fmt.Errorf("decision navigation not configured") } parsed, err := ucxl.Parse(ucxlAddress) if err != nil { return nil, fmt.Errorf("invalid UCXL address: %w", err) } if navigator, ok := s.temporalGraph.(DecisionNavigator); ok { return navigator.NavigateDecisionHops(ctx, parsed.String(), hops, direction) } return nil, fmt.Errorf("decision navigation not supported by temporal graph") } // GenerateContext generates new context for a path (admin-only operation). // // This method is restricted to admin nodes and provides intelligent // context generation using analysis of file content, structure, and // existing patterns. func (s *SLURP) GenerateContext(ctx context.Context, path string, options *GenerationOptions) (*ContextNode, error) { if !s.initialized { return nil, fmt.Errorf("SLURP not initialized") } // Enforce admin-only restriction if !s.IsCurrentNodeAdmin() { return nil, fmt.Errorf("context generation requires admin privileges") } if s.intelligence == nil { return nil, fmt.Errorf("intelligence engine not configured") } s.mu.Lock() s.metrics.GenerationRequests++ s.metrics.LastUpdated = time.Now() s.mu.Unlock() generated, err := s.intelligence.GenerateContext(ctx, path, options) if err != nil { return nil, err } contextNode, err := convertAPIToContextNode(generated) if err != nil { return nil, err } if _, err := s.UpsertContext(ctx, contextNode); err != nil { return nil, err } return generated, nil } // UpsertContext persists a context node and exposes it for immediate resolution (SEC-SLURP 1.1). func (s *SLURP) UpsertContext(ctx context.Context, node *slurpContext.ContextNode) (*slurpContext.ResolvedContext, error) { if !s.initialized { return nil, fmt.Errorf("SLURP not initialized") } if node == nil { return nil, fmt.Errorf("context node cannot be nil") } if err := node.Validate(); err != nil { return nil, err } clone := node.Clone() resolved := buildResolvedContext(clone) key := clone.UCXLAddress.String() s.contextsMu.Lock() s.contextCache[key] = clone s.resolvedCache[key] = resolved s.contextsMu.Unlock() s.mu.Lock() s.metrics.StoredContexts++ s.metrics.SuccessfulGenerations++ s.metrics.LastUpdated = time.Now() s.mu.Unlock() if err := s.persistContext(ctx, clone); err != nil && !errors.Is(err, errContextNotPersisted) { s.markPersistenceError() s.emitEvent(EventErrorOccurred, map[string]interface{}{ "action": "persist_context", "ucxl_address": key, "error": err.Error(), }) } s.emitEvent(EventContextGenerated, map[string]interface{}{ "ucxl_address": key, "summary": clone.Summary, "path": clone.Path, }) return cloneResolvedInternal(resolved), nil } func buildResolvedContext(node *slurpContext.ContextNode) *slurpContext.ResolvedContext { if node == nil { return nil } return &slurpContext.ResolvedContext{ UCXLAddress: node.UCXLAddress, Summary: node.Summary, Purpose: node.Purpose, Technologies: cloneStringSlice(node.Technologies), Tags: cloneStringSlice(node.Tags), Insights: cloneStringSlice(node.Insights), ContextSourcePath: node.Path, InheritanceChain: []string{node.UCXLAddress.String()}, ResolutionConfidence: node.RAGConfidence, BoundedDepth: 0, GlobalContextsApplied: false, ResolvedAt: time.Now(), } } func cloneResolvedInternal(resolved *slurpContext.ResolvedContext) *slurpContext.ResolvedContext { if resolved == nil { return nil } clone := *resolved clone.Technologies = cloneStringSlice(resolved.Technologies) clone.Tags = cloneStringSlice(resolved.Tags) clone.Insights = cloneStringSlice(resolved.Insights) clone.InheritanceChain = cloneStringSlice(resolved.InheritanceChain) return &clone } func convertResolvedForAPI(resolved *slurpContext.ResolvedContext) *ResolvedContext { if resolved == nil { return nil } return &ResolvedContext{ UCXLAddress: resolved.UCXLAddress.String(), Summary: resolved.Summary, Purpose: resolved.Purpose, Technologies: cloneStringSlice(resolved.Technologies), Tags: cloneStringSlice(resolved.Tags), Insights: cloneStringSlice(resolved.Insights), SourcePath: resolved.ContextSourcePath, InheritanceChain: cloneStringSlice(resolved.InheritanceChain), Confidence: resolved.ResolutionConfidence, BoundedDepth: resolved.BoundedDepth, GlobalApplied: resolved.GlobalContextsApplied, ResolvedAt: resolved.ResolvedAt, Version: 1, LastUpdated: resolved.ResolvedAt, EvolutionHistory: cloneStringSlice(resolved.InheritanceChain), NodesTraversed: len(resolved.InheritanceChain), } } func convertAPIToContextNode(node *ContextNode) (*slurpContext.ContextNode, error) { if node == nil { return nil, fmt.Errorf("context node cannot be nil") } address, err := ucxl.Parse(node.UCXLAddress) if err != nil { return nil, fmt.Errorf("invalid UCXL address: %w", err) } converted := &slurpContext.ContextNode{ Path: node.Path, UCXLAddress: *address, Summary: node.Summary, Purpose: node.Purpose, Technologies: cloneStringSlice(node.Technologies), Tags: cloneStringSlice(node.Tags), Insights: cloneStringSlice(node.Insights), OverridesParent: node.Overrides, ContextSpecificity: node.Specificity, AppliesToChildren: node.AppliesTo == ScopeChildren, GeneratedAt: node.CreatedAt, RAGConfidence: node.Confidence, EncryptedFor: cloneStringSlice(node.EncryptedFor), AccessLevel: slurpContext.RoleAccessLevel(node.AccessLevel), Metadata: cloneMetadata(node.Metadata), } converted.AppliesTo = slurpContext.ContextScope(node.AppliesTo) converted.CreatedBy = node.CreatedBy converted.UpdatedAt = node.UpdatedAt converted.WhoUpdated = node.UpdatedBy converted.Parent = node.Parent converted.Children = cloneStringSlice(node.Children) converted.FileType = node.FileType converted.Language = node.Language converted.Size = node.Size converted.LastModified = node.LastModified converted.ContentHash = node.ContentHash if converted.GeneratedAt.IsZero() { converted.GeneratedAt = time.Now() } if converted.UpdatedAt.IsZero() { converted.UpdatedAt = converted.GeneratedAt } return converted, nil } func cloneStringSlice(src []string) []string { if len(src) == 0 { return nil } dst := make([]string, len(src)) copy(dst, src) return dst } func cloneMetadata(src map[string]interface{}) map[string]interface{} { if len(src) == 0 { return nil } dst := make(map[string]interface{}, len(src)) for k, v := range src { dst[k] = v } return dst } // IsCurrentNodeAdmin returns true if the current node is the elected admin. // // This method is used throughout SLURP to enforce admin-only operations // such as context generation and hierarchy management. func (s *SLURP) IsCurrentNodeAdmin() bool { s.mu.RLock() defer s.mu.RUnlock() return s.adminMode } // GetMetrics returns current SLURP performance and operational metrics. func (s *SLURP) GetMetrics() *SLURPMetrics { s.mu.RLock() defer s.mu.RUnlock() // Return a copy to prevent modification metricsCopy := *s.metrics metricsCopy.LastUpdated = time.Now() return &metricsCopy } // markResolutionSuccess tracks cache or storage hits (Roadmap: SEC-SLURP 1.1). func (s *SLURP) markResolutionSuccess(duration time.Duration) { s.mu.Lock() defer s.mu.Unlock() s.metrics.TotalResolutions++ s.metrics.SuccessfulResolutions++ s.metrics.AverageResolutionTime = updateAverageDuration( s.metrics.AverageResolutionTime, s.metrics.TotalResolutions, duration, ) if s.metrics.TotalResolutions > 0 { s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions) } s.metrics.LastUpdated = time.Now() } // markResolutionFailure tracks lookup failures (Roadmap: SEC-SLURP 1.1). func (s *SLURP) markResolutionFailure() { s.mu.Lock() defer s.mu.Unlock() s.metrics.TotalResolutions++ s.metrics.FailedResolutions++ if s.metrics.TotalResolutions > 0 { s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions) } s.metrics.LastUpdated = time.Now() } func (s *SLURP) markCacheHit() { s.mu.Lock() defer s.mu.Unlock() s.metrics.CacheHits++ if s.metrics.TotalResolutions > 0 { s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions) } s.metrics.LastUpdated = time.Now() } func (s *SLURP) markCacheMiss() { s.mu.Lock() defer s.mu.Unlock() s.metrics.CacheMisses++ if s.metrics.TotalResolutions > 0 { s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions) } s.metrics.LastUpdated = time.Now() } func (s *SLURP) markPersistenceError() { s.mu.Lock() defer s.mu.Unlock() s.metrics.PersistenceErrors++ s.metrics.LastUpdated = time.Now() } // RegisterEventHandler registers an event handler for specific event types. // // Event handlers are called asynchronously when events occur and can be // used for monitoring, logging, and reactive operations. func (s *SLURP) RegisterEventHandler(eventType EventType, handler EventHandler) { s.eventMux.Lock() defer s.eventMux.Unlock() if _, exists := s.eventHandlers[eventType]; !exists { s.eventHandlers[eventType] = make([]EventHandler, 0) } s.eventHandlers[eventType] = append(s.eventHandlers[eventType], handler) } // Close gracefully shuts down the SLURP system. // // This method stops all background tasks, flushes caches, and releases // resources. It should be called when the CHORUS node is shutting down. func (s *SLURP) Close() error { s.mu.Lock() defer s.mu.Unlock() if !s.initialized { return nil } // Cancel context to stop background tasks s.cancel() // Wait for background tasks to complete s.backgroundTasks.Wait() // TODO: Close all components in reverse dependency order // 1. Stop retrieval engine // 2. Stop intelligence system // 3. Flush and close temporal graph // 4. Flush and close context resolver // 5. Close storage layer if s.localStorage != nil { if closer, ok := s.localStorage.(interface{ Close() error }); ok { if err := closer.Close(); err != nil { return fmt.Errorf("failed to close SLURP storage: %w", err) } } } s.initialized = false return nil } // Internal methods func (s *SLURP) updateAdminStatus() { if s.election != nil { s.adminMode = s.election.IsCurrentAdmin() s.currentAdmin = s.election.GetCurrentAdmin() } } func (s *SLURP) setupElectionHandlers() error { if s.election == nil { return nil } // Set up callbacks for admin changes s.election.SetCallbacks( s.handleAdminChanged, s.handleElectionComplete, ) return nil } func (s *SLURP) handleAdminChanged(oldAdmin, newAdmin string) { s.mu.Lock() s.currentAdmin = newAdmin s.adminMode = (newAdmin == s.config.Agent.ID) s.mu.Unlock() // Emit admin change event s.emitEvent(EventAdminChanged, map[string]interface{}{ "old_admin": oldAdmin, "new_admin": newAdmin, "is_current_node": s.adminMode, }) } func (s *SLURP) handleElectionComplete(winner string) { // Election completion handling if needed } func (s *SLURP) startBackgroundTasks() { // Start metrics collection s.backgroundTasks.Add(1) go s.metricsCollectionLoop() // Start cache maintenance s.backgroundTasks.Add(1) go s.cacheMaintenance() // Start staleness detection if s.config.Slurp.TemporalAnalysis.StalenessCheckInterval > 0 { s.backgroundTasks.Add(1) go s.stalenessDetectionLoop() } } func (s *SLURP) metricsCollectionLoop() { defer s.backgroundTasks.Done() ticker := time.NewTicker(s.config.Slurp.Performance.MetricsCollectionInterval) defer ticker.Stop() for { select { case <-s.ctx.Done(): return case <-ticker.C: s.updateMetrics() } } } func (s *SLURP) cacheMaintenance() { defer s.backgroundTasks.Done() // TODO: Implement cache maintenance loop ticker := time.NewTicker(5 * time.Minute) defer ticker.Stop() for { select { case <-s.ctx.Done(): return case <-ticker.C: // Perform cache cleanup, expire old entries, etc. } } } func (s *SLURP) stalenessDetectionLoop() { defer s.backgroundTasks.Done() ticker := time.NewTicker(s.config.Slurp.TemporalAnalysis.StalenessCheckInterval) defer ticker.Stop() for { select { case <-s.ctx.Done(): return case <-ticker.C: s.detectStaleContexts() } } } func (s *SLURP) updateMetrics() { s.mu.Lock() defer s.mu.Unlock() // TODO: Collect metrics from all components s.metrics.LastUpdated = time.Now() } // getContextNode returns cached nodes (Roadmap: SEC-SLURP 1.1 persistence). func (s *SLURP) getContextNode(key string) *slurpContext.ContextNode { s.contextsMu.RLock() defer s.contextsMu.RUnlock() if node, ok := s.contextCache[key]; ok { return node } return nil } // loadContextForKey hydrates nodes from LevelDB (Roadmap: SEC-SLURP 1.1). func (s *SLURP) loadContextForKey(ctx context.Context, key string) (*slurpContext.ContextNode, error) { if s.localStorage == nil { return nil, errContextNotPersisted } runtimeCtx := s.runtimeContext(ctx) stored, err := s.localStorage.Retrieve(runtimeCtx, contextStoragePrefix+key) if err != nil { if strings.Contains(err.Error(), "not found") { return nil, errContextNotPersisted } return nil, err } node, convErr := convertStoredToContextNode(stored) if convErr != nil { return nil, convErr } return node, nil } // setupPersistentStorage configures LevelDB persistence (Roadmap: SEC-SLURP 1.1). func (s *SLURP) setupPersistentStorage() error { if s.localStorage != nil { return nil } resolvedPath := s.storagePath if resolvedPath == "" { resolvedPath = defaultStoragePath(s.config) } store, err := storage.NewLocalStorage(resolvedPath, nil) if err != nil { return err } s.localStorage = store s.storagePath = resolvedPath return nil } // initializeContextStore constructs the multi-tier context store facade. func (s *SLURP) initializeContextStore(ctx context.Context) error { if s.contextBackend != nil { return nil } if s.localStorage == nil { return fmt.Errorf("context store requires local storage") } if s.cacheManager == nil { s.cacheManager = storage.NewNoopCacheManager() } if s.indexManager == nil { s.indexManager = storage.NewNoopIndexManager() } if s.backupManager == nil { s.backupManager = storage.NewNoopBackupManager() } if s.eventNotifier == nil { s.eventNotifier = storage.NewNoopEventNotifier() } var distributed storage.DistributedStorage if s.dht != nil { if s.distributedStorage == nil { s.distributedStorage = storage.NewDistributedStorage(s.dht, s.nodeID, nil) } distributed = s.distributedStorage } options := storage.DefaultContextStoreOptions() options.CachingEnabled = false options.IndexingEnabled = false options.EncryptionEnabled = false options.AutoReplicate = distributed != nil s.contextBackend = storage.NewContextStore( s.nodeID, s.localStorage, distributed, nil, s.cacheManager, s.indexManager, s.backupManager, s.eventNotifier, options, ) s.temporalStore = s.contextBackend return nil } // initializeTemporalSystem wires the temporal graph to the DHT-backed persistence layer. func (s *SLURP) initializeTemporalSystem(ctx context.Context) error { if s.temporalGraph != nil { return nil } if s.localStorage == nil { return fmt.Errorf("temporal persistence requires local storage") } if err := s.initializeContextStore(ctx); err != nil { return err } cfg := temporal.DefaultTemporalConfig() if cfg.PersistenceConfig == nil { cfg.PersistenceConfig = &temporal.PersistenceConfig{} } cfg.PersistenceConfig.EnableWriteBuffer = false cfg.PersistenceConfig.EnableAutoSync = false cfg.PersistenceConfig.EnableAutoBackup = false cfg.PersistenceConfig.EnableLocalStorage = true cfg.PersistenceConfig.EnableDistributedStorage = s.dht != nil cfg.PersistenceConfig.EnableEncryption = false cfg.PersistenceConfig.BatchSize = 1 cfg.PersistenceConfig.FlushInterval = 30 * time.Second if len(cfg.PersistenceConfig.EncryptionRoles) == 0 { cfg.PersistenceConfig.EncryptionRoles = []string{"default"} } nodeID := s.config.Agent.ID if nodeID == "" { nodeID = fmt.Sprintf("slurp-node-%d", time.Now().UnixNano()) } system, err := temporal.NewDHTBackedTemporalGraphSystem( s.runtimeContext(ctx), s.temporalStore, s.localStorage, s.dht, nodeID, cfg, ) if err != nil { return fmt.Errorf("failed to build DHT temporal system: %w", err) } s.temporalSystem = system s.temporalGraph = system.Graph return nil } // loadPersistedContexts warms caches from disk (Roadmap: SEC-SLURP 1.1). func (s *SLURP) loadPersistedContexts(ctx context.Context) error { if s.localStorage == nil { return nil } runtimeCtx := s.runtimeContext(ctx) keys, err := s.localStorage.List(runtimeCtx, ".*") if err != nil { return err } var loaded int64 s.contextsMu.Lock() defer s.contextsMu.Unlock() for _, key := range keys { if !strings.HasPrefix(key, contextStoragePrefix) { continue } stored, retrieveErr := s.localStorage.Retrieve(runtimeCtx, key) if retrieveErr != nil { s.markPersistenceError() s.emitEvent(EventErrorOccurred, map[string]interface{}{ "action": "load_persisted_context", "key": key, "error": retrieveErr.Error(), }) continue } node, convErr := convertStoredToContextNode(stored) if convErr != nil { s.markPersistenceError() s.emitEvent(EventErrorOccurred, map[string]interface{}{ "action": "decode_persisted_context", "key": key, "error": convErr.Error(), }) continue } address := strings.TrimPrefix(key, contextStoragePrefix) nodeClone := node.Clone() s.contextCache[address] = nodeClone s.resolvedCache[address] = buildResolvedContext(nodeClone) loaded++ } s.mu.Lock() s.metrics.StoredContexts = loaded s.metrics.LastUpdated = time.Now() s.mu.Unlock() return nil } // persistContext stores contexts to LevelDB (Roadmap: SEC-SLURP 1.1). func (s *SLURP) persistContext(ctx context.Context, node *slurpContext.ContextNode) error { if s.localStorage == nil { return errContextNotPersisted } options := &storage.StoreOptions{ Compress: true, Cache: true, Metadata: map[string]interface{}{ "path": node.Path, "summary": node.Summary, "roadmap_tag": "SEC-SLURP-1.1", }, } return s.localStorage.Store(s.runtimeContext(ctx), contextStoragePrefix+node.UCXLAddress.String(), node, options) } // runtimeContext provides a safe context for persistence (Roadmap: SEC-SLURP 1.1). func (s *SLURP) runtimeContext(ctx context.Context) context.Context { if ctx != nil { return ctx } if s.ctx != nil { return s.ctx } return context.Background() } // defaultStoragePath resolves the SLURP storage directory (Roadmap: SEC-SLURP 1.1). func defaultStoragePath(cfg *config.Config) string { if cfg != nil && cfg.UCXL.Storage.Directory != "" { return filepath.Join(cfg.UCXL.Storage.Directory, "slurp") } home, err := os.UserHomeDir() if err == nil && home != "" { return filepath.Join(home, ".chorus", "slurp") } return filepath.Join(os.TempDir(), "chorus", "slurp") } // convertStoredToContextNode rehydrates persisted contexts (Roadmap: SEC-SLURP 1.1). func convertStoredToContextNode(raw interface{}) (*slurpContext.ContextNode, error) { if raw == nil { return nil, fmt.Errorf("no context data provided") } payload, err := json.Marshal(raw) if err != nil { return nil, fmt.Errorf("failed to marshal persisted context: %w", err) } var node slurpContext.ContextNode if err := json.Unmarshal(payload, &node); err != nil { return nil, fmt.Errorf("failed to decode persisted context: %w", err) } return &node, nil } func (s *SLURP) detectStaleContexts() { // TODO: Implement staleness detection // This would scan temporal nodes for contexts that haven't been // updated recently and may be outdated due to related changes } func (s *SLURP) emitEvent(eventType EventType, data map[string]interface{}) { event := &SLURPEvent{ Type: eventType, Timestamp: time.Now(), Source: "slurp_core", Data: data, } // Call event handlers asynchronously go s.handleEvent(event) } func (s *SLURP) handleEvent(event *SLURPEvent) { s.eventMux.RLock() handlers, exists := s.eventHandlers[event.Type] if !exists { s.eventMux.RUnlock() return } // Make a copy of handlers to avoid holding lock during execution handlersCopy := make([]EventHandler, len(handlers)) copy(handlersCopy, handlers) s.eventMux.RUnlock() // Execute handlers for _, handler := range handlersCopy { func(h EventHandler) { defer func() { if r := recover(); r != nil { // Log handler panic but don't crash the system } }() ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second) defer cancel() if err := h(ctx, event); err != nil { // Log handler error but continue with other handlers } }(handler) } } // validateSLURPConfig normalises runtime tunables sourced from configuration. func validateSLURPConfig(cfg *config.SlurpConfig) error { if cfg == nil { return fmt.Errorf("slurp config is nil") } if cfg.Timeout <= 0 { cfg.Timeout = 15 * time.Second } if cfg.RetryCount < 0 { cfg.RetryCount = 0 } if cfg.RetryDelay <= 0 && cfg.RetryCount > 0 { cfg.RetryDelay = 2 * time.Second } if cfg.Performance.MaxConcurrentResolutions <= 0 { cfg.Performance.MaxConcurrentResolutions = 1 } if cfg.Performance.MetricsCollectionInterval <= 0 { cfg.Performance.MetricsCollectionInterval = time.Minute } if cfg.TemporalAnalysis.MaxDecisionHops <= 0 { cfg.TemporalAnalysis.MaxDecisionHops = 1 } if cfg.TemporalAnalysis.StalenessCheckInterval <= 0 { cfg.TemporalAnalysis.StalenessCheckInterval = 5 * time.Minute } if cfg.TemporalAnalysis.StalenessThreshold < 0 || cfg.TemporalAnalysis.StalenessThreshold > 1 { cfg.TemporalAnalysis.StalenessThreshold = 0.2 } return nil } func updateAverageDuration(current time.Duration, total int64, latest time.Duration) time.Duration { if total <= 0 { return latest } if total == 1 { return latest } prevSum := int64(current) * (total - 1) return time.Duration((prevSum + int64(latest)) / total) }