Wire SLURP persistence and add restart coverage

This commit is contained in:
anthonyrawlins
2025-09-27 15:26:25 +10:00
parent 17673c38a6
commit 0b670a535d
5 changed files with 1061 additions and 369 deletions

View File

@@ -145,7 +145,7 @@ services:
start_period: 10s start_period: 10s
whoosh: whoosh:
image: anthonyrawlins/whoosh:scaling-v1.0.0 image: anthonyrawlins/whoosh:latest
ports: ports:
- target: 8080 - target: 8080
published: 8800 published: 8800
@@ -200,6 +200,9 @@ services:
WHOOSH_BACKBEAT_AGENT_ID: "whoosh" WHOOSH_BACKBEAT_AGENT_ID: "whoosh"
WHOOSH_BACKBEAT_NATS_URL: "nats://backbeat-nats:4222" WHOOSH_BACKBEAT_NATS_URL: "nats://backbeat-nats:4222"
# Docker integration configuration (disabled for agent assignment architecture)
WHOOSH_DOCKER_ENABLED: "false"
secrets: secrets:
- whoosh_db_password - whoosh_db_password
- gitea_token - gitea_token
@@ -207,8 +210,8 @@ services:
- jwt_secret - jwt_secret
- service_tokens - service_tokens
- redis_password - redis_password
volumes: # volumes:
- /var/run/docker.sock:/var/run/docker.sock # - /var/run/docker.sock:/var/run/docker.sock # Disabled for agent assignment architecture
deploy: deploy:
replicas: 2 replicas: 2
restart_policy: restart_policy:

View File

@@ -0,0 +1,14 @@
# SEC-SLURP 1.1 Persistence Wiring Report
## Summary of Changes
- Added LevelDB-backed persistence scaffolding in `pkg/slurp/slurp.go`, capturing the storage path, local storage handle, and the roadmap-tagged metrics helpers required for SEC-SLURP1.1.
- Upgraded SLURPs lifecycle so initialization bootstraps cached context data from disk, cache misses hydrate from persistence, successful `UpsertContext` calls write back to LevelDB, and shutdown closes the store with error telemetry.
- Introduced `pkg/slurp/slurp_persistence_test.go` to confirm contexts survive process restarts and can be resolved after clearing in-memory caches.
- Instrumented cache/persistence metrics so hit/miss ratios and storage failures are tracked for observability.
- Attempted `GOWORK=off go test ./pkg/slurp`; execution was blocked by legacy references to `config.Authority*` symbols in `pkg/slurp/context`, so the new test did not run.
## Recommended Next Steps
- Address the `config.Authority*` symbol drift (or scope down the impacted packages) so the SLURP test suite can compile cleanly, then rerun `GOWORK=off go test ./pkg/slurp` to validate persistence changes.
- Feed the durable store into the resolver and temporal graph implementations to finish the remaining Phase1 SLURP roadmap items.
- Expand Prometheus metrics and logging to track cache hit/miss ratios plus persistence errors for SEC-SLURP observability goals.
- Review unrelated changes on `feature/phase-4-real-providers` (e.g., docker-compose edits) and either align them with this roadmap work or revert to keep the branch focused.

View File

@@ -8,12 +8,11 @@ import (
"sync" "sync"
"time" "time"
"chorus/pkg/election"
"chorus/pkg/dht" "chorus/pkg/dht"
"chorus/pkg/ucxl" "chorus/pkg/election"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/slurp/intelligence" "chorus/pkg/slurp/intelligence"
"chorus/pkg/slurp/storage" "chorus/pkg/slurp/storage"
slurpContext "chorus/pkg/slurp/context"
) )
// ContextManager handles leader-only context generation duties // ContextManager handles leader-only context generation duties
@@ -244,6 +243,7 @@ type LeaderContextManager struct {
intelligence intelligence.IntelligenceEngine intelligence intelligence.IntelligenceEngine
storage storage.ContextStore storage storage.ContextStore
contextResolver slurpContext.ContextResolver contextResolver slurpContext.ContextResolver
contextUpserter slurp.ContextPersister
// Context generation state // Context generation state
generationQueue chan *ContextGenerationRequest generationQueue chan *ContextGenerationRequest
@@ -269,6 +269,13 @@ type LeaderContextManager struct {
shutdownOnce sync.Once shutdownOnce sync.Once
} }
// SetContextPersister registers the SLURP persistence hook (Roadmap: SEC-SLURP 1.1).
func (cm *LeaderContextManager) SetContextPersister(persister slurp.ContextPersister) {
cm.mu.Lock()
defer cm.mu.Unlock()
cm.contextUpserter = persister
}
// NewContextManager creates a new leader context manager // NewContextManager creates a new leader context manager
func NewContextManager( func NewContextManager(
election election.Election, election election.Election,
@@ -454,10 +461,15 @@ func (cm *LeaderContextManager) handleGenerationRequest(req *ContextGenerationRe
job.Result = contextNode job.Result = contextNode
cm.stats.CompletedJobs++ cm.stats.CompletedJobs++
// Store generated context // Store generated context (SEC-SLURP 1.1 persistence bridge)
if cm.contextUpserter != nil {
if _, persistErr := cm.contextUpserter.UpsertContext(context.Background(), contextNode); persistErr != nil {
// TODO(SEC-SLURP 1.1): surface persistence errors via structured logging/telemetry
}
} else if cm.storage != nil {
if err := cm.storage.StoreContext(context.Background(), contextNode, []string{req.Role}); err != nil { if err := cm.storage.StoreContext(context.Background(), contextNode, []string{req.Role}); err != nil {
// Log storage error but don't fail the job // TODO: Add proper logging when falling back to legacy storage path
// TODO: Add proper logging }
} }
} }
} }

View File

@@ -27,7 +27,12 @@ package slurp
import ( import (
"context" "context"
"encoding/json"
"errors"
"fmt" "fmt"
"os"
"path/filepath"
"strings"
"sync" "sync"
"time" "time"
@@ -35,8 +40,15 @@ import (
"chorus/pkg/crypto" "chorus/pkg/crypto"
"chorus/pkg/dht" "chorus/pkg/dht"
"chorus/pkg/election" "chorus/pkg/election"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/slurp/storage"
"chorus/pkg/ucxl"
) )
const contextStoragePrefix = "slurp:context:"
var errContextNotPersisted = errors.New("slurp context not persisted")
// SLURP is the main coordinator for contextual intelligence operations. // SLURP is the main coordinator for contextual intelligence operations.
// //
// It orchestrates the interaction between context resolution, temporal analysis, // It orchestrates the interaction between context resolution, temporal analysis,
@@ -52,6 +64,10 @@ type SLURP struct {
crypto *crypto.AgeCrypto crypto *crypto.AgeCrypto
election *election.ElectionManager election *election.ElectionManager
// Roadmap: SEC-SLURP 1.1 persistent storage wiring
storagePath string
localStorage storage.LocalStorage
// Core components // Core components
contextResolver ContextResolver contextResolver ContextResolver
temporalGraph TemporalGraph temporalGraph TemporalGraph
@@ -65,6 +81,11 @@ type SLURP struct {
adminMode bool adminMode bool
currentAdmin string currentAdmin string
// SEC-SLURP 1.1: lightweight in-memory context persistence
contextsMu sync.RWMutex
contextStore map[string]*slurpContext.ContextNode
resolvedCache map[string]*slurpContext.ResolvedContext
// Background processing // Background processing
ctx context.Context ctx context.Context
cancel context.CancelFunc cancel context.CancelFunc
@@ -78,6 +99,11 @@ type SLURP struct {
eventMux sync.RWMutex eventMux sync.RWMutex
} }
// ContextPersister exposes the persistence contract used by leader workflows (SEC-SLURP 1.1).
type ContextPersister interface {
UpsertContext(ctx context.Context, node *slurpContext.ContextNode) (*slurpContext.ResolvedContext, error)
}
// SLURPConfig holds SLURP-specific configuration that extends the main CHORUS config // SLURPConfig holds SLURP-specific configuration that extends the main CHORUS config
type SLURPConfig struct { type SLURPConfig struct {
// Enable/disable SLURP system // Enable/disable SLURP system
@@ -251,6 +277,9 @@ type SLURPMetrics struct {
FailedResolutions int64 `json:"failed_resolutions"` FailedResolutions int64 `json:"failed_resolutions"`
AverageResolutionTime time.Duration `json:"average_resolution_time"` AverageResolutionTime time.Duration `json:"average_resolution_time"`
CacheHitRate float64 `json:"cache_hit_rate"` CacheHitRate float64 `json:"cache_hit_rate"`
CacheHits int64 `json:"cache_hits"`
CacheMisses int64 `json:"cache_misses"`
PersistenceErrors int64 `json:"persistence_errors"`
// Temporal metrics // Temporal metrics
TemporalNodes int64 `json:"temporal_nodes"` TemporalNodes int64 `json:"temporal_nodes"`
@@ -348,6 +377,8 @@ func NewSLURP(
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
storagePath := defaultStoragePath(config)
slurp := &SLURP{ slurp := &SLURP{
config: config, config: config,
dht: dhtInstance, dht: dhtInstance,
@@ -357,6 +388,9 @@ func NewSLURP(
cancel: cancel, cancel: cancel,
metrics: &SLURPMetrics{LastUpdated: time.Now()}, metrics: &SLURPMetrics{LastUpdated: time.Now()},
eventHandlers: make(map[EventType][]EventHandler), eventHandlers: make(map[EventType][]EventHandler),
contextStore: make(map[string]*slurpContext.ContextNode),
resolvedCache: make(map[string]*slurpContext.ResolvedContext),
storagePath: storagePath,
} }
return slurp, nil return slurp, nil
@@ -388,6 +422,40 @@ func (s *SLURP) Initialize(ctx context.Context) error {
return fmt.Errorf("SLURP is disabled in configuration") return fmt.Errorf("SLURP is disabled in configuration")
} }
// Establish runtime context for background operations
if ctx != nil {
if s.cancel != nil {
s.cancel()
}
s.ctx, s.cancel = context.WithCancel(ctx)
} else if s.ctx == nil {
s.ctx, s.cancel = context.WithCancel(context.Background())
}
// Ensure metrics structure is available
if s.metrics == nil {
s.metrics = &SLURPMetrics{}
}
s.metrics.LastUpdated = time.Now()
// Initialize in-memory persistence (SEC-SLURP 1.1 bootstrap)
s.contextsMu.Lock()
if s.contextStore == nil {
s.contextStore = make(map[string]*slurpContext.ContextNode)
}
if s.resolvedCache == nil {
s.resolvedCache = make(map[string]*slurpContext.ResolvedContext)
}
s.contextsMu.Unlock()
// Roadmap: SEC-SLURP 1.1 persistent storage bootstrapping
if err := s.setupPersistentStorage(); err != nil {
return fmt.Errorf("failed to initialize SLURP storage: %w", err)
}
if err := s.loadPersistedContexts(s.ctx); err != nil {
return fmt.Errorf("failed to load persisted contexts: %w", err)
}
// TODO: Initialize components in dependency order // TODO: Initialize components in dependency order
// 1. Initialize storage layer first // 1. Initialize storage layer first
// 2. Initialize context resolver with storage // 2. Initialize context resolver with storage
@@ -425,10 +493,12 @@ func (s *SLURP) Initialize(ctx context.Context) error {
// hierarchy traversal with caching and role-based access control. // hierarchy traversal with caching and role-based access control.
// //
// Parameters: // Parameters:
//
// ctx: Request context for cancellation and timeouts // ctx: Request context for cancellation and timeouts
// ucxlAddress: The UCXL address to resolve context for // ucxlAddress: The UCXL address to resolve context for
// //
// Returns: // Returns:
//
// *ResolvedContext: Complete resolved context with metadata // *ResolvedContext: Complete resolved context with metadata
// error: Any error during resolution // error: Any error during resolution
// //
@@ -444,10 +514,52 @@ func (s *SLURP) Resolve(ctx context.Context, ucxlAddress string) (*ResolvedConte
return nil, fmt.Errorf("SLURP not initialized") return nil, fmt.Errorf("SLURP not initialized")
} }
// TODO: Implement context resolution start := time.Now()
// This would delegate to the contextResolver component
return nil, fmt.Errorf("not implemented") parsed, err := ucxl.Parse(ucxlAddress)
if err != nil {
return nil, fmt.Errorf("invalid UCXL address: %w", err)
}
key := parsed.String()
s.contextsMu.RLock()
if resolved, ok := s.resolvedCache[key]; ok {
s.contextsMu.RUnlock()
s.markCacheHit()
s.markResolutionSuccess(time.Since(start))
return convertResolvedForAPI(resolved), nil
}
s.contextsMu.RUnlock()
node := s.getContextNode(key)
if node == nil {
// Roadmap: SEC-SLURP 1.1 - fallback to persistent storage when caches miss.
loadedNode, loadErr := s.loadContextForKey(ctx, key)
if loadErr != nil {
s.markResolutionFailure()
if !errors.Is(loadErr, errContextNotPersisted) {
s.markPersistenceError()
}
if errors.Is(loadErr, errContextNotPersisted) {
return nil, fmt.Errorf("context not found for %s", key)
}
return nil, fmt.Errorf("failed to load context for %s: %w", key, loadErr)
}
node = loadedNode
s.markCacheMiss()
} else {
s.markCacheMiss()
}
built := buildResolvedContext(node)
s.contextsMu.Lock()
s.contextStore[key] = node
s.resolvedCache[key] = built
s.contextsMu.Unlock()
s.markResolutionSuccess(time.Since(start))
return convertResolvedForAPI(built), nil
} }
// ResolveWithDepth resolves context with a specific depth limit. // ResolveWithDepth resolves context with a specific depth limit.
@@ -463,9 +575,14 @@ func (s *SLURP) ResolveWithDepth(ctx context.Context, ucxlAddress string, maxDep
return nil, fmt.Errorf("maxDepth cannot be negative") return nil, fmt.Errorf("maxDepth cannot be negative")
} }
// TODO: Implement depth-limited resolution resolved, err := s.Resolve(ctx, ucxlAddress)
if err != nil {
return nil, fmt.Errorf("not implemented") return nil, err
}
if resolved != nil {
resolved.BoundedDepth = maxDepth
}
return resolved, nil
} }
// BatchResolve efficiently resolves multiple UCXL addresses in parallel. // BatchResolve efficiently resolves multiple UCXL addresses in parallel.
@@ -481,9 +598,19 @@ func (s *SLURP) BatchResolve(ctx context.Context, addresses []string) (map[strin
return make(map[string]*ResolvedContext), nil return make(map[string]*ResolvedContext), nil
} }
// TODO: Implement batch resolution with concurrency control results := make(map[string]*ResolvedContext, len(addresses))
var firstErr error
return nil, fmt.Errorf("not implemented") for _, addr := range addresses {
resolved, err := s.Resolve(ctx, addr)
if err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
results[addr] = resolved
}
return results, firstErr
} }
// GetTemporalEvolution retrieves the temporal evolution history for a context. // GetTemporalEvolution retrieves the temporal evolution history for a context.
@@ -495,9 +622,16 @@ func (s *SLURP) GetTemporalEvolution(ctx context.Context, ucxlAddress string) ([
return nil, fmt.Errorf("SLURP not initialized") return nil, fmt.Errorf("SLURP not initialized")
} }
// TODO: Delegate to temporal graph component if s.temporalGraph == nil {
return nil, fmt.Errorf("temporal graph not configured")
}
return nil, fmt.Errorf("not implemented") parsed, err := ucxl.Parse(ucxlAddress)
if err != nil {
return nil, fmt.Errorf("invalid UCXL address: %w", err)
}
return s.temporalGraph.GetEvolutionHistory(ctx, *parsed)
} }
// NavigateDecisionHops navigates through the decision graph by hop distance. // NavigateDecisionHops navigates through the decision graph by hop distance.
@@ -510,9 +644,20 @@ func (s *SLURP) NavigateDecisionHops(ctx context.Context, ucxlAddress string, ho
return nil, fmt.Errorf("SLURP not initialized") return nil, fmt.Errorf("SLURP not initialized")
} }
// TODO: Implement decision-hop navigation if s.temporalGraph == nil {
return nil, fmt.Errorf("decision navigation not configured")
}
return nil, fmt.Errorf("not implemented") parsed, err := ucxl.Parse(ucxlAddress)
if err != nil {
return nil, fmt.Errorf("invalid UCXL address: %w", err)
}
if navigator, ok := s.temporalGraph.(DecisionNavigator); ok {
return navigator.NavigateDecisionHops(ctx, *parsed, hops, direction)
}
return nil, fmt.Errorf("decision navigation not supported by temporal graph")
} }
// GenerateContext generates new context for a path (admin-only operation). // GenerateContext generates new context for a path (admin-only operation).
@@ -530,9 +675,205 @@ func (s *SLURP) GenerateContext(ctx context.Context, path string, options *Gener
return nil, fmt.Errorf("context generation requires admin privileges") return nil, fmt.Errorf("context generation requires admin privileges")
} }
// TODO: Delegate to intelligence component if s.intelligence == nil {
return nil, fmt.Errorf("intelligence engine not configured")
}
return nil, fmt.Errorf("not implemented") s.mu.Lock()
s.metrics.GenerationRequests++
s.metrics.LastUpdated = time.Now()
s.mu.Unlock()
generated, err := s.intelligence.GenerateContext(ctx, path, options)
if err != nil {
return nil, err
}
contextNode, err := convertAPIToContextNode(generated)
if err != nil {
return nil, err
}
if _, err := s.UpsertContext(ctx, contextNode); err != nil {
return nil, err
}
return generated, nil
}
// UpsertContext persists a context node and exposes it for immediate resolution (SEC-SLURP 1.1).
func (s *SLURP) UpsertContext(ctx context.Context, node *slurpContext.ContextNode) (*slurpContext.ResolvedContext, error) {
if !s.initialized {
return nil, fmt.Errorf("SLURP not initialized")
}
if node == nil {
return nil, fmt.Errorf("context node cannot be nil")
}
if err := node.Validate(); err != nil {
return nil, err
}
clone := node.Clone()
resolved := buildResolvedContext(clone)
key := clone.UCXLAddress.String()
s.contextsMu.Lock()
s.contextStore[key] = clone
s.resolvedCache[key] = resolved
s.contextsMu.Unlock()
s.mu.Lock()
s.metrics.StoredContexts++
s.metrics.SuccessfulGenerations++
s.metrics.LastUpdated = time.Now()
s.mu.Unlock()
if err := s.persistContext(ctx, clone); err != nil && !errors.Is(err, errContextNotPersisted) {
s.markPersistenceError()
s.emitEvent(EventErrorOccurred, map[string]interface{}{
"action": "persist_context",
"ucxl_address": key,
"error": err.Error(),
})
}
s.emitEvent(EventContextGenerated, map[string]interface{}{
"ucxl_address": key,
"summary": clone.Summary,
"path": clone.Path,
})
return cloneResolvedInternal(resolved), nil
}
func buildResolvedContext(node *slurpContext.ContextNode) *slurpContext.ResolvedContext {
if node == nil {
return nil
}
return &slurpContext.ResolvedContext{
UCXLAddress: node.UCXLAddress,
Summary: node.Summary,
Purpose: node.Purpose,
Technologies: cloneStringSlice(node.Technologies),
Tags: cloneStringSlice(node.Tags),
Insights: cloneStringSlice(node.Insights),
ContextSourcePath: node.Path,
InheritanceChain: []string{node.UCXLAddress.String()},
ResolutionConfidence: node.RAGConfidence,
BoundedDepth: 0,
GlobalContextsApplied: false,
ResolvedAt: time.Now(),
}
}
func cloneResolvedInternal(resolved *slurpContext.ResolvedContext) *slurpContext.ResolvedContext {
if resolved == nil {
return nil
}
clone := *resolved
clone.Technologies = cloneStringSlice(resolved.Technologies)
clone.Tags = cloneStringSlice(resolved.Tags)
clone.Insights = cloneStringSlice(resolved.Insights)
clone.InheritanceChain = cloneStringSlice(resolved.InheritanceChain)
return &clone
}
func convertResolvedForAPI(resolved *slurpContext.ResolvedContext) *ResolvedContext {
if resolved == nil {
return nil
}
return &ResolvedContext{
UCXLAddress: resolved.UCXLAddress.String(),
Summary: resolved.Summary,
Purpose: resolved.Purpose,
Technologies: cloneStringSlice(resolved.Technologies),
Tags: cloneStringSlice(resolved.Tags),
Insights: cloneStringSlice(resolved.Insights),
SourcePath: resolved.ContextSourcePath,
InheritanceChain: cloneStringSlice(resolved.InheritanceChain),
Confidence: resolved.ResolutionConfidence,
BoundedDepth: resolved.BoundedDepth,
GlobalApplied: resolved.GlobalContextsApplied,
ResolvedAt: resolved.ResolvedAt,
Version: 1,
LastUpdated: resolved.ResolvedAt,
EvolutionHistory: cloneStringSlice(resolved.InheritanceChain),
NodesTraversed: len(resolved.InheritanceChain),
}
}
func convertAPIToContextNode(node *ContextNode) (*slurpContext.ContextNode, error) {
if node == nil {
return nil, fmt.Errorf("context node cannot be nil")
}
address, err := ucxl.Parse(node.UCXLAddress)
if err != nil {
return nil, fmt.Errorf("invalid UCXL address: %w", err)
}
converted := &slurpContext.ContextNode{
Path: node.Path,
UCXLAddress: *address,
Summary: node.Summary,
Purpose: node.Purpose,
Technologies: cloneStringSlice(node.Technologies),
Tags: cloneStringSlice(node.Tags),
Insights: cloneStringSlice(node.Insights),
OverridesParent: node.Overrides,
ContextSpecificity: node.Specificity,
AppliesToChildren: node.AppliesTo == ScopeChildren,
GeneratedAt: node.CreatedAt,
RAGConfidence: node.Confidence,
EncryptedFor: cloneStringSlice(node.EncryptedFor),
AccessLevel: slurpContext.RoleAccessLevel(node.AccessLevel),
Metadata: cloneMetadata(node.Metadata),
}
converted.AppliesTo = slurpContext.ContextScope(node.AppliesTo)
converted.CreatedBy = node.CreatedBy
converted.UpdatedAt = node.UpdatedAt
converted.WhoUpdated = node.UpdatedBy
converted.Parent = node.Parent
converted.Children = cloneStringSlice(node.Children)
converted.FileType = node.FileType
converted.Language = node.Language
converted.Size = node.Size
converted.LastModified = node.LastModified
converted.ContentHash = node.ContentHash
if converted.GeneratedAt.IsZero() {
converted.GeneratedAt = time.Now()
}
if converted.UpdatedAt.IsZero() {
converted.UpdatedAt = converted.GeneratedAt
}
return converted, nil
}
func cloneStringSlice(src []string) []string {
if len(src) == 0 {
return nil
}
dst := make([]string, len(src))
copy(dst, src)
return dst
}
func cloneMetadata(src map[string]interface{}) map[string]interface{} {
if len(src) == 0 {
return nil
}
dst := make(map[string]interface{}, len(src))
for k, v := range src {
dst[k] = v
}
return dst
} }
// IsCurrentNodeAdmin returns true if the current node is the elected admin. // IsCurrentNodeAdmin returns true if the current node is the elected admin.
@@ -556,6 +897,67 @@ func (s *SLURP) GetMetrics() *SLURPMetrics {
return &metricsCopy return &metricsCopy
} }
// markResolutionSuccess tracks cache or storage hits (Roadmap: SEC-SLURP 1.1).
func (s *SLURP) markResolutionSuccess(duration time.Duration) {
s.mu.Lock()
defer s.mu.Unlock()
s.metrics.TotalResolutions++
s.metrics.SuccessfulResolutions++
s.metrics.AverageResolutionTime = updateAverageDuration(
s.metrics.AverageResolutionTime,
s.metrics.TotalResolutions,
duration,
)
if s.metrics.TotalResolutions > 0 {
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
}
s.metrics.LastUpdated = time.Now()
}
// markResolutionFailure tracks lookup failures (Roadmap: SEC-SLURP 1.1).
func (s *SLURP) markResolutionFailure() {
s.mu.Lock()
defer s.mu.Unlock()
s.metrics.TotalResolutions++
s.metrics.FailedResolutions++
if s.metrics.TotalResolutions > 0 {
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
}
s.metrics.LastUpdated = time.Now()
}
func (s *SLURP) markCacheHit() {
s.mu.Lock()
defer s.mu.Unlock()
s.metrics.CacheHits++
if s.metrics.TotalResolutions > 0 {
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
}
s.metrics.LastUpdated = time.Now()
}
func (s *SLURP) markCacheMiss() {
s.mu.Lock()
defer s.mu.Unlock()
s.metrics.CacheMisses++
if s.metrics.TotalResolutions > 0 {
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
}
s.metrics.LastUpdated = time.Now()
}
func (s *SLURP) markPersistenceError() {
s.mu.Lock()
defer s.mu.Unlock()
s.metrics.PersistenceErrors++
s.metrics.LastUpdated = time.Now()
}
// RegisterEventHandler registers an event handler for specific event types. // RegisterEventHandler registers an event handler for specific event types.
// //
// Event handlers are called asynchronously when events occur and can be // Event handlers are called asynchronously when events occur and can be
@@ -595,6 +997,13 @@ func (s *SLURP) Close() error {
// 3. Flush and close temporal graph // 3. Flush and close temporal graph
// 4. Flush and close context resolver // 4. Flush and close context resolver
// 5. Close storage layer // 5. Close storage layer
if s.localStorage != nil {
if closer, ok := s.localStorage.(interface{ Close() error }); ok {
if err := closer.Close(); err != nil {
return fmt.Errorf("failed to close SLURP storage: %w", err)
}
}
}
s.initialized = false s.initialized = false
@@ -715,6 +1124,180 @@ func (s *SLURP) updateMetrics() {
s.metrics.LastUpdated = time.Now() s.metrics.LastUpdated = time.Now()
} }
// getContextNode returns cached nodes (Roadmap: SEC-SLURP 1.1 persistence).
func (s *SLURP) getContextNode(key string) *slurpContext.ContextNode {
s.contextsMu.RLock()
defer s.contextsMu.RUnlock()
if node, ok := s.contextStore[key]; ok {
return node
}
return nil
}
// loadContextForKey hydrates nodes from LevelDB (Roadmap: SEC-SLURP 1.1).
func (s *SLURP) loadContextForKey(ctx context.Context, key string) (*slurpContext.ContextNode, error) {
if s.localStorage == nil {
return nil, errContextNotPersisted
}
runtimeCtx := s.runtimeContext(ctx)
stored, err := s.localStorage.Retrieve(runtimeCtx, contextStoragePrefix+key)
if err != nil {
if strings.Contains(err.Error(), "not found") {
return nil, errContextNotPersisted
}
return nil, err
}
node, convErr := convertStoredToContextNode(stored)
if convErr != nil {
return nil, convErr
}
return node, nil
}
// setupPersistentStorage configures LevelDB persistence (Roadmap: SEC-SLURP 1.1).
func (s *SLURP) setupPersistentStorage() error {
if s.localStorage != nil {
return nil
}
resolvedPath := s.storagePath
if resolvedPath == "" {
resolvedPath = defaultStoragePath(s.config)
}
store, err := storage.NewLocalStorage(resolvedPath, nil)
if err != nil {
return err
}
s.localStorage = store
s.storagePath = resolvedPath
return nil
}
// loadPersistedContexts warms caches from disk (Roadmap: SEC-SLURP 1.1).
func (s *SLURP) loadPersistedContexts(ctx context.Context) error {
if s.localStorage == nil {
return nil
}
runtimeCtx := s.runtimeContext(ctx)
keys, err := s.localStorage.List(runtimeCtx, ".*")
if err != nil {
return err
}
var loaded int64
s.contextsMu.Lock()
defer s.contextsMu.Unlock()
for _, key := range keys {
if !strings.HasPrefix(key, contextStoragePrefix) {
continue
}
stored, retrieveErr := s.localStorage.Retrieve(runtimeCtx, key)
if retrieveErr != nil {
s.markPersistenceError()
s.emitEvent(EventErrorOccurred, map[string]interface{}{
"action": "load_persisted_context",
"key": key,
"error": retrieveErr.Error(),
})
continue
}
node, convErr := convertStoredToContextNode(stored)
if convErr != nil {
s.markPersistenceError()
s.emitEvent(EventErrorOccurred, map[string]interface{}{
"action": "decode_persisted_context",
"key": key,
"error": convErr.Error(),
})
continue
}
address := strings.TrimPrefix(key, contextStoragePrefix)
nodeClone := node.Clone()
s.contextStore[address] = nodeClone
s.resolvedCache[address] = buildResolvedContext(nodeClone)
loaded++
}
s.mu.Lock()
s.metrics.StoredContexts = loaded
s.metrics.LastUpdated = time.Now()
s.mu.Unlock()
return nil
}
// persistContext stores contexts to LevelDB (Roadmap: SEC-SLURP 1.1).
func (s *SLURP) persistContext(ctx context.Context, node *slurpContext.ContextNode) error {
if s.localStorage == nil {
return errContextNotPersisted
}
options := &storage.StoreOptions{
Compress: true,
Cache: true,
Metadata: map[string]interface{}{
"path": node.Path,
"summary": node.Summary,
"roadmap_tag": "SEC-SLURP-1.1",
},
}
return s.localStorage.Store(s.runtimeContext(ctx), contextStoragePrefix+node.UCXLAddress.String(), node, options)
}
// runtimeContext provides a safe context for persistence (Roadmap: SEC-SLURP 1.1).
func (s *SLURP) runtimeContext(ctx context.Context) context.Context {
if ctx != nil {
return ctx
}
if s.ctx != nil {
return s.ctx
}
return context.Background()
}
// defaultStoragePath resolves the SLURP storage directory (Roadmap: SEC-SLURP 1.1).
func defaultStoragePath(cfg *config.Config) string {
if cfg != nil && cfg.UCXL.Storage.Directory != "" {
return filepath.Join(cfg.UCXL.Storage.Directory, "slurp")
}
home, err := os.UserHomeDir()
if err == nil && home != "" {
return filepath.Join(home, ".chorus", "slurp")
}
return filepath.Join(os.TempDir(), "chorus", "slurp")
}
// convertStoredToContextNode rehydrates persisted contexts (Roadmap: SEC-SLURP 1.1).
func convertStoredToContextNode(raw interface{}) (*slurpContext.ContextNode, error) {
if raw == nil {
return nil, fmt.Errorf("no context data provided")
}
payload, err := json.Marshal(raw)
if err != nil {
return nil, fmt.Errorf("failed to marshal persisted context: %w", err)
}
var node slurpContext.ContextNode
if err := json.Unmarshal(payload, &node); err != nil {
return nil, fmt.Errorf("failed to decode persisted context: %w", err)
}
return &node, nil
}
func (s *SLURP) detectStaleContexts() { func (s *SLURP) detectStaleContexts() {
// TODO: Implement staleness detection // TODO: Implement staleness detection
// This would scan temporal nodes for contexts that haven't been // This would scan temporal nodes for contexts that haven't been
@@ -789,3 +1372,14 @@ func validateSLURPConfig(config *SLURPConfig) error {
return nil return nil
} }
func updateAverageDuration(current time.Duration, total int64, latest time.Duration) time.Duration {
if total <= 0 {
return latest
}
if total == 1 {
return latest
}
prevSum := int64(current) * (total - 1)
return time.Duration((prevSum + int64(latest)) / total)
}

View File

@@ -0,0 +1,69 @@
package slurp
import (
"context"
"testing"
"time"
"chorus/pkg/config"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/ucxl"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestSLURPPersistenceLoadsContexts verifies LevelDB fallback (Roadmap: SEC-SLURP 1.1).
func TestSLURPPersistenceLoadsContexts(t *testing.T) {
configDir := t.TempDir()
cfg := &config.Config{
Slurp: config.SlurpConfig{Enabled: true},
UCXL: config.UCXLConfig{
Storage: config.StorageConfig{Directory: configDir},
},
}
primary, err := NewSLURP(cfg, nil, nil, nil)
require.NoError(t, err)
require.NoError(t, primary.Initialize(context.Background()))
t.Cleanup(func() {
_ = primary.Close()
})
address, err := ucxl.Parse("ucxl://agent:resolver@chorus:task/current/docs/example.go")
require.NoError(t, err)
node := &slurpContext.ContextNode{
Path: "docs/example.go",
UCXLAddress: *address,
Summary: "Persistent context summary",
Purpose: "Verify persistence pipeline",
Technologies: []string{"Go"},
Tags: []string{"persistence", "slurp"},
GeneratedAt: time.Now().UTC(),
RAGConfidence: 0.92,
}
_, err = primary.UpsertContext(context.Background(), node)
require.NoError(t, err)
require.NoError(t, primary.Close())
restore, err := NewSLURP(cfg, nil, nil, nil)
require.NoError(t, err)
require.NoError(t, restore.Initialize(context.Background()))
t.Cleanup(func() {
_ = restore.Close()
})
// Clear in-memory caches to force disk hydration path.
restore.contextsMu.Lock()
restore.contextStore = make(map[string]*slurpContext.ContextNode)
restore.resolvedCache = make(map[string]*slurpContext.ResolvedContext)
restore.contextsMu.Unlock()
resolved, err := restore.Resolve(context.Background(), address.String())
require.NoError(t, err)
require.NotNil(t, resolved)
assert.Equal(t, node.Summary, resolved.Summary)
assert.Equal(t, node.Purpose, resolved.Purpose)
assert.Contains(t, resolved.Technologies, "Go")
}