Wire SLURP persistence and add restart coverage
This commit is contained in:
@@ -145,7 +145,7 @@ services:
|
||||
start_period: 10s
|
||||
|
||||
whoosh:
|
||||
image: anthonyrawlins/whoosh:scaling-v1.0.0
|
||||
image: anthonyrawlins/whoosh:latest
|
||||
ports:
|
||||
- target: 8080
|
||||
published: 8800
|
||||
@@ -200,6 +200,9 @@ services:
|
||||
WHOOSH_BACKBEAT_AGENT_ID: "whoosh"
|
||||
WHOOSH_BACKBEAT_NATS_URL: "nats://backbeat-nats:4222"
|
||||
|
||||
# Docker integration configuration (disabled for agent assignment architecture)
|
||||
WHOOSH_DOCKER_ENABLED: "false"
|
||||
|
||||
secrets:
|
||||
- whoosh_db_password
|
||||
- gitea_token
|
||||
@@ -207,8 +210,8 @@ services:
|
||||
- jwt_secret
|
||||
- service_tokens
|
||||
- redis_password
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# volumes:
|
||||
# - /var/run/docker.sock:/var/run/docker.sock # Disabled for agent assignment architecture
|
||||
deploy:
|
||||
replicas: 2
|
||||
restart_policy:
|
||||
|
||||
14
docs/progress/report-SEC-SLURP-1.1.md
Normal file
14
docs/progress/report-SEC-SLURP-1.1.md
Normal file
@@ -0,0 +1,14 @@
|
||||
# SEC-SLURP 1.1 Persistence Wiring Report
|
||||
|
||||
## Summary of Changes
|
||||
- Added LevelDB-backed persistence scaffolding in `pkg/slurp/slurp.go`, capturing the storage path, local storage handle, and the roadmap-tagged metrics helpers required for SEC-SLURP 1.1.
|
||||
- Upgraded SLURP’s lifecycle so initialization bootstraps cached context data from disk, cache misses hydrate from persistence, successful `UpsertContext` calls write back to LevelDB, and shutdown closes the store with error telemetry.
|
||||
- Introduced `pkg/slurp/slurp_persistence_test.go` to confirm contexts survive process restarts and can be resolved after clearing in-memory caches.
|
||||
- Instrumented cache/persistence metrics so hit/miss ratios and storage failures are tracked for observability.
|
||||
- Attempted `GOWORK=off go test ./pkg/slurp`; execution was blocked by legacy references to `config.Authority*` symbols in `pkg/slurp/context`, so the new test did not run.
|
||||
|
||||
## Recommended Next Steps
|
||||
- Address the `config.Authority*` symbol drift (or scope down the impacted packages) so the SLURP test suite can compile cleanly, then rerun `GOWORK=off go test ./pkg/slurp` to validate persistence changes.
|
||||
- Feed the durable store into the resolver and temporal graph implementations to finish the remaining Phase 1 SLURP roadmap items.
|
||||
- Expand Prometheus metrics and logging to track cache hit/miss ratios plus persistence errors for SEC-SLURP observability goals.
|
||||
- Review unrelated changes on `feature/phase-4-real-providers` (e.g., docker-compose edits) and either align them with this roadmap work or revert to keep the branch focused.
|
||||
@@ -8,12 +8,11 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"chorus/pkg/election"
|
||||
"chorus/pkg/dht"
|
||||
"chorus/pkg/ucxl"
|
||||
"chorus/pkg/election"
|
||||
slurpContext "chorus/pkg/slurp/context"
|
||||
"chorus/pkg/slurp/intelligence"
|
||||
"chorus/pkg/slurp/storage"
|
||||
slurpContext "chorus/pkg/slurp/context"
|
||||
)
|
||||
|
||||
// ContextManager handles leader-only context generation duties
|
||||
@@ -244,6 +243,7 @@ type LeaderContextManager struct {
|
||||
intelligence intelligence.IntelligenceEngine
|
||||
storage storage.ContextStore
|
||||
contextResolver slurpContext.ContextResolver
|
||||
contextUpserter slurp.ContextPersister
|
||||
|
||||
// Context generation state
|
||||
generationQueue chan *ContextGenerationRequest
|
||||
@@ -269,6 +269,13 @@ type LeaderContextManager struct {
|
||||
shutdownOnce sync.Once
|
||||
}
|
||||
|
||||
// SetContextPersister registers the SLURP persistence hook (Roadmap: SEC-SLURP 1.1).
|
||||
func (cm *LeaderContextManager) SetContextPersister(persister slurp.ContextPersister) {
|
||||
cm.mu.Lock()
|
||||
defer cm.mu.Unlock()
|
||||
cm.contextUpserter = persister
|
||||
}
|
||||
|
||||
// NewContextManager creates a new leader context manager
|
||||
func NewContextManager(
|
||||
election election.Election,
|
||||
@@ -454,10 +461,15 @@ func (cm *LeaderContextManager) handleGenerationRequest(req *ContextGenerationRe
|
||||
job.Result = contextNode
|
||||
cm.stats.CompletedJobs++
|
||||
|
||||
// Store generated context
|
||||
// Store generated context (SEC-SLURP 1.1 persistence bridge)
|
||||
if cm.contextUpserter != nil {
|
||||
if _, persistErr := cm.contextUpserter.UpsertContext(context.Background(), contextNode); persistErr != nil {
|
||||
// TODO(SEC-SLURP 1.1): surface persistence errors via structured logging/telemetry
|
||||
}
|
||||
} else if cm.storage != nil {
|
||||
if err := cm.storage.StoreContext(context.Background(), contextNode, []string{req.Role}); err != nil {
|
||||
// Log storage error but don't fail the job
|
||||
// TODO: Add proper logging
|
||||
// TODO: Add proper logging when falling back to legacy storage path
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,12 @@ package slurp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -35,8 +40,15 @@ import (
|
||||
"chorus/pkg/crypto"
|
||||
"chorus/pkg/dht"
|
||||
"chorus/pkg/election"
|
||||
slurpContext "chorus/pkg/slurp/context"
|
||||
"chorus/pkg/slurp/storage"
|
||||
"chorus/pkg/ucxl"
|
||||
)
|
||||
|
||||
const contextStoragePrefix = "slurp:context:"
|
||||
|
||||
var errContextNotPersisted = errors.New("slurp context not persisted")
|
||||
|
||||
// SLURP is the main coordinator for contextual intelligence operations.
|
||||
//
|
||||
// It orchestrates the interaction between context resolution, temporal analysis,
|
||||
@@ -52,6 +64,10 @@ type SLURP struct {
|
||||
crypto *crypto.AgeCrypto
|
||||
election *election.ElectionManager
|
||||
|
||||
// Roadmap: SEC-SLURP 1.1 persistent storage wiring
|
||||
storagePath string
|
||||
localStorage storage.LocalStorage
|
||||
|
||||
// Core components
|
||||
contextResolver ContextResolver
|
||||
temporalGraph TemporalGraph
|
||||
@@ -65,6 +81,11 @@ type SLURP struct {
|
||||
adminMode bool
|
||||
currentAdmin string
|
||||
|
||||
// SEC-SLURP 1.1: lightweight in-memory context persistence
|
||||
contextsMu sync.RWMutex
|
||||
contextStore map[string]*slurpContext.ContextNode
|
||||
resolvedCache map[string]*slurpContext.ResolvedContext
|
||||
|
||||
// Background processing
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
@@ -78,6 +99,11 @@ type SLURP struct {
|
||||
eventMux sync.RWMutex
|
||||
}
|
||||
|
||||
// ContextPersister exposes the persistence contract used by leader workflows (SEC-SLURP 1.1).
|
||||
type ContextPersister interface {
|
||||
UpsertContext(ctx context.Context, node *slurpContext.ContextNode) (*slurpContext.ResolvedContext, error)
|
||||
}
|
||||
|
||||
// SLURPConfig holds SLURP-specific configuration that extends the main CHORUS config
|
||||
type SLURPConfig struct {
|
||||
// Enable/disable SLURP system
|
||||
@@ -251,6 +277,9 @@ type SLURPMetrics struct {
|
||||
FailedResolutions int64 `json:"failed_resolutions"`
|
||||
AverageResolutionTime time.Duration `json:"average_resolution_time"`
|
||||
CacheHitRate float64 `json:"cache_hit_rate"`
|
||||
CacheHits int64 `json:"cache_hits"`
|
||||
CacheMisses int64 `json:"cache_misses"`
|
||||
PersistenceErrors int64 `json:"persistence_errors"`
|
||||
|
||||
// Temporal metrics
|
||||
TemporalNodes int64 `json:"temporal_nodes"`
|
||||
@@ -348,6 +377,8 @@ func NewSLURP(
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
storagePath := defaultStoragePath(config)
|
||||
|
||||
slurp := &SLURP{
|
||||
config: config,
|
||||
dht: dhtInstance,
|
||||
@@ -357,6 +388,9 @@ func NewSLURP(
|
||||
cancel: cancel,
|
||||
metrics: &SLURPMetrics{LastUpdated: time.Now()},
|
||||
eventHandlers: make(map[EventType][]EventHandler),
|
||||
contextStore: make(map[string]*slurpContext.ContextNode),
|
||||
resolvedCache: make(map[string]*slurpContext.ResolvedContext),
|
||||
storagePath: storagePath,
|
||||
}
|
||||
|
||||
return slurp, nil
|
||||
@@ -388,6 +422,40 @@ func (s *SLURP) Initialize(ctx context.Context) error {
|
||||
return fmt.Errorf("SLURP is disabled in configuration")
|
||||
}
|
||||
|
||||
// Establish runtime context for background operations
|
||||
if ctx != nil {
|
||||
if s.cancel != nil {
|
||||
s.cancel()
|
||||
}
|
||||
s.ctx, s.cancel = context.WithCancel(ctx)
|
||||
} else if s.ctx == nil {
|
||||
s.ctx, s.cancel = context.WithCancel(context.Background())
|
||||
}
|
||||
|
||||
// Ensure metrics structure is available
|
||||
if s.metrics == nil {
|
||||
s.metrics = &SLURPMetrics{}
|
||||
}
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
|
||||
// Initialize in-memory persistence (SEC-SLURP 1.1 bootstrap)
|
||||
s.contextsMu.Lock()
|
||||
if s.contextStore == nil {
|
||||
s.contextStore = make(map[string]*slurpContext.ContextNode)
|
||||
}
|
||||
if s.resolvedCache == nil {
|
||||
s.resolvedCache = make(map[string]*slurpContext.ResolvedContext)
|
||||
}
|
||||
s.contextsMu.Unlock()
|
||||
|
||||
// Roadmap: SEC-SLURP 1.1 persistent storage bootstrapping
|
||||
if err := s.setupPersistentStorage(); err != nil {
|
||||
return fmt.Errorf("failed to initialize SLURP storage: %w", err)
|
||||
}
|
||||
if err := s.loadPersistedContexts(s.ctx); err != nil {
|
||||
return fmt.Errorf("failed to load persisted contexts: %w", err)
|
||||
}
|
||||
|
||||
// TODO: Initialize components in dependency order
|
||||
// 1. Initialize storage layer first
|
||||
// 2. Initialize context resolver with storage
|
||||
@@ -425,10 +493,12 @@ func (s *SLURP) Initialize(ctx context.Context) error {
|
||||
// hierarchy traversal with caching and role-based access control.
|
||||
//
|
||||
// Parameters:
|
||||
//
|
||||
// ctx: Request context for cancellation and timeouts
|
||||
// ucxlAddress: The UCXL address to resolve context for
|
||||
//
|
||||
// Returns:
|
||||
//
|
||||
// *ResolvedContext: Complete resolved context with metadata
|
||||
// error: Any error during resolution
|
||||
//
|
||||
@@ -444,10 +514,52 @@ func (s *SLURP) Resolve(ctx context.Context, ucxlAddress string) (*ResolvedConte
|
||||
return nil, fmt.Errorf("SLURP not initialized")
|
||||
}
|
||||
|
||||
// TODO: Implement context resolution
|
||||
// This would delegate to the contextResolver component
|
||||
start := time.Now()
|
||||
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
parsed, err := ucxl.Parse(ucxlAddress)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid UCXL address: %w", err)
|
||||
}
|
||||
|
||||
key := parsed.String()
|
||||
|
||||
s.contextsMu.RLock()
|
||||
if resolved, ok := s.resolvedCache[key]; ok {
|
||||
s.contextsMu.RUnlock()
|
||||
s.markCacheHit()
|
||||
s.markResolutionSuccess(time.Since(start))
|
||||
return convertResolvedForAPI(resolved), nil
|
||||
}
|
||||
s.contextsMu.RUnlock()
|
||||
|
||||
node := s.getContextNode(key)
|
||||
if node == nil {
|
||||
// Roadmap: SEC-SLURP 1.1 - fallback to persistent storage when caches miss.
|
||||
loadedNode, loadErr := s.loadContextForKey(ctx, key)
|
||||
if loadErr != nil {
|
||||
s.markResolutionFailure()
|
||||
if !errors.Is(loadErr, errContextNotPersisted) {
|
||||
s.markPersistenceError()
|
||||
}
|
||||
if errors.Is(loadErr, errContextNotPersisted) {
|
||||
return nil, fmt.Errorf("context not found for %s", key)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to load context for %s: %w", key, loadErr)
|
||||
}
|
||||
node = loadedNode
|
||||
s.markCacheMiss()
|
||||
} else {
|
||||
s.markCacheMiss()
|
||||
}
|
||||
|
||||
built := buildResolvedContext(node)
|
||||
s.contextsMu.Lock()
|
||||
s.contextStore[key] = node
|
||||
s.resolvedCache[key] = built
|
||||
s.contextsMu.Unlock()
|
||||
|
||||
s.markResolutionSuccess(time.Since(start))
|
||||
return convertResolvedForAPI(built), nil
|
||||
}
|
||||
|
||||
// ResolveWithDepth resolves context with a specific depth limit.
|
||||
@@ -463,9 +575,14 @@ func (s *SLURP) ResolveWithDepth(ctx context.Context, ucxlAddress string, maxDep
|
||||
return nil, fmt.Errorf("maxDepth cannot be negative")
|
||||
}
|
||||
|
||||
// TODO: Implement depth-limited resolution
|
||||
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
resolved, err := s.Resolve(ctx, ucxlAddress)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolved != nil {
|
||||
resolved.BoundedDepth = maxDepth
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
// BatchResolve efficiently resolves multiple UCXL addresses in parallel.
|
||||
@@ -481,9 +598,19 @@ func (s *SLURP) BatchResolve(ctx context.Context, addresses []string) (map[strin
|
||||
return make(map[string]*ResolvedContext), nil
|
||||
}
|
||||
|
||||
// TODO: Implement batch resolution with concurrency control
|
||||
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
results := make(map[string]*ResolvedContext, len(addresses))
|
||||
var firstErr error
|
||||
for _, addr := range addresses {
|
||||
resolved, err := s.Resolve(ctx, addr)
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
results[addr] = resolved
|
||||
}
|
||||
return results, firstErr
|
||||
}
|
||||
|
||||
// GetTemporalEvolution retrieves the temporal evolution history for a context.
|
||||
@@ -495,9 +622,16 @@ func (s *SLURP) GetTemporalEvolution(ctx context.Context, ucxlAddress string) ([
|
||||
return nil, fmt.Errorf("SLURP not initialized")
|
||||
}
|
||||
|
||||
// TODO: Delegate to temporal graph component
|
||||
if s.temporalGraph == nil {
|
||||
return nil, fmt.Errorf("temporal graph not configured")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
parsed, err := ucxl.Parse(ucxlAddress)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid UCXL address: %w", err)
|
||||
}
|
||||
|
||||
return s.temporalGraph.GetEvolutionHistory(ctx, *parsed)
|
||||
}
|
||||
|
||||
// NavigateDecisionHops navigates through the decision graph by hop distance.
|
||||
@@ -510,9 +644,20 @@ func (s *SLURP) NavigateDecisionHops(ctx context.Context, ucxlAddress string, ho
|
||||
return nil, fmt.Errorf("SLURP not initialized")
|
||||
}
|
||||
|
||||
// TODO: Implement decision-hop navigation
|
||||
if s.temporalGraph == nil {
|
||||
return nil, fmt.Errorf("decision navigation not configured")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
parsed, err := ucxl.Parse(ucxlAddress)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid UCXL address: %w", err)
|
||||
}
|
||||
|
||||
if navigator, ok := s.temporalGraph.(DecisionNavigator); ok {
|
||||
return navigator.NavigateDecisionHops(ctx, *parsed, hops, direction)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("decision navigation not supported by temporal graph")
|
||||
}
|
||||
|
||||
// GenerateContext generates new context for a path (admin-only operation).
|
||||
@@ -530,9 +675,205 @@ func (s *SLURP) GenerateContext(ctx context.Context, path string, options *Gener
|
||||
return nil, fmt.Errorf("context generation requires admin privileges")
|
||||
}
|
||||
|
||||
// TODO: Delegate to intelligence component
|
||||
if s.intelligence == nil {
|
||||
return nil, fmt.Errorf("intelligence engine not configured")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
s.mu.Lock()
|
||||
s.metrics.GenerationRequests++
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
s.mu.Unlock()
|
||||
|
||||
generated, err := s.intelligence.GenerateContext(ctx, path, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
contextNode, err := convertAPIToContextNode(generated)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err := s.UpsertContext(ctx, contextNode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return generated, nil
|
||||
}
|
||||
|
||||
// UpsertContext persists a context node and exposes it for immediate resolution (SEC-SLURP 1.1).
|
||||
func (s *SLURP) UpsertContext(ctx context.Context, node *slurpContext.ContextNode) (*slurpContext.ResolvedContext, error) {
|
||||
if !s.initialized {
|
||||
return nil, fmt.Errorf("SLURP not initialized")
|
||||
}
|
||||
if node == nil {
|
||||
return nil, fmt.Errorf("context node cannot be nil")
|
||||
}
|
||||
|
||||
if err := node.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clone := node.Clone()
|
||||
resolved := buildResolvedContext(clone)
|
||||
key := clone.UCXLAddress.String()
|
||||
|
||||
s.contextsMu.Lock()
|
||||
s.contextStore[key] = clone
|
||||
s.resolvedCache[key] = resolved
|
||||
s.contextsMu.Unlock()
|
||||
|
||||
s.mu.Lock()
|
||||
s.metrics.StoredContexts++
|
||||
s.metrics.SuccessfulGenerations++
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
s.mu.Unlock()
|
||||
|
||||
if err := s.persistContext(ctx, clone); err != nil && !errors.Is(err, errContextNotPersisted) {
|
||||
s.markPersistenceError()
|
||||
s.emitEvent(EventErrorOccurred, map[string]interface{}{
|
||||
"action": "persist_context",
|
||||
"ucxl_address": key,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
s.emitEvent(EventContextGenerated, map[string]interface{}{
|
||||
"ucxl_address": key,
|
||||
"summary": clone.Summary,
|
||||
"path": clone.Path,
|
||||
})
|
||||
|
||||
return cloneResolvedInternal(resolved), nil
|
||||
}
|
||||
|
||||
func buildResolvedContext(node *slurpContext.ContextNode) *slurpContext.ResolvedContext {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &slurpContext.ResolvedContext{
|
||||
UCXLAddress: node.UCXLAddress,
|
||||
Summary: node.Summary,
|
||||
Purpose: node.Purpose,
|
||||
Technologies: cloneStringSlice(node.Technologies),
|
||||
Tags: cloneStringSlice(node.Tags),
|
||||
Insights: cloneStringSlice(node.Insights),
|
||||
ContextSourcePath: node.Path,
|
||||
InheritanceChain: []string{node.UCXLAddress.String()},
|
||||
ResolutionConfidence: node.RAGConfidence,
|
||||
BoundedDepth: 0,
|
||||
GlobalContextsApplied: false,
|
||||
ResolvedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneResolvedInternal(resolved *slurpContext.ResolvedContext) *slurpContext.ResolvedContext {
|
||||
if resolved == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
clone := *resolved
|
||||
clone.Technologies = cloneStringSlice(resolved.Technologies)
|
||||
clone.Tags = cloneStringSlice(resolved.Tags)
|
||||
clone.Insights = cloneStringSlice(resolved.Insights)
|
||||
clone.InheritanceChain = cloneStringSlice(resolved.InheritanceChain)
|
||||
return &clone
|
||||
}
|
||||
|
||||
func convertResolvedForAPI(resolved *slurpContext.ResolvedContext) *ResolvedContext {
|
||||
if resolved == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &ResolvedContext{
|
||||
UCXLAddress: resolved.UCXLAddress.String(),
|
||||
Summary: resolved.Summary,
|
||||
Purpose: resolved.Purpose,
|
||||
Technologies: cloneStringSlice(resolved.Technologies),
|
||||
Tags: cloneStringSlice(resolved.Tags),
|
||||
Insights: cloneStringSlice(resolved.Insights),
|
||||
SourcePath: resolved.ContextSourcePath,
|
||||
InheritanceChain: cloneStringSlice(resolved.InheritanceChain),
|
||||
Confidence: resolved.ResolutionConfidence,
|
||||
BoundedDepth: resolved.BoundedDepth,
|
||||
GlobalApplied: resolved.GlobalContextsApplied,
|
||||
ResolvedAt: resolved.ResolvedAt,
|
||||
Version: 1,
|
||||
LastUpdated: resolved.ResolvedAt,
|
||||
EvolutionHistory: cloneStringSlice(resolved.InheritanceChain),
|
||||
NodesTraversed: len(resolved.InheritanceChain),
|
||||
}
|
||||
}
|
||||
|
||||
func convertAPIToContextNode(node *ContextNode) (*slurpContext.ContextNode, error) {
|
||||
if node == nil {
|
||||
return nil, fmt.Errorf("context node cannot be nil")
|
||||
}
|
||||
|
||||
address, err := ucxl.Parse(node.UCXLAddress)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid UCXL address: %w", err)
|
||||
}
|
||||
|
||||
converted := &slurpContext.ContextNode{
|
||||
Path: node.Path,
|
||||
UCXLAddress: *address,
|
||||
Summary: node.Summary,
|
||||
Purpose: node.Purpose,
|
||||
Technologies: cloneStringSlice(node.Technologies),
|
||||
Tags: cloneStringSlice(node.Tags),
|
||||
Insights: cloneStringSlice(node.Insights),
|
||||
OverridesParent: node.Overrides,
|
||||
ContextSpecificity: node.Specificity,
|
||||
AppliesToChildren: node.AppliesTo == ScopeChildren,
|
||||
GeneratedAt: node.CreatedAt,
|
||||
RAGConfidence: node.Confidence,
|
||||
EncryptedFor: cloneStringSlice(node.EncryptedFor),
|
||||
AccessLevel: slurpContext.RoleAccessLevel(node.AccessLevel),
|
||||
Metadata: cloneMetadata(node.Metadata),
|
||||
}
|
||||
|
||||
converted.AppliesTo = slurpContext.ContextScope(node.AppliesTo)
|
||||
converted.CreatedBy = node.CreatedBy
|
||||
converted.UpdatedAt = node.UpdatedAt
|
||||
converted.WhoUpdated = node.UpdatedBy
|
||||
converted.Parent = node.Parent
|
||||
converted.Children = cloneStringSlice(node.Children)
|
||||
converted.FileType = node.FileType
|
||||
converted.Language = node.Language
|
||||
converted.Size = node.Size
|
||||
converted.LastModified = node.LastModified
|
||||
converted.ContentHash = node.ContentHash
|
||||
|
||||
if converted.GeneratedAt.IsZero() {
|
||||
converted.GeneratedAt = time.Now()
|
||||
}
|
||||
if converted.UpdatedAt.IsZero() {
|
||||
converted.UpdatedAt = converted.GeneratedAt
|
||||
}
|
||||
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
func cloneStringSlice(src []string) []string {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
dst := make([]string, len(src))
|
||||
copy(dst, src)
|
||||
return dst
|
||||
}
|
||||
|
||||
func cloneMetadata(src map[string]interface{}) map[string]interface{} {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
dst := make(map[string]interface{}, len(src))
|
||||
for k, v := range src {
|
||||
dst[k] = v
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// IsCurrentNodeAdmin returns true if the current node is the elected admin.
|
||||
@@ -556,6 +897,67 @@ func (s *SLURP) GetMetrics() *SLURPMetrics {
|
||||
return &metricsCopy
|
||||
}
|
||||
|
||||
// markResolutionSuccess tracks cache or storage hits (Roadmap: SEC-SLURP 1.1).
|
||||
func (s *SLURP) markResolutionSuccess(duration time.Duration) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.metrics.TotalResolutions++
|
||||
s.metrics.SuccessfulResolutions++
|
||||
s.metrics.AverageResolutionTime = updateAverageDuration(
|
||||
s.metrics.AverageResolutionTime,
|
||||
s.metrics.TotalResolutions,
|
||||
duration,
|
||||
)
|
||||
if s.metrics.TotalResolutions > 0 {
|
||||
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
|
||||
}
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
}
|
||||
|
||||
// markResolutionFailure tracks lookup failures (Roadmap: SEC-SLURP 1.1).
|
||||
func (s *SLURP) markResolutionFailure() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.metrics.TotalResolutions++
|
||||
s.metrics.FailedResolutions++
|
||||
if s.metrics.TotalResolutions > 0 {
|
||||
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
|
||||
}
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
}
|
||||
|
||||
func (s *SLURP) markCacheHit() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.metrics.CacheHits++
|
||||
if s.metrics.TotalResolutions > 0 {
|
||||
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
|
||||
}
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
}
|
||||
|
||||
func (s *SLURP) markCacheMiss() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.metrics.CacheMisses++
|
||||
if s.metrics.TotalResolutions > 0 {
|
||||
s.metrics.CacheHitRate = float64(s.metrics.CacheHits) / float64(s.metrics.TotalResolutions)
|
||||
}
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
}
|
||||
|
||||
func (s *SLURP) markPersistenceError() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.metrics.PersistenceErrors++
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
}
|
||||
|
||||
// RegisterEventHandler registers an event handler for specific event types.
|
||||
//
|
||||
// Event handlers are called asynchronously when events occur and can be
|
||||
@@ -595,6 +997,13 @@ func (s *SLURP) Close() error {
|
||||
// 3. Flush and close temporal graph
|
||||
// 4. Flush and close context resolver
|
||||
// 5. Close storage layer
|
||||
if s.localStorage != nil {
|
||||
if closer, ok := s.localStorage.(interface{ Close() error }); ok {
|
||||
if err := closer.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close SLURP storage: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s.initialized = false
|
||||
|
||||
@@ -715,6 +1124,180 @@ func (s *SLURP) updateMetrics() {
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
}
|
||||
|
||||
// getContextNode returns cached nodes (Roadmap: SEC-SLURP 1.1 persistence).
|
||||
func (s *SLURP) getContextNode(key string) *slurpContext.ContextNode {
|
||||
s.contextsMu.RLock()
|
||||
defer s.contextsMu.RUnlock()
|
||||
|
||||
if node, ok := s.contextStore[key]; ok {
|
||||
return node
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadContextForKey hydrates nodes from LevelDB (Roadmap: SEC-SLURP 1.1).
|
||||
func (s *SLURP) loadContextForKey(ctx context.Context, key string) (*slurpContext.ContextNode, error) {
|
||||
if s.localStorage == nil {
|
||||
return nil, errContextNotPersisted
|
||||
}
|
||||
|
||||
runtimeCtx := s.runtimeContext(ctx)
|
||||
stored, err := s.localStorage.Retrieve(runtimeCtx, contextStoragePrefix+key)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return nil, errContextNotPersisted
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
node, convErr := convertStoredToContextNode(stored)
|
||||
if convErr != nil {
|
||||
return nil, convErr
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// setupPersistentStorage configures LevelDB persistence (Roadmap: SEC-SLURP 1.1).
|
||||
func (s *SLURP) setupPersistentStorage() error {
|
||||
if s.localStorage != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
resolvedPath := s.storagePath
|
||||
if resolvedPath == "" {
|
||||
resolvedPath = defaultStoragePath(s.config)
|
||||
}
|
||||
|
||||
store, err := storage.NewLocalStorage(resolvedPath, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.localStorage = store
|
||||
s.storagePath = resolvedPath
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadPersistedContexts warms caches from disk (Roadmap: SEC-SLURP 1.1).
|
||||
func (s *SLURP) loadPersistedContexts(ctx context.Context) error {
|
||||
if s.localStorage == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
runtimeCtx := s.runtimeContext(ctx)
|
||||
keys, err := s.localStorage.List(runtimeCtx, ".*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var loaded int64
|
||||
s.contextsMu.Lock()
|
||||
defer s.contextsMu.Unlock()
|
||||
|
||||
for _, key := range keys {
|
||||
if !strings.HasPrefix(key, contextStoragePrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
stored, retrieveErr := s.localStorage.Retrieve(runtimeCtx, key)
|
||||
if retrieveErr != nil {
|
||||
s.markPersistenceError()
|
||||
s.emitEvent(EventErrorOccurred, map[string]interface{}{
|
||||
"action": "load_persisted_context",
|
||||
"key": key,
|
||||
"error": retrieveErr.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
node, convErr := convertStoredToContextNode(stored)
|
||||
if convErr != nil {
|
||||
s.markPersistenceError()
|
||||
s.emitEvent(EventErrorOccurred, map[string]interface{}{
|
||||
"action": "decode_persisted_context",
|
||||
"key": key,
|
||||
"error": convErr.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
address := strings.TrimPrefix(key, contextStoragePrefix)
|
||||
nodeClone := node.Clone()
|
||||
s.contextStore[address] = nodeClone
|
||||
s.resolvedCache[address] = buildResolvedContext(nodeClone)
|
||||
loaded++
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.metrics.StoredContexts = loaded
|
||||
s.metrics.LastUpdated = time.Now()
|
||||
s.mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// persistContext stores contexts to LevelDB (Roadmap: SEC-SLURP 1.1).
|
||||
func (s *SLURP) persistContext(ctx context.Context, node *slurpContext.ContextNode) error {
|
||||
if s.localStorage == nil {
|
||||
return errContextNotPersisted
|
||||
}
|
||||
|
||||
options := &storage.StoreOptions{
|
||||
Compress: true,
|
||||
Cache: true,
|
||||
Metadata: map[string]interface{}{
|
||||
"path": node.Path,
|
||||
"summary": node.Summary,
|
||||
"roadmap_tag": "SEC-SLURP-1.1",
|
||||
},
|
||||
}
|
||||
|
||||
return s.localStorage.Store(s.runtimeContext(ctx), contextStoragePrefix+node.UCXLAddress.String(), node, options)
|
||||
}
|
||||
|
||||
// runtimeContext provides a safe context for persistence (Roadmap: SEC-SLURP 1.1).
|
||||
func (s *SLURP) runtimeContext(ctx context.Context) context.Context {
|
||||
if ctx != nil {
|
||||
return ctx
|
||||
}
|
||||
if s.ctx != nil {
|
||||
return s.ctx
|
||||
}
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
// defaultStoragePath resolves the SLURP storage directory (Roadmap: SEC-SLURP 1.1).
|
||||
func defaultStoragePath(cfg *config.Config) string {
|
||||
if cfg != nil && cfg.UCXL.Storage.Directory != "" {
|
||||
return filepath.Join(cfg.UCXL.Storage.Directory, "slurp")
|
||||
}
|
||||
home, err := os.UserHomeDir()
|
||||
if err == nil && home != "" {
|
||||
return filepath.Join(home, ".chorus", "slurp")
|
||||
}
|
||||
return filepath.Join(os.TempDir(), "chorus", "slurp")
|
||||
}
|
||||
|
||||
// convertStoredToContextNode rehydrates persisted contexts (Roadmap: SEC-SLURP 1.1).
|
||||
func convertStoredToContextNode(raw interface{}) (*slurpContext.ContextNode, error) {
|
||||
if raw == nil {
|
||||
return nil, fmt.Errorf("no context data provided")
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal persisted context: %w", err)
|
||||
}
|
||||
|
||||
var node slurpContext.ContextNode
|
||||
if err := json.Unmarshal(payload, &node); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode persisted context: %w", err)
|
||||
}
|
||||
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
func (s *SLURP) detectStaleContexts() {
|
||||
// TODO: Implement staleness detection
|
||||
// This would scan temporal nodes for contexts that haven't been
|
||||
@@ -789,3 +1372,14 @@ func validateSLURPConfig(config *SLURPConfig) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateAverageDuration(current time.Duration, total int64, latest time.Duration) time.Duration {
|
||||
if total <= 0 {
|
||||
return latest
|
||||
}
|
||||
if total == 1 {
|
||||
return latest
|
||||
}
|
||||
prevSum := int64(current) * (total - 1)
|
||||
return time.Duration((prevSum + int64(latest)) / total)
|
||||
}
|
||||
|
||||
69
pkg/slurp/slurp_persistence_test.go
Normal file
69
pkg/slurp/slurp_persistence_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package slurp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"chorus/pkg/config"
|
||||
slurpContext "chorus/pkg/slurp/context"
|
||||
"chorus/pkg/ucxl"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestSLURPPersistenceLoadsContexts verifies LevelDB fallback (Roadmap: SEC-SLURP 1.1).
|
||||
func TestSLURPPersistenceLoadsContexts(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
Slurp: config.SlurpConfig{Enabled: true},
|
||||
UCXL: config.UCXLConfig{
|
||||
Storage: config.StorageConfig{Directory: configDir},
|
||||
},
|
||||
}
|
||||
|
||||
primary, err := NewSLURP(cfg, nil, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, primary.Initialize(context.Background()))
|
||||
t.Cleanup(func() {
|
||||
_ = primary.Close()
|
||||
})
|
||||
|
||||
address, err := ucxl.Parse("ucxl://agent:resolver@chorus:task/current/docs/example.go")
|
||||
require.NoError(t, err)
|
||||
|
||||
node := &slurpContext.ContextNode{
|
||||
Path: "docs/example.go",
|
||||
UCXLAddress: *address,
|
||||
Summary: "Persistent context summary",
|
||||
Purpose: "Verify persistence pipeline",
|
||||
Technologies: []string{"Go"},
|
||||
Tags: []string{"persistence", "slurp"},
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
RAGConfidence: 0.92,
|
||||
}
|
||||
|
||||
_, err = primary.UpsertContext(context.Background(), node)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, primary.Close())
|
||||
|
||||
restore, err := NewSLURP(cfg, nil, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, restore.Initialize(context.Background()))
|
||||
t.Cleanup(func() {
|
||||
_ = restore.Close()
|
||||
})
|
||||
|
||||
// Clear in-memory caches to force disk hydration path.
|
||||
restore.contextsMu.Lock()
|
||||
restore.contextStore = make(map[string]*slurpContext.ContextNode)
|
||||
restore.resolvedCache = make(map[string]*slurpContext.ResolvedContext)
|
||||
restore.contextsMu.Unlock()
|
||||
|
||||
resolved, err := restore.Resolve(context.Background(), address.String())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolved)
|
||||
assert.Equal(t, node.Summary, resolved.Summary)
|
||||
assert.Equal(t, node.Purpose, resolved.Purpose)
|
||||
assert.Contains(t, resolved.Technologies, "Go")
|
||||
}
|
||||
Reference in New Issue
Block a user