chore: align slurp config and scaffolding

This commit is contained in:
anthonyrawlins
2025-09-27 21:03:12 +10:00
parent acc4361463
commit 4a77862289
47 changed files with 5133 additions and 4274 deletions

View File

@@ -9,12 +9,12 @@ import (
"sync"
"time"
"chorus/pkg/dht"
"chorus/pkg/crypto"
"chorus/pkg/election"
"chorus/pkg/ucxl"
"chorus/pkg/config"
"chorus/pkg/crypto"
"chorus/pkg/dht"
"chorus/pkg/election"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/ucxl"
)
// ContextDistributor handles distributed context operations via DHT
@@ -27,62 +27,68 @@ type ContextDistributor interface {
// The context is encrypted for each specified role and distributed across
// the cluster with the configured replication factor
DistributeContext(ctx context.Context, node *slurpContext.ContextNode, roles []string) error
// RetrieveContext gets context from DHT and decrypts for the requesting role
// Automatically handles role-based decryption and returns the resolved context
RetrieveContext(ctx context.Context, address ucxl.Address, role string) (*slurpContext.ResolvedContext, error)
// UpdateContext updates existing distributed context with conflict resolution
// Uses vector clocks and leader coordination for consistent updates
UpdateContext(ctx context.Context, node *slurpContext.ContextNode, roles []string) (*ConflictResolution, error)
// DeleteContext removes context from distributed storage
// Handles distributed deletion across all replicas
DeleteContext(ctx context.Context, address ucxl.Address) error
// ListDistributedContexts lists contexts available in the DHT for a role
// Provides efficient enumeration with role-based filtering
ListDistributedContexts(ctx context.Context, role string, criteria *DistributionCriteria) ([]*DistributedContextInfo, error)
// Sync synchronizes local state with distributed DHT
// Ensures eventual consistency by exchanging metadata with peers
Sync(ctx context.Context) (*SyncResult, error)
// Replicate ensures context has the desired replication factor
// Manages replica placement and health across cluster nodes
Replicate(ctx context.Context, address ucxl.Address, replicationFactor int) error
// GetReplicaHealth returns health status of context replicas
// Provides visibility into replication status and node health
GetReplicaHealth(ctx context.Context, address ucxl.Address) (*ReplicaHealth, error)
// GetDistributionStats returns distribution performance statistics
GetDistributionStats() (*DistributionStatistics, error)
// SetReplicationPolicy configures replication behavior
SetReplicationPolicy(policy *ReplicationPolicy) error
// Start initializes background distribution routines
Start(ctx context.Context) error
// Stop releases distribution resources
Stop(ctx context.Context) error
}
// DHTStorage provides direct DHT storage operations for context data
type DHTStorage interface {
// Put stores encrypted context data in the DHT
Put(ctx context.Context, key string, data []byte, options *DHTStoreOptions) error
// Get retrieves encrypted context data from the DHT
Get(ctx context.Context, key string) ([]byte, *DHTMetadata, error)
// Delete removes data from the DHT
Delete(ctx context.Context, key string) error
// Exists checks if data exists in the DHT
Exists(ctx context.Context, key string) (bool, error)
// FindProviders finds nodes that have the specified data
FindProviders(ctx context.Context, key string) ([]string, error)
// ListKeys lists all keys matching a pattern
ListKeys(ctx context.Context, pattern string) ([]string, error)
// GetStats returns DHT operation statistics
GetStats() (*DHTStatistics, error)
}
@@ -92,18 +98,18 @@ type ConflictResolver interface {
// ResolveConflict resolves conflicts between concurrent context updates
// Uses vector clocks and semantic merging rules for resolution
ResolveConflict(ctx context.Context, local *slurpContext.ContextNode, remote *slurpContext.ContextNode) (*ConflictResolution, error)
// DetectConflicts detects potential conflicts before they occur
// Provides early warning for conflicting operations
DetectConflicts(ctx context.Context, update *slurpContext.ContextNode) ([]*PotentialConflict, error)
// MergeContexts merges multiple context versions semantically
// Combines changes from different sources intelligently
MergeContexts(ctx context.Context, contexts []*slurpContext.ContextNode) (*slurpContext.ContextNode, error)
// GetConflictHistory returns history of resolved conflicts
GetConflictHistory(ctx context.Context, address ucxl.Address) ([]*ConflictResolution, error)
// SetResolutionStrategy configures conflict resolution strategy
SetResolutionStrategy(strategy *ResolutionStrategy) error
}
@@ -112,19 +118,19 @@ type ConflictResolver interface {
type ReplicationManager interface {
// EnsureReplication ensures context meets replication requirements
EnsureReplication(ctx context.Context, address ucxl.Address, factor int) error
// RepairReplicas repairs missing or corrupted replicas
RepairReplicas(ctx context.Context, address ucxl.Address) (*RepairResult, error)
// BalanceReplicas rebalances replicas across cluster nodes
BalanceReplicas(ctx context.Context) (*RebalanceResult, error)
// GetReplicationStatus returns current replication status
GetReplicationStatus(ctx context.Context, address ucxl.Address) (*ReplicationStatus, error)
// SetReplicationFactor sets the desired replication factor
SetReplicationFactor(factor int) error
// GetReplicationStats returns replication statistics
GetReplicationStats() (*ReplicationStatistics, error)
}
@@ -133,19 +139,19 @@ type ReplicationManager interface {
type GossipProtocol interface {
// StartGossip begins gossip protocol for metadata synchronization
StartGossip(ctx context.Context) error
// StopGossip stops gossip protocol
StopGossip(ctx context.Context) error
// GossipMetadata exchanges metadata with peer nodes
GossipMetadata(ctx context.Context, peer string) error
// GetGossipState returns current gossip protocol state
GetGossipState() (*GossipState, error)
// SetGossipInterval configures gossip frequency
SetGossipInterval(interval time.Duration) error
// GetGossipStats returns gossip protocol statistics
GetGossipStats() (*GossipStatistics, error)
}
@@ -154,19 +160,19 @@ type GossipProtocol interface {
type NetworkManager interface {
// DetectPartition detects network partitions in the cluster
DetectPartition(ctx context.Context) (*PartitionInfo, error)
// GetTopology returns current network topology
GetTopology(ctx context.Context) (*NetworkTopology, error)
// GetPeers returns list of available peer nodes
GetPeers(ctx context.Context) ([]*PeerInfo, error)
// CheckConnectivity checks connectivity to peer nodes
CheckConnectivity(ctx context.Context, peers []string) (*ConnectivityReport, error)
// RecoverFromPartition attempts to recover from network partition
RecoverFromPartition(ctx context.Context) (*RecoveryResult, error)
// GetNetworkStats returns network performance statistics
GetNetworkStats() (*NetworkStatistics, error)
}
@@ -175,59 +181,59 @@ type NetworkManager interface {
// DistributionCriteria represents criteria for listing distributed contexts
type DistributionCriteria struct {
Tags []string `json:"tags"` // Required tags
Technologies []string `json:"technologies"` // Required technologies
MinReplicas int `json:"min_replicas"` // Minimum replica count
MaxAge *time.Duration `json:"max_age"` // Maximum age
HealthyOnly bool `json:"healthy_only"` // Only healthy replicas
Limit int `json:"limit"` // Maximum results
Offset int `json:"offset"` // Result offset
Tags []string `json:"tags"` // Required tags
Technologies []string `json:"technologies"` // Required technologies
MinReplicas int `json:"min_replicas"` // Minimum replica count
MaxAge *time.Duration `json:"max_age"` // Maximum age
HealthyOnly bool `json:"healthy_only"` // Only healthy replicas
Limit int `json:"limit"` // Maximum results
Offset int `json:"offset"` // Result offset
}
// DistributedContextInfo represents information about distributed context
type DistributedContextInfo struct {
Address ucxl.Address `json:"address"` // Context address
Roles []string `json:"roles"` // Accessible roles
ReplicaCount int `json:"replica_count"` // Number of replicas
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
LastUpdated time.Time `json:"last_updated"` // Last update time
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
Address ucxl.Address `json:"address"` // Context address
Roles []string `json:"roles"` // Accessible roles
ReplicaCount int `json:"replica_count"` // Number of replicas
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
LastUpdated time.Time `json:"last_updated"` // Last update time
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
}
// ConflictResolution represents the result of conflict resolution
type ConflictResolution struct {
Address ucxl.Address `json:"address"` // Context address
ResolutionType ResolutionType `json:"resolution_type"` // How conflict was resolved
MergedContext *slurpContext.ContextNode `json:"merged_context"` // Resulting merged context
ConflictingSources []string `json:"conflicting_sources"` // Sources of conflict
ResolutionTime time.Duration `json:"resolution_time"` // Time taken to resolve
ResolvedAt time.Time `json:"resolved_at"` // When resolved
Confidence float64 `json:"confidence"` // Confidence in resolution
ManualReview bool `json:"manual_review"` // Whether manual review needed
Address ucxl.Address `json:"address"` // Context address
ResolutionType ResolutionType `json:"resolution_type"` // How conflict was resolved
MergedContext *slurpContext.ContextNode `json:"merged_context"` // Resulting merged context
ConflictingSources []string `json:"conflicting_sources"` // Sources of conflict
ResolutionTime time.Duration `json:"resolution_time"` // Time taken to resolve
ResolvedAt time.Time `json:"resolved_at"` // When resolved
Confidence float64 `json:"confidence"` // Confidence in resolution
ManualReview bool `json:"manual_review"` // Whether manual review needed
}
// ResolutionType represents different types of conflict resolution
type ResolutionType string
const (
ResolutionMerged ResolutionType = "merged" // Contexts were merged
ResolutionLastWriter ResolutionType = "last_writer" // Last writer wins
ResolutionMerged ResolutionType = "merged" // Contexts were merged
ResolutionLastWriter ResolutionType = "last_writer" // Last writer wins
ResolutionLeaderDecision ResolutionType = "leader_decision" // Leader made decision
ResolutionManual ResolutionType = "manual" // Manual resolution required
ResolutionFailed ResolutionType = "failed" // Resolution failed
ResolutionManual ResolutionType = "manual" // Manual resolution required
ResolutionFailed ResolutionType = "failed" // Resolution failed
)
// PotentialConflict represents a detected potential conflict
type PotentialConflict struct {
Address ucxl.Address `json:"address"` // Context address
ConflictType ConflictType `json:"conflict_type"` // Type of conflict
Description string `json:"description"` // Conflict description
Severity ConflictSeverity `json:"severity"` // Conflict severity
AffectedFields []string `json:"affected_fields"` // Fields in conflict
Suggestions []string `json:"suggestions"` // Resolution suggestions
DetectedAt time.Time `json:"detected_at"` // When detected
Address ucxl.Address `json:"address"` // Context address
ConflictType ConflictType `json:"conflict_type"` // Type of conflict
Description string `json:"description"` // Conflict description
Severity ConflictSeverity `json:"severity"` // Conflict severity
AffectedFields []string `json:"affected_fields"` // Fields in conflict
Suggestions []string `json:"suggestions"` // Resolution suggestions
DetectedAt time.Time `json:"detected_at"` // When detected
}
// ConflictType represents different types of conflicts
@@ -245,88 +251,88 @@ const (
type ConflictSeverity string
const (
SeverityLow ConflictSeverity = "low" // Low severity - auto-resolvable
SeverityMedium ConflictSeverity = "medium" // Medium severity - may need review
SeverityHigh ConflictSeverity = "high" // High severity - needs attention
SeverityCritical ConflictSeverity = "critical" // Critical - manual intervention required
ConflictSeverityLow ConflictSeverity = "low" // Low severity - auto-resolvable
ConflictSeverityMedium ConflictSeverity = "medium" // Medium severity - may need review
ConflictSeverityHigh ConflictSeverity = "high" // High severity - needs attention
ConflictSeverityCritical ConflictSeverity = "critical" // Critical - manual intervention required
)
// ResolutionStrategy represents conflict resolution strategy configuration
type ResolutionStrategy struct {
DefaultResolution ResolutionType `json:"default_resolution"` // Default resolution method
FieldPriorities map[string]int `json:"field_priorities"` // Field priority mapping
AutoMergeEnabled bool `json:"auto_merge_enabled"` // Enable automatic merging
RequireConsensus bool `json:"require_consensus"` // Require node consensus
LeaderBreaksTies bool `json:"leader_breaks_ties"` // Leader resolves ties
MaxConflictAge time.Duration `json:"max_conflict_age"` // Max age before escalation
EscalationRoles []string `json:"escalation_roles"` // Roles for manual escalation
DefaultResolution ResolutionType `json:"default_resolution"` // Default resolution method
FieldPriorities map[string]int `json:"field_priorities"` // Field priority mapping
AutoMergeEnabled bool `json:"auto_merge_enabled"` // Enable automatic merging
RequireConsensus bool `json:"require_consensus"` // Require node consensus
LeaderBreaksTies bool `json:"leader_breaks_ties"` // Leader resolves ties
MaxConflictAge time.Duration `json:"max_conflict_age"` // Max age before escalation
EscalationRoles []string `json:"escalation_roles"` // Roles for manual escalation
}
// SyncResult represents the result of synchronization operation
type SyncResult struct {
SyncedContexts int `json:"synced_contexts"` // Contexts synchronized
ConflictsResolved int `json:"conflicts_resolved"` // Conflicts resolved
Errors []string `json:"errors"` // Synchronization errors
SyncTime time.Duration `json:"sync_time"` // Total sync time
PeersContacted int `json:"peers_contacted"` // Number of peers contacted
DataTransferred int64 `json:"data_transferred"` // Bytes transferred
SyncedAt time.Time `json:"synced_at"` // When sync completed
SyncedContexts int `json:"synced_contexts"` // Contexts synchronized
ConflictsResolved int `json:"conflicts_resolved"` // Conflicts resolved
Errors []string `json:"errors"` // Synchronization errors
SyncTime time.Duration `json:"sync_time"` // Total sync time
PeersContacted int `json:"peers_contacted"` // Number of peers contacted
DataTransferred int64 `json:"data_transferred"` // Bytes transferred
SyncedAt time.Time `json:"synced_at"` // When sync completed
}
// ReplicaHealth represents health status of context replicas
type ReplicaHealth struct {
Address ucxl.Address `json:"address"` // Context address
TotalReplicas int `json:"total_replicas"` // Total replica count
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
FailedReplicas int `json:"failed_replicas"` // Failed replica count
ReplicaNodes []*ReplicaNode `json:"replica_nodes"` // Individual replica status
OverallHealth HealthStatus `json:"overall_health"` // Overall health status
LastChecked time.Time `json:"last_checked"` // When last checked
RepairNeeded bool `json:"repair_needed"` // Whether repair is needed
Address ucxl.Address `json:"address"` // Context address
TotalReplicas int `json:"total_replicas"` // Total replica count
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
FailedReplicas int `json:"failed_replicas"` // Failed replica count
ReplicaNodes []*ReplicaNode `json:"replica_nodes"` // Individual replica status
OverallHealth HealthStatus `json:"overall_health"` // Overall health status
LastChecked time.Time `json:"last_checked"` // When last checked
RepairNeeded bool `json:"repair_needed"` // Whether repair is needed
}
// ReplicaNode represents status of individual replica node
type ReplicaNode struct {
NodeID string `json:"node_id"` // Node identifier
Status ReplicaStatus `json:"status"` // Replica status
LastSeen time.Time `json:"last_seen"` // When last seen
Version int64 `json:"version"` // Context version
Checksum string `json:"checksum"` // Data checksum
Latency time.Duration `json:"latency"` // Network latency
NetworkAddress string `json:"network_address"` // Network address
NodeID string `json:"node_id"` // Node identifier
Status ReplicaStatus `json:"status"` // Replica status
LastSeen time.Time `json:"last_seen"` // When last seen
Version int64 `json:"version"` // Context version
Checksum string `json:"checksum"` // Data checksum
Latency time.Duration `json:"latency"` // Network latency
NetworkAddress string `json:"network_address"` // Network address
}
// ReplicaStatus represents status of individual replica
type ReplicaStatus string
const (
ReplicaHealthy ReplicaStatus = "healthy" // Replica is healthy
ReplicaStale ReplicaStatus = "stale" // Replica is stale
ReplicaCorrupted ReplicaStatus = "corrupted" // Replica is corrupted
ReplicaUnreachable ReplicaStatus = "unreachable" // Replica is unreachable
ReplicaSyncing ReplicaStatus = "syncing" // Replica is syncing
ReplicaHealthy ReplicaStatus = "healthy" // Replica is healthy
ReplicaStale ReplicaStatus = "stale" // Replica is stale
ReplicaCorrupted ReplicaStatus = "corrupted" // Replica is corrupted
ReplicaUnreachable ReplicaStatus = "unreachable" // Replica is unreachable
ReplicaSyncing ReplicaStatus = "syncing" // Replica is syncing
)
// HealthStatus represents overall health status
type HealthStatus string
const (
HealthHealthy HealthStatus = "healthy" // All replicas healthy
HealthDegraded HealthStatus = "degraded" // Some replicas unhealthy
HealthCritical HealthStatus = "critical" // Most replicas unhealthy
HealthFailed HealthStatus = "failed" // All replicas failed
HealthHealthy HealthStatus = "healthy" // All replicas healthy
HealthDegraded HealthStatus = "degraded" // Some replicas unhealthy
HealthCritical HealthStatus = "critical" // Most replicas unhealthy
HealthFailed HealthStatus = "failed" // All replicas failed
)
// ReplicationPolicy represents replication behavior configuration
type ReplicationPolicy struct {
DefaultFactor int `json:"default_factor"` // Default replication factor
MinFactor int `json:"min_factor"` // Minimum replication factor
MaxFactor int `json:"max_factor"` // Maximum replication factor
PreferredZones []string `json:"preferred_zones"` // Preferred availability zones
AvoidSameNode bool `json:"avoid_same_node"` // Avoid same physical node
ConsistencyLevel ConsistencyLevel `json:"consistency_level"` // Consistency requirements
RepairThreshold float64 `json:"repair_threshold"` // Health threshold for repair
RebalanceInterval time.Duration `json:"rebalance_interval"` // Rebalancing frequency
DefaultFactor int `json:"default_factor"` // Default replication factor
MinFactor int `json:"min_factor"` // Minimum replication factor
MaxFactor int `json:"max_factor"` // Maximum replication factor
PreferredZones []string `json:"preferred_zones"` // Preferred availability zones
AvoidSameNode bool `json:"avoid_same_node"` // Avoid same physical node
ConsistencyLevel ConsistencyLevel `json:"consistency_level"` // Consistency requirements
RepairThreshold float64 `json:"repair_threshold"` // Health threshold for repair
RebalanceInterval time.Duration `json:"rebalance_interval"` // Rebalancing frequency
}
// ConsistencyLevel represents consistency requirements
@@ -340,12 +346,12 @@ const (
// DHTStoreOptions represents options for DHT storage operations
type DHTStoreOptions struct {
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Duration `json:"ttl,omitempty"` // Time to live
Priority Priority `json:"priority"` // Storage priority
Compress bool `json:"compress"` // Whether to compress
Checksum bool `json:"checksum"` // Whether to checksum
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Duration `json:"ttl,omitempty"` // Time to live
Priority Priority `json:"priority"` // Storage priority
Compress bool `json:"compress"` // Whether to compress
Checksum bool `json:"checksum"` // Whether to checksum
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
}
// Priority represents storage operation priority
@@ -360,12 +366,12 @@ const (
// DHTMetadata represents metadata for DHT stored data
type DHTMetadata struct {
StoredAt time.Time `json:"stored_at"` // When stored
UpdatedAt time.Time `json:"updated_at"` // When last updated
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Time `json:"ttl,omitempty"` // Time to live
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
}
StoredAt time.Time `json:"stored_at"` // When stored
UpdatedAt time.Time `json:"updated_at"` // When last updated
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Time `json:"ttl,omitempty"` // Time to live
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
}