tony/CHORUS
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

chore: align slurp config and scaffolding

Browse Source
This commit is contained in:
anthonyrawlins
2025-09-27 21:03:12 +10:00
parent acc4361463
commit 4a77862289
47 changed files with 5133 additions and 4274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
pkg/slurp/distribution/consistent_hash.go
View File

@@ -40,7 +40,7 @@ func (ch *ConsistentHashingImpl) AddNode(nodeID string) error {
for i := 0; i < ch.virtualNodes; i++ {
virtualNodeKey := fmt.Sprintf("%s:%d", nodeID, i)
hash := ch.hashKey(virtualNodeKey)
ch.ring[hash] = nodeID
ch.sortedHashes = append(ch.sortedHashes, hash)
}
@@ -88,7 +88,7 @@ func (ch *ConsistentHashingImpl) GetNode(key string) (string, error) {
}
hash := ch.hashKey(key)
// Find the first node with hash >= key hash
idx := sort.Search(len(ch.sortedHashes), func(i int) bool {
return ch.sortedHashes[i] >= hash
@@ -175,7 +175,7 @@ func (ch *ConsistentHashingImpl) GetNodeDistribution() map[string]float64 {
// Calculate the range each node is responsible for
for i, hash := range ch.sortedHashes {
nodeID := ch.ring[hash]
var rangeSize uint64
if i == len(ch.sortedHashes)-1 {
// Last hash wraps around to first
@@ -230,7 +230,7 @@ func (ch *ConsistentHashingImpl) calculateLoadBalance() float64 {
}
avgVariance := totalVariance / float64(len(distribution))
// Convert to a balance score (higher is better, 1.0 is perfect)
// Use 1/(1+variance) to map variance to [0,1] range
return 1.0 / (1.0 + avgVariance/100.0)
@@ -261,11 +261,11 @@ func (ch *ConsistentHashingImpl) GetMetrics() *ConsistentHashMetrics {
defer ch.mu.RUnlock()
return &ConsistentHashMetrics{
TotalKeys: 0, // Would be maintained by usage tracking
NodeUtilization: ch.GetNodeDistribution(),
RebalanceEvents: 0, // Would be maintained by event tracking
AverageSeekTime: 0.1, // Placeholder - would be measured
LoadBalanceScore: ch.calculateLoadBalance(),
TotalKeys: 0, // Would be maintained by usage tracking
NodeUtilization: ch.GetNodeDistribution(),
RebalanceEvents: 0, // Would be maintained by event tracking
AverageSeekTime: 0.1, // Placeholder - would be measured
LoadBalanceScore: ch.calculateLoadBalance(),
LastRebalanceTime: 0, // Would be maintained by event tracking
}
}
@@ -306,7 +306,7 @@ func (ch *ConsistentHashingImpl) addNodeUnsafe(nodeID string) error {
for i := 0; i < ch.virtualNodes; i++ {
virtualNodeKey := fmt.Sprintf("%s:%d", nodeID, i)
hash := ch.hashKey(virtualNodeKey)
ch.ring[hash] = nodeID
ch.sortedHashes = append(ch.sortedHashes, hash)
}
@@ -333,7 +333,7 @@ func (ch *ConsistentHashingImpl) SetVirtualNodeCount(count int) error {
defer ch.mu.Unlock()
ch.virtualNodes = count
// Rehash with new virtual node count
return ch.Rehash()
}
@@ -364,8 +364,8 @@ func (ch *ConsistentHashingImpl) FindClosestNodes(key string, count int) ([]stri
if hash >= keyHash {
distance = hash - keyHash
} else {
// Wrap around distance
distance = (1<<32 - keyHash) + hash
// Wrap around distance without overflowing 32-bit space
distance = uint32((uint64(1)<<32 - uint64(keyHash)) + uint64(hash))
}
distances = append(distances, struct {
@@ -397,4 +397,4 @@ func (ch *ConsistentHashingImpl) FindClosestNodes(key string, count int) ([]stri
}
return nodes, hashes, nil
}
}

240
pkg/slurp/distribution/coordinator.go
View File

@@ -7,39 +7,39 @@ import (
"sync"
"time"
"chorus/pkg/dht"
"chorus/pkg/crypto"
"chorus/pkg/election"
"chorus/pkg/config"
"chorus/pkg/ucxl"
"chorus/pkg/crypto"
"chorus/pkg/dht"
"chorus/pkg/election"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/ucxl"
)
// DistributionCoordinator orchestrates distributed context operations across the cluster
type DistributionCoordinator struct {
mu sync.RWMutex
config *config.Config
dht *dht.DHT
roleCrypto *crypto.RoleCrypto
election election.Election
distributor ContextDistributor
replicationMgr ReplicationManager
conflictResolver ConflictResolver
gossipProtocol GossipProtocol
networkMgr NetworkManager
mu sync.RWMutex
config *config.Config
dht dht.DHT
roleCrypto *crypto.RoleCrypto
election election.Election
distributor ContextDistributor
replicationMgr ReplicationManager
conflictResolver ConflictResolver
gossipProtocol GossipProtocol
networkMgr NetworkManager
// Coordination state
isLeader bool
leaderID string
coordinationTasks chan *CoordinationTask
distributionQueue chan *DistributionRequest
roleFilters map[string]*RoleFilter
healthMonitors map[string]*HealthMonitor
isLeader bool
leaderID string
coordinationTasks chan *CoordinationTask
distributionQueue chan *DistributionRequest
roleFilters map[string]*RoleFilter
healthMonitors map[string]*HealthMonitor
// Statistics and metrics
stats *CoordinationStatistics
performanceMetrics *PerformanceMetrics
stats *CoordinationStatistics
performanceMetrics *PerformanceMetrics
// Configuration
maxConcurrentTasks int
healthCheckInterval time.Duration
@@ -49,14 +49,14 @@ type DistributionCoordinator struct {
// CoordinationTask represents a task for the coordinator
type CoordinationTask struct {
TaskID string `json:"task_id"`
TaskType CoordinationTaskType `json:"task_type"`
Priority Priority `json:"priority"`
CreatedAt time.Time `json:"created_at"`
RequestedBy string `json:"requested_by"`
Payload interface{} `json:"payload"`
Context context.Context `json:"-"`
Callback func(error) `json:"-"`
TaskID string `json:"task_id"`
TaskType CoordinationTaskType `json:"task_type"`
Priority Priority `json:"priority"`
CreatedAt time.Time `json:"created_at"`
RequestedBy string `json:"requested_by"`
Payload interface{} `json:"payload"`
Context context.Context `json:"-"`
Callback func(error) `json:"-"`
}
// CoordinationTaskType represents different types of coordination tasks
@@ -74,55 +74,55 @@ const (
// DistributionRequest represents a request for context distribution
type DistributionRequest struct {
RequestID string `json:"request_id"`
ContextNode *slurpContext.ContextNode `json:"context_node"`
TargetRoles []string `json:"target_roles"`
Priority Priority `json:"priority"`
RequesterID string `json:"requester_id"`
CreatedAt time.Time `json:"created_at"`
Options *DistributionOptions `json:"options"`
Callback func(*DistributionResult, error) `json:"-"`
RequestID string `json:"request_id"`
ContextNode *slurpContext.ContextNode `json:"context_node"`
TargetRoles []string `json:"target_roles"`
Priority Priority `json:"priority"`
RequesterID string `json:"requester_id"`
CreatedAt time.Time `json:"created_at"`
Options *DistributionOptions `json:"options"`
Callback func(*DistributionResult, error) `json:"-"`
}
// DistributionOptions contains options for context distribution
type DistributionOptions struct {
ReplicationFactor int `json:"replication_factor"`
ConsistencyLevel ConsistencyLevel `json:"consistency_level"`
EncryptionLevel crypto.AccessLevel `json:"encryption_level"`
TTL *time.Duration `json:"ttl,omitempty"`
PreferredZones []string `json:"preferred_zones"`
ExcludedNodes []string `json:"excluded_nodes"`
ConflictResolution ResolutionType `json:"conflict_resolution"`
ReplicationFactor int `json:"replication_factor"`
ConsistencyLevel ConsistencyLevel `json:"consistency_level"`
EncryptionLevel crypto.AccessLevel `json:"encryption_level"`
TTL *time.Duration `json:"ttl,omitempty"`
PreferredZones []string `json:"preferred_zones"`
ExcludedNodes []string `json:"excluded_nodes"`
ConflictResolution ResolutionType `json:"conflict_resolution"`
}
// DistributionResult represents the result of a distribution operation
type DistributionResult struct {
RequestID string `json:"request_id"`
Success bool `json:"success"`
DistributedNodes []string `json:"distributed_nodes"`
ReplicationFactor int `json:"replication_factor"`
ProcessingTime time.Duration `json:"processing_time"`
Errors []string `json:"errors"`
ConflictResolved *ConflictResolution `json:"conflict_resolved,omitempty"`
CompletedAt time.Time `json:"completed_at"`
RequestID string `json:"request_id"`
Success bool `json:"success"`
DistributedNodes []string `json:"distributed_nodes"`
ReplicationFactor int `json:"replication_factor"`
ProcessingTime time.Duration `json:"processing_time"`
Errors []string `json:"errors"`
ConflictResolved *ConflictResolution `json:"conflict_resolved,omitempty"`
CompletedAt time.Time `json:"completed_at"`
}
// RoleFilter manages role-based filtering for context access
type RoleFilter struct {
RoleID string `json:"role_id"`
AccessLevel crypto.AccessLevel `json:"access_level"`
AllowedCompartments []string `json:"allowed_compartments"`
FilterRules []*FilterRule `json:"filter_rules"`
LastUpdated time.Time `json:"last_updated"`
RoleID string `json:"role_id"`
AccessLevel crypto.AccessLevel `json:"access_level"`
AllowedCompartments []string `json:"allowed_compartments"`
FilterRules []*FilterRule `json:"filter_rules"`
LastUpdated time.Time `json:"last_updated"`
}
// FilterRule represents a single filtering rule
type FilterRule struct {
RuleID string `json:"rule_id"`
RuleType FilterRuleType `json:"rule_type"`
Pattern string `json:"pattern"`
Action FilterAction `json:"action"`
Metadata map[string]interface{} `json:"metadata"`
RuleID string `json:"rule_id"`
RuleType FilterRuleType `json:"rule_type"`
Pattern string `json:"pattern"`
Action FilterAction `json:"action"`
Metadata map[string]interface{} `json:"metadata"`
}
// FilterRuleType represents different types of filter rules
@@ -139,10 +139,10 @@ const (
type FilterAction string
const (
FilterActionAllow FilterAction = "allow"
FilterActionDeny FilterAction = "deny"
FilterActionModify FilterAction = "modify"
FilterActionAudit FilterAction = "audit"
FilterActionAllow FilterAction = "allow"
FilterActionDeny FilterAction = "deny"
FilterActionModify FilterAction = "modify"
FilterActionAudit FilterAction = "audit"
)
// HealthMonitor monitors the health of a specific component
@@ -160,10 +160,10 @@ type HealthMonitor struct {
type ComponentType string
const (
ComponentTypeDHT ComponentType = "dht"
ComponentTypeReplication ComponentType = "replication"
ComponentTypeGossip ComponentType = "gossip"
ComponentTypeNetwork ComponentType = "network"
ComponentTypeDHT ComponentType = "dht"
ComponentTypeReplication ComponentType = "replication"
ComponentTypeGossip ComponentType = "gossip"
ComponentTypeNetwork ComponentType = "network"
ComponentTypeConflictResolver ComponentType = "conflict_resolver"
)
@@ -190,13 +190,13 @@ type CoordinationStatistics struct {
// PerformanceMetrics tracks detailed performance metrics
type PerformanceMetrics struct {
ThroughputPerSecond float64 `json:"throughput_per_second"`
LatencyPercentiles map[string]float64 `json:"latency_percentiles"`
ErrorRateByType map[string]float64 `json:"error_rate_by_type"`
ResourceUtilization map[string]float64 `json:"resource_utilization"`
NetworkMetrics *NetworkMetrics `json:"network_metrics"`
StorageMetrics *StorageMetrics `json:"storage_metrics"`
LastCalculated time.Time `json:"last_calculated"`
ThroughputPerSecond float64 `json:"throughput_per_second"`
LatencyPercentiles map[string]float64 `json:"latency_percentiles"`
ErrorRateByType map[string]float64 `json:"error_rate_by_type"`
ResourceUtilization map[string]float64 `json:"resource_utilization"`
NetworkMetrics *NetworkMetrics `json:"network_metrics"`
StorageMetrics *StorageMetrics `json:"storage_metrics"`
LastCalculated time.Time `json:"last_calculated"`
}
// NetworkMetrics tracks network-related performance
@@ -210,24 +210,24 @@ type NetworkMetrics struct {
// StorageMetrics tracks storage-related performance
type StorageMetrics struct {
TotalContexts int64 `json:"total_contexts"`
StorageUtilization float64 `json:"storage_utilization"`
CompressionRatio float64 `json:"compression_ratio"`
TotalContexts int64 `json:"total_contexts"`
StorageUtilization float64 `json:"storage_utilization"`
CompressionRatio float64 `json:"compression_ratio"`
ReplicationEfficiency float64 `json:"replication_efficiency"`
CacheHitRate float64 `json:"cache_hit_rate"`
CacheHitRate float64 `json:"cache_hit_rate"`
}
// NewDistributionCoordinator creates a new distribution coordinator
func NewDistributionCoordinator(
config *config.Config,
dht *dht.DHT,
dhtInstance dht.DHT,
roleCrypto *crypto.RoleCrypto,
election election.Election,
) (*DistributionCoordinator, error) {
if config == nil {
return nil, fmt.Errorf("config is required")
}
if dht == nil {
if dhtInstance == nil {
return nil, fmt.Errorf("DHT instance is required")
}
if roleCrypto == nil {
@@ -238,14 +238,14 @@ func NewDistributionCoordinator(
}
// Create distributor
distributor, err := NewDHTContextDistributor(dht, roleCrypto, election, config)
distributor, err := NewDHTContextDistributor(dhtInstance, roleCrypto, election, config)
if err != nil {
return nil, fmt.Errorf("failed to create context distributor: %w", err)
}
coord := &DistributionCoordinator{
config: config,
dht: dht,
dht: dhtInstance,
roleCrypto: roleCrypto,
election: election,
distributor: distributor,
@@ -264,9 +264,9 @@ func NewDistributionCoordinator(
LatencyPercentiles: make(map[string]float64),
ErrorRateByType: make(map[string]float64),
ResourceUtilization: make(map[string]float64),
NetworkMetrics: &NetworkMetrics{},
StorageMetrics: &StorageMetrics{},
LastCalculated: time.Now(),
NetworkMetrics: &NetworkMetrics{},
StorageMetrics: &StorageMetrics{},
LastCalculated: time.Now(),
},
}
@@ -356,7 +356,7 @@ func (dc *DistributionCoordinator) CoordinateReplication(
CreatedAt: time.Now(),
RequestedBy: dc.config.Agent.ID,
Payload: map[string]interface{}{
"address": address,
"address": address,
"target_factor": targetFactor,
},
Context: ctx,
@@ -398,14 +398,14 @@ func (dc *DistributionCoordinator) GetClusterHealth() (*ClusterHealth, error) {
defer dc.mu.RUnlock()
health := &ClusterHealth{
OverallStatus: dc.calculateOverallHealth(),
NodeCount: len(dc.dht.GetConnectedPeers()) + 1, // +1 for current node
HealthyNodes: 0,
UnhealthyNodes: 0,
ComponentHealth: make(map[string]*ComponentHealth),
LastUpdated: time.Now(),
Alerts: []string{},
Recommendations: []string{},
OverallStatus: dc.calculateOverallHealth(),
NodeCount: len(dc.healthMonitors) + 1, // Placeholder count including current node
HealthyNodes: 0,
UnhealthyNodes: 0,
ComponentHealth: make(map[string]*ComponentHealth),
LastUpdated: time.Now(),
Alerts: []string{},
Recommendations: []string{},
}
// Calculate component health
@@ -582,7 +582,7 @@ func (dc *DistributionCoordinator) initializeComponents() error {
func (dc *DistributionCoordinator) initializeRoleFilters() {
// Initialize role filters based on configuration
roles := []string{"senior_architect", "project_manager", "devops_engineer", "backend_developer", "frontend_developer"}
for _, role := range roles {
dc.roleFilters[role] = &RoleFilter{
RoleID: role,
@@ -598,8 +598,8 @@ func (dc *DistributionCoordinator) initializeHealthMonitors() {
components := map[string]ComponentType{
"dht": ComponentTypeDHT,
"replication": ComponentTypeReplication,
"gossip": ComponentTypeGossip,
"network": ComponentTypeNetwork,
"gossip": ComponentTypeGossip,
"network": ComponentTypeNetwork,
"conflict_resolver": ComponentTypeConflictResolver,
}
@@ -682,8 +682,8 @@ func (dc *DistributionCoordinator) executeDistribution(ctx context.Context, requ
Success: false,
DistributedNodes: []string{},
ProcessingTime: 0,
Errors: []string{},
CompletedAt: time.Now(),
Errors: []string{},
CompletedAt: time.Now(),
}
// Execute distribution via distributor
@@ -703,14 +703,14 @@ func (dc *DistributionCoordinator) executeDistribution(ctx context.Context, requ
// ClusterHealth represents overall cluster health
type ClusterHealth struct {
OverallStatus HealthStatus `json:"overall_status"`
NodeCount int `json:"node_count"`
HealthyNodes int `json:"healthy_nodes"`
UnhealthyNodes int `json:"unhealthy_nodes"`
ComponentHealth map[string]*ComponentHealth `json:"component_health"`
LastUpdated time.Time `json:"last_updated"`
Alerts []string `json:"alerts"`
Recommendations []string `json:"recommendations"`
OverallStatus HealthStatus `json:"overall_status"`
NodeCount int `json:"node_count"`
HealthyNodes int `json:"healthy_nodes"`
UnhealthyNodes int `json:"unhealthy_nodes"`
ComponentHealth map[string]*ComponentHealth `json:"component_health"`
LastUpdated time.Time `json:"last_updated"`
Alerts []string `json:"alerts"`
Recommendations []string `json:"recommendations"`
}
// ComponentHealth represents individual component health
@@ -736,14 +736,14 @@ func (dc *DistributionCoordinator) getDefaultDistributionOptions() *Distribution
return &DistributionOptions{
ReplicationFactor: 3,
ConsistencyLevel: ConsistencyEventual,
EncryptionLevel: crypto.AccessMedium,
EncryptionLevel: crypto.AccessLevel(slurpContext.AccessMedium),
ConflictResolution: ResolutionMerged,
}
}
func (dc *DistributionCoordinator) getAccessLevelForRole(role string) crypto.AccessLevel {
// Placeholder implementation
return crypto.AccessMedium
return crypto.AccessLevel(slurpContext.AccessMedium)
}
func (dc *DistributionCoordinator) getAllowedCompartments(role string) []string {
@@ -796,13 +796,13 @@ func (dc *DistributionCoordinator) updatePerformanceMetrics() {
func (dc *DistributionCoordinator) priorityFromSeverity(severity ConflictSeverity) Priority {
switch severity {
case SeverityCritical:
case ConflictSeverityCritical:
return PriorityCritical
case SeverityHigh:
case ConflictSeverityHigh:
return PriorityHigh
case SeverityMedium:
case ConflictSeverityMedium:
return PriorityNormal
default:
return PriorityLow
}
}
}

280
pkg/slurp/distribution/dht.go
View File

@@ -9,12 +9,12 @@ import (
"sync"
"time"
"chorus/pkg/dht"
"chorus/pkg/crypto"
"chorus/pkg/election"
"chorus/pkg/ucxl"
"chorus/pkg/config"
"chorus/pkg/crypto"
"chorus/pkg/dht"
"chorus/pkg/election"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/ucxl"
)
// ContextDistributor handles distributed context operations via DHT
@@ -27,62 +27,68 @@ type ContextDistributor interface {
// The context is encrypted for each specified role and distributed across
// the cluster with the configured replication factor
DistributeContext(ctx context.Context, node *slurpContext.ContextNode, roles []string) error
// RetrieveContext gets context from DHT and decrypts for the requesting role
// Automatically handles role-based decryption and returns the resolved context
RetrieveContext(ctx context.Context, address ucxl.Address, role string) (*slurpContext.ResolvedContext, error)
// UpdateContext updates existing distributed context with conflict resolution
// Uses vector clocks and leader coordination for consistent updates
UpdateContext(ctx context.Context, node *slurpContext.ContextNode, roles []string) (*ConflictResolution, error)
// DeleteContext removes context from distributed storage
// Handles distributed deletion across all replicas
DeleteContext(ctx context.Context, address ucxl.Address) error
// ListDistributedContexts lists contexts available in the DHT for a role
// Provides efficient enumeration with role-based filtering
ListDistributedContexts(ctx context.Context, role string, criteria *DistributionCriteria) ([]*DistributedContextInfo, error)
// Sync synchronizes local state with distributed DHT
// Ensures eventual consistency by exchanging metadata with peers
Sync(ctx context.Context) (*SyncResult, error)
// Replicate ensures context has the desired replication factor
// Manages replica placement and health across cluster nodes
Replicate(ctx context.Context, address ucxl.Address, replicationFactor int) error
// GetReplicaHealth returns health status of context replicas
// Provides visibility into replication status and node health
GetReplicaHealth(ctx context.Context, address ucxl.Address) (*ReplicaHealth, error)
// GetDistributionStats returns distribution performance statistics
GetDistributionStats() (*DistributionStatistics, error)
// SetReplicationPolicy configures replication behavior
SetReplicationPolicy(policy *ReplicationPolicy) error
// Start initializes background distribution routines
Start(ctx context.Context) error
// Stop releases distribution resources
Stop(ctx context.Context) error
}
// DHTStorage provides direct DHT storage operations for context data
type DHTStorage interface {
// Put stores encrypted context data in the DHT
Put(ctx context.Context, key string, data []byte, options *DHTStoreOptions) error
// Get retrieves encrypted context data from the DHT
Get(ctx context.Context, key string) ([]byte, *DHTMetadata, error)
// Delete removes data from the DHT
Delete(ctx context.Context, key string) error
// Exists checks if data exists in the DHT
Exists(ctx context.Context, key string) (bool, error)
// FindProviders finds nodes that have the specified data
FindProviders(ctx context.Context, key string) ([]string, error)
// ListKeys lists all keys matching a pattern
ListKeys(ctx context.Context, pattern string) ([]string, error)
// GetStats returns DHT operation statistics
GetStats() (*DHTStatistics, error)
}
@@ -92,18 +98,18 @@ type ConflictResolver interface {
// ResolveConflict resolves conflicts between concurrent context updates
// Uses vector clocks and semantic merging rules for resolution
ResolveConflict(ctx context.Context, local *slurpContext.ContextNode, remote *slurpContext.ContextNode) (*ConflictResolution, error)
// DetectConflicts detects potential conflicts before they occur
// Provides early warning for conflicting operations
DetectConflicts(ctx context.Context, update *slurpContext.ContextNode) ([]*PotentialConflict, error)
// MergeContexts merges multiple context versions semantically
// Combines changes from different sources intelligently
MergeContexts(ctx context.Context, contexts []*slurpContext.ContextNode) (*slurpContext.ContextNode, error)
// GetConflictHistory returns history of resolved conflicts
GetConflictHistory(ctx context.Context, address ucxl.Address) ([]*ConflictResolution, error)
// SetResolutionStrategy configures conflict resolution strategy
SetResolutionStrategy(strategy *ResolutionStrategy) error
}
@@ -112,19 +118,19 @@ type ConflictResolver interface {
type ReplicationManager interface {
// EnsureReplication ensures context meets replication requirements
EnsureReplication(ctx context.Context, address ucxl.Address, factor int) error
// RepairReplicas repairs missing or corrupted replicas
RepairReplicas(ctx context.Context, address ucxl.Address) (*RepairResult, error)
// BalanceReplicas rebalances replicas across cluster nodes
BalanceReplicas(ctx context.Context) (*RebalanceResult, error)
// GetReplicationStatus returns current replication status
GetReplicationStatus(ctx context.Context, address ucxl.Address) (*ReplicationStatus, error)
// SetReplicationFactor sets the desired replication factor
SetReplicationFactor(factor int) error
// GetReplicationStats returns replication statistics
GetReplicationStats() (*ReplicationStatistics, error)
}
@@ -133,19 +139,19 @@ type ReplicationManager interface {
type GossipProtocol interface {
// StartGossip begins gossip protocol for metadata synchronization
StartGossip(ctx context.Context) error
// StopGossip stops gossip protocol
StopGossip(ctx context.Context) error
// GossipMetadata exchanges metadata with peer nodes
GossipMetadata(ctx context.Context, peer string) error
// GetGossipState returns current gossip protocol state
GetGossipState() (*GossipState, error)
// SetGossipInterval configures gossip frequency
SetGossipInterval(interval time.Duration) error
// GetGossipStats returns gossip protocol statistics
GetGossipStats() (*GossipStatistics, error)
}
@@ -154,19 +160,19 @@ type GossipProtocol interface {
type NetworkManager interface {
// DetectPartition detects network partitions in the cluster
DetectPartition(ctx context.Context) (*PartitionInfo, error)
// GetTopology returns current network topology
GetTopology(ctx context.Context) (*NetworkTopology, error)
// GetPeers returns list of available peer nodes
GetPeers(ctx context.Context) ([]*PeerInfo, error)
// CheckConnectivity checks connectivity to peer nodes
CheckConnectivity(ctx context.Context, peers []string) (*ConnectivityReport, error)
// RecoverFromPartition attempts to recover from network partition
RecoverFromPartition(ctx context.Context) (*RecoveryResult, error)
// GetNetworkStats returns network performance statistics
GetNetworkStats() (*NetworkStatistics, error)
}
@@ -175,59 +181,59 @@ type NetworkManager interface {
// DistributionCriteria represents criteria for listing distributed contexts
type DistributionCriteria struct {
Tags []string `json:"tags"` // Required tags
Technologies []string `json:"technologies"` // Required technologies
MinReplicas int `json:"min_replicas"` // Minimum replica count
MaxAge *time.Duration `json:"max_age"` // Maximum age
HealthyOnly bool `json:"healthy_only"` // Only healthy replicas
Limit int `json:"limit"` // Maximum results
Offset int `json:"offset"` // Result offset
Tags []string `json:"tags"` // Required tags
Technologies []string `json:"technologies"` // Required technologies
MinReplicas int `json:"min_replicas"` // Minimum replica count
MaxAge *time.Duration `json:"max_age"` // Maximum age
HealthyOnly bool `json:"healthy_only"` // Only healthy replicas
Limit int `json:"limit"` // Maximum results
Offset int `json:"offset"` // Result offset
}
// DistributedContextInfo represents information about distributed context
type DistributedContextInfo struct {
Address ucxl.Address `json:"address"` // Context address
Roles []string `json:"roles"` // Accessible roles
ReplicaCount int `json:"replica_count"` // Number of replicas
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
LastUpdated time.Time `json:"last_updated"` // Last update time
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
Address ucxl.Address `json:"address"` // Context address
Roles []string `json:"roles"` // Accessible roles
ReplicaCount int `json:"replica_count"` // Number of replicas
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
LastUpdated time.Time `json:"last_updated"` // Last update time
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
}
// ConflictResolution represents the result of conflict resolution
type ConflictResolution struct {
Address ucxl.Address `json:"address"` // Context address
ResolutionType ResolutionType `json:"resolution_type"` // How conflict was resolved
MergedContext *slurpContext.ContextNode `json:"merged_context"` // Resulting merged context
ConflictingSources []string `json:"conflicting_sources"` // Sources of conflict
ResolutionTime time.Duration `json:"resolution_time"` // Time taken to resolve
ResolvedAt time.Time `json:"resolved_at"` // When resolved
Confidence float64 `json:"confidence"` // Confidence in resolution
ManualReview bool `json:"manual_review"` // Whether manual review needed
Address ucxl.Address `json:"address"` // Context address
ResolutionType ResolutionType `json:"resolution_type"` // How conflict was resolved
MergedContext *slurpContext.ContextNode `json:"merged_context"` // Resulting merged context
ConflictingSources []string `json:"conflicting_sources"` // Sources of conflict
ResolutionTime time.Duration `json:"resolution_time"` // Time taken to resolve
ResolvedAt time.Time `json:"resolved_at"` // When resolved
Confidence float64 `json:"confidence"` // Confidence in resolution
ManualReview bool `json:"manual_review"` // Whether manual review needed
}
// ResolutionType represents different types of conflict resolution
type ResolutionType string
const (
ResolutionMerged ResolutionType = "merged" // Contexts were merged
ResolutionLastWriter ResolutionType = "last_writer" // Last writer wins
ResolutionMerged ResolutionType = "merged" // Contexts were merged
ResolutionLastWriter ResolutionType = "last_writer" // Last writer wins
ResolutionLeaderDecision ResolutionType = "leader_decision" // Leader made decision
ResolutionManual ResolutionType = "manual" // Manual resolution required
ResolutionFailed ResolutionType = "failed" // Resolution failed
ResolutionManual ResolutionType = "manual" // Manual resolution required
ResolutionFailed ResolutionType = "failed" // Resolution failed
)
// PotentialConflict represents a detected potential conflict
type PotentialConflict struct {
Address ucxl.Address `json:"address"` // Context address
ConflictType ConflictType `json:"conflict_type"` // Type of conflict
Description string `json:"description"` // Conflict description
Severity ConflictSeverity `json:"severity"` // Conflict severity
AffectedFields []string `json:"affected_fields"` // Fields in conflict
Suggestions []string `json:"suggestions"` // Resolution suggestions
DetectedAt time.Time `json:"detected_at"` // When detected
Address ucxl.Address `json:"address"` // Context address
ConflictType ConflictType `json:"conflict_type"` // Type of conflict
Description string `json:"description"` // Conflict description
Severity ConflictSeverity `json:"severity"` // Conflict severity
AffectedFields []string `json:"affected_fields"` // Fields in conflict
Suggestions []string `json:"suggestions"` // Resolution suggestions
DetectedAt time.Time `json:"detected_at"` // When detected
}
// ConflictType represents different types of conflicts
@@ -245,88 +251,88 @@ const (
type ConflictSeverity string
const (
SeverityLow ConflictSeverity = "low" // Low severity - auto-resolvable
SeverityMedium ConflictSeverity = "medium" // Medium severity - may need review
SeverityHigh ConflictSeverity = "high" // High severity - needs attention
SeverityCritical ConflictSeverity = "critical" // Critical - manual intervention required
ConflictSeverityLow ConflictSeverity = "low" // Low severity - auto-resolvable
ConflictSeverityMedium ConflictSeverity = "medium" // Medium severity - may need review
ConflictSeverityHigh ConflictSeverity = "high" // High severity - needs attention
ConflictSeverityCritical ConflictSeverity = "critical" // Critical - manual intervention required
)
// ResolutionStrategy represents conflict resolution strategy configuration
type ResolutionStrategy struct {
DefaultResolution ResolutionType `json:"default_resolution"` // Default resolution method
FieldPriorities map[string]int `json:"field_priorities"` // Field priority mapping
AutoMergeEnabled bool `json:"auto_merge_enabled"` // Enable automatic merging
RequireConsensus bool `json:"require_consensus"` // Require node consensus
LeaderBreaksTies bool `json:"leader_breaks_ties"` // Leader resolves ties
MaxConflictAge time.Duration `json:"max_conflict_age"` // Max age before escalation
EscalationRoles []string `json:"escalation_roles"` // Roles for manual escalation
DefaultResolution ResolutionType `json:"default_resolution"` // Default resolution method
FieldPriorities map[string]int `json:"field_priorities"` // Field priority mapping
AutoMergeEnabled bool `json:"auto_merge_enabled"` // Enable automatic merging
RequireConsensus bool `json:"require_consensus"` // Require node consensus
LeaderBreaksTies bool `json:"leader_breaks_ties"` // Leader resolves ties
MaxConflictAge time.Duration `json:"max_conflict_age"` // Max age before escalation
EscalationRoles []string `json:"escalation_roles"` // Roles for manual escalation
}
// SyncResult represents the result of synchronization operation
type SyncResult struct {
SyncedContexts int `json:"synced_contexts"` // Contexts synchronized
ConflictsResolved int `json:"conflicts_resolved"` // Conflicts resolved
Errors []string `json:"errors"` // Synchronization errors
SyncTime time.Duration `json:"sync_time"` // Total sync time
PeersContacted int `json:"peers_contacted"` // Number of peers contacted
DataTransferred int64 `json:"data_transferred"` // Bytes transferred
SyncedAt time.Time `json:"synced_at"` // When sync completed
SyncedContexts int `json:"synced_contexts"` // Contexts synchronized
ConflictsResolved int `json:"conflicts_resolved"` // Conflicts resolved
Errors []string `json:"errors"` // Synchronization errors
SyncTime time.Duration `json:"sync_time"` // Total sync time
PeersContacted int `json:"peers_contacted"` // Number of peers contacted
DataTransferred int64 `json:"data_transferred"` // Bytes transferred
SyncedAt time.Time `json:"synced_at"` // When sync completed
}
// ReplicaHealth represents health status of context replicas
type ReplicaHealth struct {
Address ucxl.Address `json:"address"` // Context address
TotalReplicas int `json:"total_replicas"` // Total replica count
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
FailedReplicas int `json:"failed_replicas"` // Failed replica count
ReplicaNodes []*ReplicaNode `json:"replica_nodes"` // Individual replica status
OverallHealth HealthStatus `json:"overall_health"` // Overall health status
LastChecked time.Time `json:"last_checked"` // When last checked
RepairNeeded bool `json:"repair_needed"` // Whether repair is needed
Address ucxl.Address `json:"address"` // Context address
TotalReplicas int `json:"total_replicas"` // Total replica count
HealthyReplicas int `json:"healthy_replicas"` // Healthy replica count
FailedReplicas int `json:"failed_replicas"` // Failed replica count
ReplicaNodes []*ReplicaNode `json:"replica_nodes"` // Individual replica status
OverallHealth HealthStatus `json:"overall_health"` // Overall health status
LastChecked time.Time `json:"last_checked"` // When last checked
RepairNeeded bool `json:"repair_needed"` // Whether repair is needed
}
// ReplicaNode represents status of individual replica node
type ReplicaNode struct {
NodeID string `json:"node_id"` // Node identifier
Status ReplicaStatus `json:"status"` // Replica status
LastSeen time.Time `json:"last_seen"` // When last seen
Version int64 `json:"version"` // Context version
Checksum string `json:"checksum"` // Data checksum
Latency time.Duration `json:"latency"` // Network latency
NetworkAddress string `json:"network_address"` // Network address
NodeID string `json:"node_id"` // Node identifier
Status ReplicaStatus `json:"status"` // Replica status
LastSeen time.Time `json:"last_seen"` // When last seen
Version int64 `json:"version"` // Context version
Checksum string `json:"checksum"` // Data checksum
Latency time.Duration `json:"latency"` // Network latency
NetworkAddress string `json:"network_address"` // Network address
}
// ReplicaStatus represents status of individual replica
type ReplicaStatus string
const (
ReplicaHealthy ReplicaStatus = "healthy" // Replica is healthy
ReplicaStale ReplicaStatus = "stale" // Replica is stale
ReplicaCorrupted ReplicaStatus = "corrupted" // Replica is corrupted
ReplicaUnreachable ReplicaStatus = "unreachable" // Replica is unreachable
ReplicaSyncing ReplicaStatus = "syncing" // Replica is syncing
ReplicaHealthy ReplicaStatus = "healthy" // Replica is healthy
ReplicaStale ReplicaStatus = "stale" // Replica is stale
ReplicaCorrupted ReplicaStatus = "corrupted" // Replica is corrupted
ReplicaUnreachable ReplicaStatus = "unreachable" // Replica is unreachable
ReplicaSyncing ReplicaStatus = "syncing" // Replica is syncing
)
// HealthStatus represents overall health status
type HealthStatus string
const (
HealthHealthy HealthStatus = "healthy" // All replicas healthy
HealthDegraded HealthStatus = "degraded" // Some replicas unhealthy
HealthCritical HealthStatus = "critical" // Most replicas unhealthy
HealthFailed HealthStatus = "failed" // All replicas failed
HealthHealthy HealthStatus = "healthy" // All replicas healthy
HealthDegraded HealthStatus = "degraded" // Some replicas unhealthy
HealthCritical HealthStatus = "critical" // Most replicas unhealthy
HealthFailed HealthStatus = "failed" // All replicas failed
)
// ReplicationPolicy represents replication behavior configuration
type ReplicationPolicy struct {
DefaultFactor int `json:"default_factor"` // Default replication factor
MinFactor int `json:"min_factor"` // Minimum replication factor
MaxFactor int `json:"max_factor"` // Maximum replication factor
PreferredZones []string `json:"preferred_zones"` // Preferred availability zones
AvoidSameNode bool `json:"avoid_same_node"` // Avoid same physical node
ConsistencyLevel ConsistencyLevel `json:"consistency_level"` // Consistency requirements
RepairThreshold float64 `json:"repair_threshold"` // Health threshold for repair
RebalanceInterval time.Duration `json:"rebalance_interval"` // Rebalancing frequency
DefaultFactor int `json:"default_factor"` // Default replication factor
MinFactor int `json:"min_factor"` // Minimum replication factor
MaxFactor int `json:"max_factor"` // Maximum replication factor
PreferredZones []string `json:"preferred_zones"` // Preferred availability zones
AvoidSameNode bool `json:"avoid_same_node"` // Avoid same physical node
ConsistencyLevel ConsistencyLevel `json:"consistency_level"` // Consistency requirements
RepairThreshold float64 `json:"repair_threshold"` // Health threshold for repair
RebalanceInterval time.Duration `json:"rebalance_interval"` // Rebalancing frequency
}
// ConsistencyLevel represents consistency requirements
@@ -340,12 +346,12 @@ const (
// DHTStoreOptions represents options for DHT storage operations
type DHTStoreOptions struct {
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Duration `json:"ttl,omitempty"` // Time to live
Priority Priority `json:"priority"` // Storage priority
Compress bool `json:"compress"` // Whether to compress
Checksum bool `json:"checksum"` // Whether to checksum
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Duration `json:"ttl,omitempty"` // Time to live
Priority Priority `json:"priority"` // Storage priority
Compress bool `json:"compress"` // Whether to compress
Checksum bool `json:"checksum"` // Whether to checksum
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
}
// Priority represents storage operation priority
@@ -360,12 +366,12 @@ const (
// DHTMetadata represents metadata for DHT stored data
type DHTMetadata struct {
StoredAt time.Time `json:"stored_at"` // When stored
UpdatedAt time.Time `json:"updated_at"` // When last updated
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Time `json:"ttl,omitempty"` // Time to live
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
}
StoredAt time.Time `json:"stored_at"` // When stored
UpdatedAt time.Time `json:"updated_at"` // When last updated
Version int64 `json:"version"` // Version number
Size int64 `json:"size"` // Data size
Checksum string `json:"checksum"` // Data checksum
ReplicationFactor int `json:"replication_factor"` // Number of replicas
TTL *time.Time `json:"ttl,omitempty"` // Time to live
Metadata map[string]interface{} `json:"metadata"` // Additional metadata
}

289
pkg/slurp/distribution/dht_impl.go
View File

@@ -10,18 +10,18 @@ import (
"sync"
"time"
"chorus/pkg/dht"
"chorus/pkg/crypto"
"chorus/pkg/election"
"chorus/pkg/ucxl"
"chorus/pkg/config"
"chorus/pkg/crypto"
"chorus/pkg/dht"
"chorus/pkg/election"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/ucxl"
)
// DHTContextDistributor implements ContextDistributor using CHORUS DHT infrastructure
type DHTContextDistributor struct {
mu sync.RWMutex
dht *dht.DHT
dht dht.DHT
roleCrypto *crypto.RoleCrypto
election election.Election
config *config.Config
@@ -37,7 +37,7 @@ type DHTContextDistributor struct {
// NewDHTContextDistributor creates a new DHT-based context distributor
func NewDHTContextDistributor(
dht *dht.DHT,
dht dht.DHT,
roleCrypto *crypto.RoleCrypto,
election election.Election,
config *config.Config,
@@ -147,36 +147,43 @@ func (d *DHTContextDistributor) DistributeContext(ctx context.Context, node *slu
return d.recordError(fmt.Sprintf("failed to get vector clock: %v", err))
}
// Encrypt context for roles
encryptedData, err := d.roleCrypto.EncryptContextForRoles(node, roles, []string{})
// Prepare context payload for role encryption
rawContext, err := json.Marshal(node)
if err != nil {
return d.recordError(fmt.Sprintf("failed to encrypt context: %v", err))
return d.recordError(fmt.Sprintf("failed to marshal context: %v", err))
}
// Create distribution metadata
// Create distribution metadata (checksum calculated per-role below)
metadata := &DistributionMetadata{
Address: node.UCXLAddress,
Roles: roles,
Version: 1,
VectorClock: clock,
DistributedBy: d.config.Agent.ID,
DistributedAt: time.Now(),
Roles: roles,
Version: 1,
VectorClock: clock,
DistributedBy: d.config.Agent.ID,
DistributedAt: time.Now(),
ReplicationFactor: d.getReplicationFactor(),
Checksum: d.calculateChecksum(encryptedData),
}
// Store encrypted data in DHT for each role
for _, role := range roles {
key := d.keyGenerator.GenerateContextKey(node.UCXLAddress.String(), role)
cipher, fingerprint, err := d.roleCrypto.EncryptForRole(rawContext, role)
if err != nil {
return d.recordError(fmt.Sprintf("failed to encrypt context for role %s: %v", role, err))
}
// Create role-specific storage package
storagePackage := &ContextStoragePackage{
EncryptedData: encryptedData,
Metadata: metadata,
Role: role,
StoredAt: time.Now(),
EncryptedData: cipher,
KeyFingerprint: fingerprint,
Metadata: metadata,
Role: role,
StoredAt: time.Now(),
}
metadata.Checksum = d.calculateChecksum(cipher)
// Serialize for storage
storageBytes, err := json.Marshal(storagePackage)
if err != nil {
@@ -252,25 +259,30 @@ func (d *DHTContextDistributor) RetrieveContext(ctx context.Context, address ucx
}
// Decrypt context for role
contextNode, err := d.roleCrypto.DecryptContextForRole(storagePackage.EncryptedData, role)
plain, err := d.roleCrypto.DecryptForRole(storagePackage.EncryptedData, role, storagePackage.KeyFingerprint)
if err != nil {
return nil, d.recordRetrievalError(fmt.Sprintf("failed to decrypt context: %v", err))
}
var contextNode slurpContext.ContextNode
if err := json.Unmarshal(plain, &contextNode); err != nil {
return nil, d.recordRetrievalError(fmt.Sprintf("failed to decode context: %v", err))
}
// Convert to resolved context
resolvedContext := &slurpContext.ResolvedContext{
UCXLAddress: contextNode.UCXLAddress,
Summary: contextNode.Summary,
Purpose: contextNode.Purpose,
Technologies: contextNode.Technologies,
Tags: contextNode.Tags,
Insights: contextNode.Insights,
ContextSourcePath: contextNode.Path,
InheritanceChain: []string{contextNode.Path},
ResolutionConfidence: contextNode.RAGConfidence,
BoundedDepth: 1,
GlobalContextsApplied: false,
ResolvedAt: time.Now(),
UCXLAddress: contextNode.UCXLAddress,
Summary: contextNode.Summary,
Purpose: contextNode.Purpose,
Technologies: contextNode.Technologies,
Tags: contextNode.Tags,
Insights: contextNode.Insights,
ContextSourcePath: contextNode.Path,
InheritanceChain: []string{contextNode.Path},
ResolutionConfidence: contextNode.RAGConfidence,
BoundedDepth: 1,
GlobalContextsApplied: false,
ResolvedAt: time.Now(),
}
// Update statistics
@@ -304,15 +316,15 @@ func (d *DHTContextDistributor) UpdateContext(ctx context.Context, node *slurpCo
// Convert existing resolved context back to context node for comparison
existingNode := &slurpContext.ContextNode{
Path: existingContext.ContextSourcePath,
UCXLAddress: existingContext.UCXLAddress,
Summary: existingContext.Summary,
Purpose: existingContext.Purpose,
Technologies: existingContext.Technologies,
Tags: existingContext.Tags,
Insights: existingContext.Insights,
RAGConfidence: existingContext.ResolutionConfidence,
GeneratedAt: existingContext.ResolvedAt,
Path: existingContext.ContextSourcePath,
UCXLAddress: existingContext.UCXLAddress,
Summary: existingContext.Summary,
Purpose: existingContext.Purpose,
Technologies: existingContext.Technologies,
Tags: existingContext.Tags,
Insights: existingContext.Insights,
RAGConfidence: existingContext.ResolutionConfidence,
GeneratedAt: existingContext.ResolvedAt,
}
// Use conflict resolver to handle the update
@@ -357,7 +369,7 @@ func (d *DHTContextDistributor) DeleteContext(ctx context.Context, address ucxl.
func (d *DHTContextDistributor) ListDistributedContexts(ctx context.Context, role string, criteria *DistributionCriteria) ([]*DistributedContextInfo, error) {
// This is a simplified implementation
// In production, we'd maintain proper indexes and filtering
results := []*DistributedContextInfo{}
limit := 100
if criteria != nil && criteria.Limit > 0 {
@@ -380,13 +392,13 @@ func (d *DHTContextDistributor) Sync(ctx context.Context) (*SyncResult, error) {
}
result := &SyncResult{
SyncedContexts: 0, // Would be populated in real implementation
SyncedContexts: 0, // Would be populated in real implementation
ConflictsResolved: 0,
Errors: []string{},
SyncTime: time.Since(start),
PeersContacted: len(d.dht.GetConnectedPeers()),
DataTransferred: 0,
SyncedAt: time.Now(),
Errors: []string{},
SyncTime: time.Since(start),
PeersContacted: len(d.dht.GetConnectedPeers()),
DataTransferred: 0,
SyncedAt: time.Now(),
}
return result, nil
@@ -453,28 +465,13 @@ func (d *DHTContextDistributor) calculateChecksum(data interface{}) string {
return hex.EncodeToString(hash[:])
}
// Ensure DHT is bootstrapped before operations
func (d *DHTContextDistributor) ensureDHTReady() error {
if !d.dht.IsBootstrapped() {
return fmt.Errorf("DHT not bootstrapped")
}
return nil
}
// Start starts the distribution service
func (d *DHTContextDistributor) Start(ctx context.Context) error {
// Bootstrap DHT if not already done
if !d.dht.IsBootstrapped() {
if err := d.dht.Bootstrap(); err != nil {
return fmt.Errorf("failed to bootstrap DHT: %w", err)
if d.gossipProtocol != nil {
if err := d.gossipProtocol.StartGossip(ctx); err != nil {
return fmt.Errorf("failed to start gossip protocol: %w", err)
}
}
// Start gossip protocol
if err := d.gossipProtocol.StartGossip(ctx); err != nil {
return fmt.Errorf("failed to start gossip protocol: %w", err)
}
return nil
}
@@ -488,22 +485,23 @@ func (d *DHTContextDistributor) Stop(ctx context.Context) error {
// ContextStoragePackage represents a complete package for DHT storage
type ContextStoragePackage struct {
EncryptedData *crypto.EncryptedContextData `json:"encrypted_data"`
Metadata *DistributionMetadata `json:"metadata"`
Role string `json:"role"`
StoredAt time.Time `json:"stored_at"`
EncryptedData []byte `json:"encrypted_data"`
KeyFingerprint string `json:"key_fingerprint,omitempty"`
Metadata *DistributionMetadata `json:"metadata"`
Role string `json:"role"`
StoredAt time.Time `json:"stored_at"`
}
// DistributionMetadata contains metadata for distributed context
type DistributionMetadata struct {
Address ucxl.Address `json:"address"`
Roles []string `json:"roles"`
Version int64 `json:"version"`
VectorClock *VectorClock `json:"vector_clock"`
DistributedBy string `json:"distributed_by"`
DistributedAt time.Time `json:"distributed_at"`
ReplicationFactor int `json:"replication_factor"`
Checksum string `json:"checksum"`
Address ucxl.Address `json:"address"`
Roles []string `json:"roles"`
Version int64 `json:"version"`
VectorClock *VectorClock `json:"vector_clock"`
DistributedBy string `json:"distributed_by"`
DistributedAt time.Time `json:"distributed_at"`
ReplicationFactor int `json:"replication_factor"`
Checksum string `json:"checksum"`
}
// DHTKeyGenerator implements KeyGenerator interface
@@ -532,65 +530,124 @@ func (kg *DHTKeyGenerator) GenerateReplicationKey(address string) string {
// Component constructors - these would be implemented in separate files
// NewReplicationManager creates a new replication manager
func NewReplicationManager(dht *dht.DHT, config *config.Config) (ReplicationManager, error) {
// Placeholder implementation
return &ReplicationManagerImpl{}, nil
func NewReplicationManager(dht dht.DHT, config *config.Config) (ReplicationManager, error) {
impl, err := NewReplicationManagerImpl(dht, config)
if err != nil {
return nil, err
}
return impl, nil
}
// NewConflictResolver creates a new conflict resolver
func NewConflictResolver(dht *dht.DHT, config *config.Config) (ConflictResolver, error) {
// Placeholder implementation
func NewConflictResolver(dht dht.DHT, config *config.Config) (ConflictResolver, error) {
// Placeholder implementation until full resolver is wired
return &ConflictResolverImpl{}, nil
}
// NewGossipProtocol creates a new gossip protocol
func NewGossipProtocol(dht *dht.DHT, config *config.Config) (GossipProtocol, error) {
// Placeholder implementation
return &GossipProtocolImpl{}, nil
func NewGossipProtocol(dht dht.DHT, config *config.Config) (GossipProtocol, error) {
impl, err := NewGossipProtocolImpl(dht, config)
if err != nil {
return nil, err
}
return impl, nil
}
// NewNetworkManager creates a new network manager
func NewNetworkManager(dht *dht.DHT, config *config.Config) (NetworkManager, error) {
// Placeholder implementation
return &NetworkManagerImpl{}, nil
func NewNetworkManager(dht dht.DHT, config *config.Config) (NetworkManager, error) {
impl, err := NewNetworkManagerImpl(dht, config)
if err != nil {
return nil, err
}
return impl, nil
}
// NewVectorClockManager creates a new vector clock manager
func NewVectorClockManager(dht *dht.DHT, nodeID string) (VectorClockManager, error) {
// Placeholder implementation
return &VectorClockManagerImpl{}, nil
func NewVectorClockManager(dht dht.DHT, nodeID string) (VectorClockManager, error) {
return &defaultVectorClockManager{
clocks: make(map[string]*VectorClock),
}, nil
}
// Placeholder structs for components - these would be properly implemented
type ReplicationManagerImpl struct{}
func (rm *ReplicationManagerImpl) EnsureReplication(ctx context.Context, address ucxl.Address, factor int) error { return nil }
func (rm *ReplicationManagerImpl) GetReplicationStatus(ctx context.Context, address ucxl.Address) (*ReplicaHealth, error) {
return &ReplicaHealth{}, nil
}
func (rm *ReplicationManagerImpl) SetReplicationFactor(factor int) error { return nil }
// ConflictResolverImpl is a temporary stub until the full resolver is implemented
type ConflictResolverImpl struct{}
func (cr *ConflictResolverImpl) ResolveConflict(ctx context.Context, local, remote *slurpContext.ContextNode) (*ConflictResolution, error) {
return &ConflictResolution{
Address: local.UCXLAddress,
Address: local.UCXLAddress,
ResolutionType: ResolutionMerged,
MergedContext: local,
MergedContext: local,
ResolutionTime: time.Millisecond,
ResolvedAt: time.Now(),
Confidence: 0.95,
ResolvedAt: time.Now(),
Confidence: 0.95,
}, nil
}
type GossipProtocolImpl struct{}
func (gp *GossipProtocolImpl) StartGossip(ctx context.Context) error { return nil }
// defaultVectorClockManager provides a minimal vector clock store for SEC-SLURP scaffolding.
type defaultVectorClockManager struct {
mu sync.Mutex
clocks map[string]*VectorClock
}
type NetworkManagerImpl struct{}
func (vcm *defaultVectorClockManager) GetClock(nodeID string) (*VectorClock, error) {
vcm.mu.Lock()
defer vcm.mu.Unlock()
type VectorClockManagerImpl struct{}
func (vcm *VectorClockManagerImpl) GetClock(nodeID string) (*VectorClock, error) {
return &VectorClock{
Clock: map[string]int64{nodeID: time.Now().Unix()},
if clock, ok := vcm.clocks[nodeID]; ok {
return clock, nil
}
clock := &VectorClock{
Clock: map[string]int64{nodeID: time.Now().Unix()},
UpdatedAt: time.Now(),
}, nil
}
}
vcm.clocks[nodeID] = clock
return clock, nil
}
func (vcm *defaultVectorClockManager) UpdateClock(nodeID string, clock *VectorClock) error {
vcm.mu.Lock()
defer vcm.mu.Unlock()
vcm.clocks[nodeID] = clock
return nil
}
func (vcm *defaultVectorClockManager) CompareClock(clock1, clock2 *VectorClock) ClockRelation {
if clock1 == nil || clock2 == nil {
return ClockConcurrent
}
if clock1.UpdatedAt.Before(clock2.UpdatedAt) {
return ClockBefore
}
if clock1.UpdatedAt.After(clock2.UpdatedAt) {
return ClockAfter
}
return ClockEqual
}
func (vcm *defaultVectorClockManager) MergeClock(clocks []*VectorClock) *VectorClock {
if len(clocks) == 0 {
return &VectorClock{
Clock: map[string]int64{},
UpdatedAt: time.Now(),
}
}
merged := &VectorClock{
Clock: make(map[string]int64),
UpdatedAt: clocks[0].UpdatedAt,
}
for _, clock := range clocks {
if clock == nil {
continue
}
if clock.UpdatedAt.After(merged.UpdatedAt) {
merged.UpdatedAt = clock.UpdatedAt
}
for node, value := range clock.Clock {
if existing, ok := merged.Clock[node]; !ok || value > existing {
merged.Clock[node] = value
}
}
}
return merged
}

626
pkg/slurp/distribution/monitoring.go
View File

@@ -15,48 +15,48 @@ import (
// MonitoringSystem provides comprehensive monitoring for the distributed context system
type MonitoringSystem struct {
mu sync.RWMutex
config *config.Config
metrics *MetricsCollector
healthChecks *HealthCheckManager
alertManager *AlertManager
dashboard *DashboardServer
logManager *LogManager
traceManager *TraceManager
mu sync.RWMutex
config *config.Config
metrics *MetricsCollector
healthChecks *HealthCheckManager
alertManager *AlertManager
dashboard *DashboardServer
logManager *LogManager
traceManager *TraceManager
// State
running bool
monitoringPort int
updateInterval time.Duration
retentionPeriod time.Duration
running bool
monitoringPort int
updateInterval time.Duration
retentionPeriod time.Duration
}
// MetricsCollector collects and aggregates system metrics
type MetricsCollector struct {
mu sync.RWMutex
timeSeries map[string]*TimeSeries
counters map[string]*Counter
gauges map[string]*Gauge
histograms map[string]*Histogram
customMetrics map[string]*CustomMetric
aggregatedStats *AggregatedStatistics
exporters []MetricsExporter
lastCollection time.Time
mu sync.RWMutex
timeSeries map[string]*TimeSeries
counters map[string]*Counter
gauges map[string]*Gauge
histograms map[string]*Histogram
customMetrics map[string]*CustomMetric
aggregatedStats *AggregatedStatistics
exporters []MetricsExporter
lastCollection time.Time
}
// TimeSeries represents a time-series metric
type TimeSeries struct {
Name string `json:"name"`
Labels map[string]string `json:"labels"`
DataPoints []*TimeSeriesPoint `json:"data_points"`
Name string `json:"name"`
Labels map[string]string `json:"labels"`
DataPoints []*TimeSeriesPoint `json:"data_points"`
RetentionTTL time.Duration `json:"retention_ttl"`
LastUpdated time.Time `json:"last_updated"`
LastUpdated time.Time `json:"last_updated"`
}
// TimeSeriesPoint represents a single data point in a time series
type TimeSeriesPoint struct {
Timestamp time.Time `json:"timestamp"`
Value float64 `json:"value"`
Timestamp time.Time `json:"timestamp"`
Value float64 `json:"value"`
Labels map[string]string `json:"labels,omitempty"`
}
@@ -64,7 +64,7 @@ type TimeSeriesPoint struct {
type Counter struct {
Name string `json:"name"`
Value int64 `json:"value"`
Rate float64 `json:"rate"` // per second
Rate float64 `json:"rate"` // per second
Labels map[string]string `json:"labels"`
LastUpdated time.Time `json:"last_updated"`
}
@@ -82,13 +82,13 @@ type Gauge struct {
// Histogram represents distribution of values
type Histogram struct {
Name string `json:"name"`
Buckets map[float64]int64 `json:"buckets"`
Count int64 `json:"count"`
Sum float64 `json:"sum"`
Labels map[string]string `json:"labels"`
Name string `json:"name"`
Buckets map[float64]int64 `json:"buckets"`
Count int64 `json:"count"`
Sum float64 `json:"sum"`
Labels map[string]string `json:"labels"`
Percentiles map[float64]float64 `json:"percentiles"`
LastUpdated time.Time `json:"last_updated"`
LastUpdated time.Time `json:"last_updated"`
}
// CustomMetric represents application-specific metrics
@@ -114,81 +114,81 @@ const (
// AggregatedStatistics provides high-level system statistics
type AggregatedStatistics struct {
SystemOverview *SystemOverview `json:"system_overview"`
PerformanceMetrics *PerformanceOverview `json:"performance_metrics"`
HealthMetrics *HealthOverview `json:"health_metrics"`
ErrorMetrics *ErrorOverview `json:"error_metrics"`
ResourceMetrics *ResourceOverview `json:"resource_metrics"`
NetworkMetrics *NetworkOverview `json:"network_metrics"`
LastUpdated time.Time `json:"last_updated"`
SystemOverview *SystemOverview `json:"system_overview"`
PerformanceMetrics *PerformanceOverview `json:"performance_metrics"`
HealthMetrics *HealthOverview `json:"health_metrics"`
ErrorMetrics *ErrorOverview `json:"error_metrics"`
ResourceMetrics *ResourceOverview `json:"resource_metrics"`
NetworkMetrics *NetworkOverview `json:"network_metrics"`
LastUpdated time.Time `json:"last_updated"`
}
// SystemOverview provides system-wide overview metrics
type SystemOverview struct {
TotalNodes int `json:"total_nodes"`
HealthyNodes int `json:"healthy_nodes"`
TotalContexts int64 `json:"total_contexts"`
DistributedContexts int64 `json:"distributed_contexts"`
ReplicationFactor float64 `json:"average_replication_factor"`
SystemUptime time.Duration `json:"system_uptime"`
ClusterVersion string `json:"cluster_version"`
LastRestart time.Time `json:"last_restart"`
TotalNodes int `json:"total_nodes"`
HealthyNodes int `json:"healthy_nodes"`
TotalContexts int64 `json:"total_contexts"`
DistributedContexts int64 `json:"distributed_contexts"`
ReplicationFactor float64 `json:"average_replication_factor"`
SystemUptime time.Duration `json:"system_uptime"`
ClusterVersion string `json:"cluster_version"`
LastRestart time.Time `json:"last_restart"`
}
// PerformanceOverview provides performance metrics
type PerformanceOverview struct {
RequestsPerSecond float64 `json:"requests_per_second"`
AverageResponseTime time.Duration `json:"average_response_time"`
P95ResponseTime time.Duration `json:"p95_response_time"`
P99ResponseTime time.Duration `json:"p99_response_time"`
Throughput float64 `json:"throughput_mbps"`
CacheHitRate float64 `json:"cache_hit_rate"`
QueueDepth int `json:"queue_depth"`
ActiveConnections int `json:"active_connections"`
RequestsPerSecond float64 `json:"requests_per_second"`
AverageResponseTime time.Duration `json:"average_response_time"`
P95ResponseTime time.Duration `json:"p95_response_time"`
P99ResponseTime time.Duration `json:"p99_response_time"`
Throughput float64 `json:"throughput_mbps"`
CacheHitRate float64 `json:"cache_hit_rate"`
QueueDepth int `json:"queue_depth"`
ActiveConnections int `json:"active_connections"`
}
// HealthOverview provides health-related metrics
type HealthOverview struct {
OverallHealthScore float64 `json:"overall_health_score"`
ComponentHealth map[string]float64 `json:"component_health"`
FailedHealthChecks int `json:"failed_health_checks"`
LastHealthCheck time.Time `json:"last_health_check"`
HealthTrend string `json:"health_trend"` // improving, stable, degrading
CriticalAlerts int `json:"critical_alerts"`
WarningAlerts int `json:"warning_alerts"`
OverallHealthScore float64 `json:"overall_health_score"`
ComponentHealth map[string]float64 `json:"component_health"`
FailedHealthChecks int `json:"failed_health_checks"`
LastHealthCheck time.Time `json:"last_health_check"`
HealthTrend string `json:"health_trend"` // improving, stable, degrading
CriticalAlerts int `json:"critical_alerts"`
WarningAlerts int `json:"warning_alerts"`
}
// ErrorOverview provides error-related metrics
type ErrorOverview struct {
TotalErrors int64 `json:"total_errors"`
ErrorRate float64 `json:"error_rate"`
ErrorsByType map[string]int64 `json:"errors_by_type"`
ErrorsByComponent map[string]int64 `json:"errors_by_component"`
LastError *ErrorEvent `json:"last_error"`
ErrorTrend string `json:"error_trend"` // increasing, stable, decreasing
TotalErrors int64 `json:"total_errors"`
ErrorRate float64 `json:"error_rate"`
ErrorsByType map[string]int64 `json:"errors_by_type"`
ErrorsByComponent map[string]int64 `json:"errors_by_component"`
LastError *ErrorEvent `json:"last_error"`
ErrorTrend string `json:"error_trend"` // increasing, stable, decreasing
}
// ResourceOverview provides resource utilization metrics
type ResourceOverview struct {
CPUUtilization float64 `json:"cpu_utilization"`
MemoryUtilization float64 `json:"memory_utilization"`
DiskUtilization float64 `json:"disk_utilization"`
NetworkUtilization float64 `json:"network_utilization"`
StorageUsed int64 `json:"storage_used_bytes"`
StorageAvailable int64 `json:"storage_available_bytes"`
FileDescriptors int `json:"open_file_descriptors"`
Goroutines int `json:"goroutines"`
CPUUtilization float64 `json:"cpu_utilization"`
MemoryUtilization float64 `json:"memory_utilization"`
DiskUtilization float64 `json:"disk_utilization"`
NetworkUtilization float64 `json:"network_utilization"`
StorageUsed int64 `json:"storage_used_bytes"`
StorageAvailable int64 `json:"storage_available_bytes"`
FileDescriptors int `json:"open_file_descriptors"`
Goroutines int `json:"goroutines"`
}
// NetworkOverview provides network-related metrics
type NetworkOverview struct {
TotalConnections int `json:"total_connections"`
ActiveConnections int `json:"active_connections"`
BandwidthUtilization float64 `json:"bandwidth_utilization"`
PacketLossRate float64 `json:"packet_loss_rate"`
AverageLatency time.Duration `json:"average_latency"`
NetworkPartitions int `json:"network_partitions"`
DataTransferred int64 `json:"data_transferred_bytes"`
TotalConnections int `json:"total_connections"`
ActiveConnections int `json:"active_connections"`
BandwidthUtilization float64 `json:"bandwidth_utilization"`
PacketLossRate float64 `json:"packet_loss_rate"`
AverageLatency time.Duration `json:"average_latency"`
NetworkPartitions int `json:"network_partitions"`
DataTransferred int64 `json:"data_transferred_bytes"`
}
// MetricsExporter exports metrics to external systems
@@ -200,49 +200,49 @@ type MetricsExporter interface {
// HealthCheckManager manages system health checks
type HealthCheckManager struct {
mu sync.RWMutex
healthChecks map[string]*HealthCheck
checkResults map[string]*HealthCheckResult
schedules map[string]*HealthCheckSchedule
running bool
mu sync.RWMutex
healthChecks map[string]*HealthCheck
checkResults map[string]*HealthCheckResult
schedules map[string]*HealthCheckSchedule
running bool
}
// HealthCheck represents a single health check
type HealthCheck struct {
Name string `json:"name"`
Description string `json:"description"`
CheckType HealthCheckType `json:"check_type"`
Target string `json:"target"`
Timeout time.Duration `json:"timeout"`
Interval time.Duration `json:"interval"`
Retries int `json:"retries"`
Metadata map[string]interface{} `json:"metadata"`
Enabled bool `json:"enabled"`
CheckFunction func(context.Context) (*HealthCheckResult, error) `json:"-"`
Name string `json:"name"`
Description string `json:"description"`
CheckType HealthCheckType `json:"check_type"`
Target string `json:"target"`
Timeout time.Duration `json:"timeout"`
Interval time.Duration `json:"interval"`
Retries int `json:"retries"`
Metadata map[string]interface{} `json:"metadata"`
Enabled bool `json:"enabled"`
CheckFunction func(context.Context) (*HealthCheckResult, error) `json:"-"`
}
// HealthCheckType represents different types of health checks
type HealthCheckType string
const (
HealthCheckTypeHTTP HealthCheckType = "http"
HealthCheckTypeTCP HealthCheckType = "tcp"
HealthCheckTypeCustom HealthCheckType = "custom"
HealthCheckTypeComponent HealthCheckType = "component"
HealthCheckTypeDatabase HealthCheckType = "database"
HealthCheckTypeService HealthCheckType = "service"
HealthCheckTypeHTTP HealthCheckType = "http"
HealthCheckTypeTCP HealthCheckType = "tcp"
HealthCheckTypeCustom HealthCheckType = "custom"
HealthCheckTypeComponent HealthCheckType = "component"
HealthCheckTypeDatabase HealthCheckType = "database"
HealthCheckTypeService HealthCheckType = "service"
)
// HealthCheckResult represents the result of a health check
type HealthCheckResult struct {
CheckName string `json:"check_name"`
Status HealthCheckStatus `json:"status"`
ResponseTime time.Duration `json:"response_time"`
Message string `json:"message"`
Details map[string]interface{} `json:"details"`
Error string `json:"error,omitempty"`
Timestamp time.Time `json:"timestamp"`
Attempt int `json:"attempt"`
CheckName string `json:"check_name"`
Status HealthCheckStatus `json:"status"`
ResponseTime time.Duration `json:"response_time"`
Message string `json:"message"`
Details map[string]interface{} `json:"details"`
Error string `json:"error,omitempty"`
Timestamp time.Time `json:"timestamp"`
Attempt int `json:"attempt"`
}
// HealthCheckStatus represents the status of a health check
@@ -258,45 +258,45 @@ const (
// HealthCheckSchedule defines when health checks should run
type HealthCheckSchedule struct {
CheckName string `json:"check_name"`
Interval time.Duration `json:"interval"`
NextRun time.Time `json:"next_run"`
LastRun time.Time `json:"last_run"`
Enabled bool `json:"enabled"`
FailureCount int `json:"failure_count"`
CheckName string `json:"check_name"`
Interval time.Duration `json:"interval"`
NextRun time.Time `json:"next_run"`
LastRun time.Time `json:"last_run"`
Enabled bool `json:"enabled"`
FailureCount int `json:"failure_count"`
}
// AlertManager manages system alerts and notifications
type AlertManager struct {
mu sync.RWMutex
alertRules map[string]*AlertRule
activeAlerts map[string]*Alert
alertHistory []*Alert
notifiers []AlertNotifier
silences map[string]*AlertSilence
running bool
mu sync.RWMutex
alertRules map[string]*AlertRule
activeAlerts map[string]*Alert
alertHistory []*Alert
notifiers []AlertNotifier
silences map[string]*AlertSilence
running bool
}
// AlertRule defines conditions for triggering alerts
type AlertRule struct {
Name string `json:"name"`
Description string `json:"description"`
Severity AlertSeverity `json:"severity"`
Conditions []*AlertCondition `json:"conditions"`
Duration time.Duration `json:"duration"` // How long condition must persist
Cooldown time.Duration `json:"cooldown"` // Minimum time between alerts
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
Enabled bool `json:"enabled"`
LastTriggered *time.Time `json:"last_triggered,omitempty"`
Name string `json:"name"`
Description string `json:"description"`
Severity AlertSeverity `json:"severity"`
Conditions []*AlertCondition `json:"conditions"`
Duration time.Duration `json:"duration"` // How long condition must persist
Cooldown time.Duration `json:"cooldown"` // Minimum time between alerts
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
Enabled bool `json:"enabled"`
LastTriggered *time.Time `json:"last_triggered,omitempty"`
}
// AlertCondition defines a single condition for an alert
type AlertCondition struct {
MetricName string `json:"metric_name"`
Operator ConditionOperator `json:"operator"`
Threshold float64 `json:"threshold"`
Duration time.Duration `json:"duration"`
MetricName string `json:"metric_name"`
Operator ConditionOperator `json:"operator"`
Threshold float64 `json:"threshold"`
Duration time.Duration `json:"duration"`
}
// ConditionOperator represents comparison operators for alert conditions
@@ -313,39 +313,39 @@ const (
// Alert represents an active alert
type Alert struct {
ID string `json:"id"`
RuleName string `json:"rule_name"`
Severity AlertSeverity `json:"severity"`
Status AlertStatus `json:"status"`
Message string `json:"message"`
Details map[string]interface{} `json:"details"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
StartsAt time.Time `json:"starts_at"`
EndsAt *time.Time `json:"ends_at,omitempty"`
LastUpdated time.Time `json:"last_updated"`
AckBy string `json:"acknowledged_by,omitempty"`
AckAt *time.Time `json:"acknowledged_at,omitempty"`
ID string `json:"id"`
RuleName string `json:"rule_name"`
Severity AlertSeverity `json:"severity"`
Status AlertStatus `json:"status"`
Message string `json:"message"`
Details map[string]interface{} `json:"details"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
StartsAt time.Time `json:"starts_at"`
EndsAt *time.Time `json:"ends_at,omitempty"`
LastUpdated time.Time `json:"last_updated"`
AckBy string `json:"acknowledged_by,omitempty"`
AckAt *time.Time `json:"acknowledged_at,omitempty"`
}
// AlertSeverity represents the severity level of an alert
type AlertSeverity string
const (
SeverityInfo AlertSeverity = "info"
SeverityWarning AlertSeverity = "warning"
SeverityError AlertSeverity = "error"
SeverityCritical AlertSeverity = "critical"
AlertAlertSeverityInfo AlertSeverity = "info"
AlertAlertSeverityWarning AlertSeverity = "warning"
AlertAlertSeverityError AlertSeverity = "error"
AlertAlertSeverityCritical AlertSeverity = "critical"
)
// AlertStatus represents the current status of an alert
type AlertStatus string
const (
AlertStatusFiring AlertStatus = "firing"
AlertStatusResolved AlertStatus = "resolved"
AlertStatusFiring AlertStatus = "firing"
AlertStatusResolved AlertStatus = "resolved"
AlertStatusAcknowledged AlertStatus = "acknowledged"
AlertStatusSilenced AlertStatus = "silenced"
AlertStatusSilenced AlertStatus = "silenced"
)
// AlertNotifier sends alert notifications
@@ -357,64 +357,64 @@ type AlertNotifier interface {
// AlertSilence represents a silenced alert
type AlertSilence struct {
ID string `json:"id"`
Matchers map[string]string `json:"matchers"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
CreatedBy string `json:"created_by"`
Comment string `json:"comment"`
Active bool `json:"active"`
ID string `json:"id"`
Matchers map[string]string `json:"matchers"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
CreatedBy string `json:"created_by"`
Comment string `json:"comment"`
Active bool `json:"active"`
}
// DashboardServer provides web-based monitoring dashboard
type DashboardServer struct {
mu sync.RWMutex
server *http.Server
dashboards map[string]*Dashboard
widgets map[string]*Widget
customPages map[string]*CustomPage
running bool
port int
mu sync.RWMutex
server *http.Server
dashboards map[string]*Dashboard
widgets map[string]*Widget
customPages map[string]*CustomPage
running bool
port int
}
// Dashboard represents a monitoring dashboard
type Dashboard struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Widgets []*Widget `json:"widgets"`
Layout *DashboardLayout `json:"layout"`
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Widgets []*Widget `json:"widgets"`
Layout *DashboardLayout `json:"layout"`
Settings *DashboardSettings `json:"settings"`
CreatedBy string `json:"created_by"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
CreatedBy string `json:"created_by"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// Widget represents a dashboard widget
type Widget struct {
ID string `json:"id"`
Type WidgetType `json:"type"`
Title string `json:"title"`
DataSource string `json:"data_source"`
Query string `json:"query"`
Settings map[string]interface{} `json:"settings"`
Position *WidgetPosition `json:"position"`
RefreshRate time.Duration `json:"refresh_rate"`
LastUpdated time.Time `json:"last_updated"`
ID string `json:"id"`
Type WidgetType `json:"type"`
Title string `json:"title"`
DataSource string `json:"data_source"`
Query string `json:"query"`
Settings map[string]interface{} `json:"settings"`
Position *WidgetPosition `json:"position"`
RefreshRate time.Duration `json:"refresh_rate"`
LastUpdated time.Time `json:"last_updated"`
}
// WidgetType represents different types of dashboard widgets
type WidgetType string
const (
WidgetTypeMetric WidgetType = "metric"
WidgetTypeChart WidgetType = "chart"
WidgetTypeTable WidgetType = "table"
WidgetTypeAlert WidgetType = "alert"
WidgetTypeHealth WidgetType = "health"
WidgetTypeTopology WidgetType = "topology"
WidgetTypeLog WidgetType = "log"
WidgetTypeCustom WidgetType = "custom"
WidgetTypeMetric WidgetType = "metric"
WidgetTypeChart WidgetType = "chart"
WidgetTypeTable WidgetType = "table"
WidgetTypeAlert WidgetType = "alert"
WidgetTypeHealth WidgetType = "health"
WidgetTypeTopology WidgetType = "topology"
WidgetTypeLog WidgetType = "log"
WidgetTypeCustom WidgetType = "custom"
)
// WidgetPosition defines widget position and size
@@ -427,11 +427,11 @@ type WidgetPosition struct {
// DashboardLayout defines dashboard layout settings
type DashboardLayout struct {
Columns int `json:"columns"`
RowHeight int `json:"row_height"`
Margins [2]int `json:"margins"` // [x, y]
Spacing [2]int `json:"spacing"` // [x, y]
Breakpoints map[string]int `json:"breakpoints"`
Columns int `json:"columns"`
RowHeight int `json:"row_height"`
Margins [2]int `json:"margins"` // [x, y]
Spacing [2]int `json:"spacing"` // [x, y]
Breakpoints map[string]int `json:"breakpoints"`
}
// DashboardSettings contains dashboard configuration
@@ -446,43 +446,43 @@ type DashboardSettings struct {
// CustomPage represents a custom monitoring page
type CustomPage struct {
Path string `json:"path"`
Title string `json:"title"`
Content string `json:"content"`
ContentType string `json:"content_type"`
Handler http.HandlerFunc `json:"-"`
Path string `json:"path"`
Title string `json:"title"`
Content string `json:"content"`
ContentType string `json:"content_type"`
Handler http.HandlerFunc `json:"-"`
}
// LogManager manages system logs and log analysis
type LogManager struct {
mu sync.RWMutex
logSources map[string]*LogSource
logEntries []*LogEntry
logAnalyzers []LogAnalyzer
mu sync.RWMutex
logSources map[string]*LogSource
logEntries []*LogEntry
logAnalyzers []LogAnalyzer
retentionPolicy *LogRetentionPolicy
running bool
running bool
}
// LogSource represents a source of log data
type LogSource struct {
Name string `json:"name"`
Type LogSourceType `json:"type"`
Location string `json:"location"`
Format LogFormat `json:"format"`
Labels map[string]string `json:"labels"`
Enabled bool `json:"enabled"`
LastRead time.Time `json:"last_read"`
Name string `json:"name"`
Type LogSourceType `json:"type"`
Location string `json:"location"`
Format LogFormat `json:"format"`
Labels map[string]string `json:"labels"`
Enabled bool `json:"enabled"`
LastRead time.Time `json:"last_read"`
}
// LogSourceType represents different types of log sources
type LogSourceType string
const (
LogSourceTypeFile LogSourceType = "file"
LogSourceTypeHTTP LogSourceType = "http"
LogSourceTypeStream LogSourceType = "stream"
LogSourceTypeDatabase LogSourceType = "database"
LogSourceTypeCustom LogSourceType = "custom"
LogSourceTypeFile LogSourceType = "file"
LogSourceTypeHTTP LogSourceType = "http"
LogSourceTypeStream LogSourceType = "stream"
LogSourceTypeDatabase LogSourceType = "database"
LogSourceTypeCustom LogSourceType = "custom"
)
// LogFormat represents log entry format
@@ -497,14 +497,14 @@ const (
// LogEntry represents a single log entry
type LogEntry struct {
Timestamp time.Time `json:"timestamp"`
Level LogLevel `json:"level"`
Source string `json:"source"`
Message string `json:"message"`
Fields map[string]interface{} `json:"fields"`
Labels map[string]string `json:"labels"`
TraceID string `json:"trace_id,omitempty"`
SpanID string `json:"span_id,omitempty"`
Timestamp time.Time `json:"timestamp"`
Level LogLevel `json:"level"`
Source string `json:"source"`
Message string `json:"message"`
Fields map[string]interface{} `json:"fields"`
Labels map[string]string `json:"labels"`
TraceID string `json:"trace_id,omitempty"`
SpanID string `json:"span_id,omitempty"`
}
// LogLevel represents log entry severity
@@ -527,22 +527,22 @@ type LogAnalyzer interface {
// LogAnalysisResult represents the result of log analysis
type LogAnalysisResult struct {
AnalyzerName string `json:"analyzer_name"`
Anomalies []*LogAnomaly `json:"anomalies"`
Patterns []*LogPattern `json:"patterns"`
Statistics *LogStatistics `json:"statistics"`
Recommendations []string `json:"recommendations"`
AnalyzedAt time.Time `json:"analyzed_at"`
AnalyzerName string `json:"analyzer_name"`
Anomalies []*LogAnomaly `json:"anomalies"`
Patterns []*LogPattern `json:"patterns"`
Statistics *LogStatistics `json:"statistics"`
Recommendations []string `json:"recommendations"`
AnalyzedAt time.Time `json:"analyzed_at"`
}
// LogAnomaly represents detected log anomaly
type LogAnomaly struct {
Type AnomalyType `json:"type"`
Severity AlertSeverity `json:"severity"`
Description string `json:"description"`
Entries []*LogEntry `json:"entries"`
Confidence float64 `json:"confidence"`
DetectedAt time.Time `json:"detected_at"`
Type AnomalyType `json:"type"`
Severity AlertSeverity `json:"severity"`
Description string `json:"description"`
Entries []*LogEntry `json:"entries"`
Confidence float64 `json:"confidence"`
DetectedAt time.Time `json:"detected_at"`
}
// AnomalyType represents different types of log anomalies
@@ -558,38 +558,38 @@ const (
// LogPattern represents detected log pattern
type LogPattern struct {
Pattern string `json:"pattern"`
Frequency int `json:"frequency"`
LastSeen time.Time `json:"last_seen"`
Sources []string `json:"sources"`
Confidence float64 `json:"confidence"`
Pattern string `json:"pattern"`
Frequency int `json:"frequency"`
LastSeen time.Time `json:"last_seen"`
Sources []string `json:"sources"`
Confidence float64 `json:"confidence"`
}
// LogStatistics provides log statistics
type LogStatistics struct {
TotalEntries int64 `json:"total_entries"`
EntriesByLevel map[LogLevel]int64 `json:"entries_by_level"`
EntriesBySource map[string]int64 `json:"entries_by_source"`
ErrorRate float64 `json:"error_rate"`
AverageRate float64 `json:"average_rate"`
TimeRange [2]time.Time `json:"time_range"`
TotalEntries int64 `json:"total_entries"`
EntriesByLevel map[LogLevel]int64 `json:"entries_by_level"`
EntriesBySource map[string]int64 `json:"entries_by_source"`
ErrorRate float64 `json:"error_rate"`
AverageRate float64 `json:"average_rate"`
TimeRange [2]time.Time `json:"time_range"`
}
// LogRetentionPolicy defines log retention rules
type LogRetentionPolicy struct {
RetentionPeriod time.Duration `json:"retention_period"`
MaxEntries int64 `json:"max_entries"`
CompressionAge time.Duration `json:"compression_age"`
ArchiveAge time.Duration `json:"archive_age"`
Rules []*RetentionRule `json:"rules"`
RetentionPeriod time.Duration `json:"retention_period"`
MaxEntries int64 `json:"max_entries"`
CompressionAge time.Duration `json:"compression_age"`
ArchiveAge time.Duration `json:"archive_age"`
Rules []*RetentionRule `json:"rules"`
}
// RetentionRule defines specific retention rules
type RetentionRule struct {
Name string `json:"name"`
Condition string `json:"condition"` // Query expression
Retention time.Duration `json:"retention"`
Action RetentionAction `json:"action"`
Name string `json:"name"`
Condition string `json:"condition"` // Query expression
Retention time.Duration `json:"retention"`
Action RetentionAction `json:"action"`
}
// RetentionAction represents retention actions
@@ -603,47 +603,47 @@ const (
// TraceManager manages distributed tracing
type TraceManager struct {
mu sync.RWMutex
traces map[string]*Trace
spans map[string]*Span
samplers []TraceSampler
exporters []TraceExporter
running bool
mu sync.RWMutex
traces map[string]*Trace
spans map[string]*Span
samplers []TraceSampler
exporters []TraceExporter
running bool
}
// Trace represents a distributed trace
type Trace struct {
TraceID string `json:"trace_id"`
Spans []*Span `json:"spans"`
Duration time.Duration `json:"duration"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Status TraceStatus `json:"status"`
Tags map[string]string `json:"tags"`
Operations []string `json:"operations"`
TraceID string `json:"trace_id"`
Spans []*Span `json:"spans"`
Duration time.Duration `json:"duration"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Status TraceStatus `json:"status"`
Tags map[string]string `json:"tags"`
Operations []string `json:"operations"`
}
// Span represents a single span in a trace
type Span struct {
SpanID string `json:"span_id"`
TraceID string `json:"trace_id"`
ParentID string `json:"parent_id,omitempty"`
Operation string `json:"operation"`
Service string `json:"service"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Duration time.Duration `json:"duration"`
Status SpanStatus `json:"status"`
Tags map[string]string `json:"tags"`
Logs []*SpanLog `json:"logs"`
SpanID string `json:"span_id"`
TraceID string `json:"trace_id"`
ParentID string `json:"parent_id,omitempty"`
Operation string `json:"operation"`
Service string `json:"service"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Duration time.Duration `json:"duration"`
Status SpanStatus `json:"status"`
Tags map[string]string `json:"tags"`
Logs []*SpanLog `json:"logs"`
}
// TraceStatus represents the status of a trace
type TraceStatus string
const (
TraceStatusOK TraceStatus = "ok"
TraceStatusError TraceStatus = "error"
TraceStatusOK TraceStatus = "ok"
TraceStatusError TraceStatus = "error"
TraceStatusTimeout TraceStatus = "timeout"
)
@@ -675,18 +675,18 @@ type TraceExporter interface {
// ErrorEvent represents a system error event
type ErrorEvent struct {
ID string `json:"id"`
Timestamp time.Time `json:"timestamp"`
Level LogLevel `json:"level"`
Component string `json:"component"`
Message string `json:"message"`
Error string `json:"error"`
Context map[string]interface{} `json:"context"`
TraceID string `json:"trace_id,omitempty"`
SpanID string `json:"span_id,omitempty"`
Count int `json:"count"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
ID string `json:"id"`
Timestamp time.Time `json:"timestamp"`
Level LogLevel `json:"level"`
Component string `json:"component"`
Message string `json:"message"`
Error string `json:"error"`
Context map[string]interface{} `json:"context"`
TraceID string `json:"trace_id,omitempty"`
SpanID string `json:"span_id,omitempty"`
Count int `json:"count"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
}
// NewMonitoringSystem creates a comprehensive monitoring system
@@ -722,7 +722,7 @@ func (ms *MonitoringSystem) initializeComponents() error {
aggregatedStats: &AggregatedStatistics{
LastUpdated: time.Now(),
},
exporters: []MetricsExporter{},
exporters: []MetricsExporter{},
lastCollection: time.Now(),
}
@@ -1134,15 +1134,15 @@ func (ms *MonitoringSystem) createDefaultDashboards() {
func (ms *MonitoringSystem) severityWeight(severity AlertSeverity) int {
switch severity {
case SeverityCritical:
case AlertSeverityCritical:
return 4
case SeverityError:
case AlertSeverityError:
return 3
case SeverityWarning:
case AlertSeverityWarning:
return 2
case SeverityInfo:
case AlertSeverityInfo:
return 1
default:
return 0
}
}
}

330
pkg/slurp/distribution/network.go
View File

@@ -9,74 +9,74 @@ import (
"sync"
"time"
"chorus/pkg/dht"
"chorus/pkg/config"
"chorus/pkg/dht"
"github.com/libp2p/go-libp2p/core/peer"
)
// NetworkManagerImpl implements NetworkManager interface for network topology and partition management
type NetworkManagerImpl struct {
mu sync.RWMutex
dht *dht.DHT
config *config.Config
topology *NetworkTopology
partitionInfo *PartitionInfo
connectivity *ConnectivityMatrix
stats *NetworkStatistics
healthChecker *NetworkHealthChecker
partitionDetector *PartitionDetector
recoveryManager *RecoveryManager
mu sync.RWMutex
dht *dht.DHT
config *config.Config
topology *NetworkTopology
partitionInfo *PartitionInfo
connectivity *ConnectivityMatrix
stats *NetworkStatistics
healthChecker *NetworkHealthChecker
partitionDetector *PartitionDetector
recoveryManager *RecoveryManager
// Configuration
healthCheckInterval time.Duration
healthCheckInterval time.Duration
partitionCheckInterval time.Duration
connectivityTimeout time.Duration
maxPartitionDuration time.Duration
connectivityTimeout time.Duration
maxPartitionDuration time.Duration
// State
lastTopologyUpdate time.Time
lastPartitionCheck time.Time
running bool
recoveryInProgress bool
lastTopologyUpdate time.Time
lastPartitionCheck time.Time
running bool
recoveryInProgress bool
}
// ConnectivityMatrix tracks connectivity between all nodes
type ConnectivityMatrix struct {
Matrix map[string]map[string]*ConnectionInfo `json:"matrix"`
LastUpdated time.Time `json:"last_updated"`
LastUpdated time.Time `json:"last_updated"`
mu sync.RWMutex
}
// ConnectionInfo represents connectivity information between two nodes
type ConnectionInfo struct {
Connected bool `json:"connected"`
Latency time.Duration `json:"latency"`
PacketLoss float64 `json:"packet_loss"`
Bandwidth int64 `json:"bandwidth"`
LastChecked time.Time `json:"last_checked"`
ErrorCount int `json:"error_count"`
LastError string `json:"last_error,omitempty"`
Connected bool `json:"connected"`
Latency time.Duration `json:"latency"`
PacketLoss float64 `json:"packet_loss"`
Bandwidth int64 `json:"bandwidth"`
LastChecked time.Time `json:"last_checked"`
ErrorCount int `json:"error_count"`
LastError string `json:"last_error,omitempty"`
}
// NetworkHealthChecker performs network health checks
type NetworkHealthChecker struct {
mu sync.RWMutex
nodeHealth map[string]*NodeHealth
healthHistory map[string][]*HealthCheckResult
healthHistory map[string][]*NetworkHealthCheckResult
alertThresholds *NetworkAlertThresholds
}
// NodeHealth represents health status of a network node
type NodeHealth struct {
NodeID string `json:"node_id"`
Status NodeStatus `json:"status"`
HealthScore float64 `json:"health_score"`
LastSeen time.Time `json:"last_seen"`
ResponseTime time.Duration `json:"response_time"`
PacketLossRate float64 `json:"packet_loss_rate"`
BandwidthUtil float64 `json:"bandwidth_utilization"`
Uptime time.Duration `json:"uptime"`
ErrorRate float64 `json:"error_rate"`
NodeID string `json:"node_id"`
Status NodeStatus `json:"status"`
HealthScore float64 `json:"health_score"`
LastSeen time.Time `json:"last_seen"`
ResponseTime time.Duration `json:"response_time"`
PacketLossRate float64 `json:"packet_loss_rate"`
BandwidthUtil float64 `json:"bandwidth_utilization"`
Uptime time.Duration `json:"uptime"`
ErrorRate float64 `json:"error_rate"`
}
// NodeStatus represents the status of a network node
@@ -91,23 +91,23 @@ const (
)
// HealthCheckResult represents the result of a health check
type HealthCheckResult struct {
NodeID string `json:"node_id"`
Timestamp time.Time `json:"timestamp"`
Success bool `json:"success"`
ResponseTime time.Duration `json:"response_time"`
ErrorMessage string `json:"error_message,omitempty"`
type NetworkHealthCheckResult struct {
NodeID string `json:"node_id"`
Timestamp time.Time `json:"timestamp"`
Success bool `json:"success"`
ResponseTime time.Duration `json:"response_time"`
ErrorMessage string `json:"error_message,omitempty"`
NetworkMetrics *NetworkMetrics `json:"network_metrics"`
}
// NetworkAlertThresholds defines thresholds for network alerts
type NetworkAlertThresholds struct {
LatencyWarning time.Duration `json:"latency_warning"`
LatencyCritical time.Duration `json:"latency_critical"`
PacketLossWarning float64 `json:"packet_loss_warning"`
PacketLossCritical float64 `json:"packet_loss_critical"`
HealthScoreWarning float64 `json:"health_score_warning"`
HealthScoreCritical float64 `json:"health_score_critical"`
LatencyWarning time.Duration `json:"latency_warning"`
LatencyCritical time.Duration `json:"latency_critical"`
PacketLossWarning float64 `json:"packet_loss_warning"`
PacketLossCritical float64 `json:"packet_loss_critical"`
HealthScoreWarning float64 `json:"health_score_warning"`
HealthScoreCritical float64 `json:"health_score_critical"`
}
// PartitionDetector detects network partitions
@@ -131,14 +131,14 @@ const (
// PartitionEvent represents a partition detection event
type PartitionEvent struct {
EventID string `json:"event_id"`
DetectedAt time.Time `json:"detected_at"`
EventID string `json:"event_id"`
DetectedAt time.Time `json:"detected_at"`
Algorithm PartitionDetectionAlgorithm `json:"algorithm"`
PartitionedNodes []string `json:"partitioned_nodes"`
Confidence float64 `json:"confidence"`
Duration time.Duration `json:"duration"`
Resolved bool `json:"resolved"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
PartitionedNodes []string `json:"partitioned_nodes"`
Confidence float64 `json:"confidence"`
Duration time.Duration `json:"duration"`
Resolved bool `json:"resolved"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
}
// FalsePositiveFilter helps reduce false partition detections
@@ -159,10 +159,10 @@ type PartitionDetectorConfig struct {
// RecoveryManager manages network partition recovery
type RecoveryManager struct {
mu sync.RWMutex
mu sync.RWMutex
recoveryStrategies map[RecoveryStrategy]*RecoveryStrategyConfig
activeRecoveries map[string]*RecoveryOperation
recoveryHistory []*RecoveryResult
activeRecoveries map[string]*RecoveryOperation
recoveryHistory []*RecoveryResult
}
// RecoveryStrategy represents different recovery strategies
@@ -177,25 +177,25 @@ const (
// RecoveryStrategyConfig configures a recovery strategy
type RecoveryStrategyConfig struct {
Strategy RecoveryStrategy `json:"strategy"`
Timeout time.Duration `json:"timeout"`
RetryAttempts int `json:"retry_attempts"`
RetryInterval time.Duration `json:"retry_interval"`
RequireConsensus bool `json:"require_consensus"`
ForcedThreshold time.Duration `json:"forced_threshold"`
Strategy RecoveryStrategy `json:"strategy"`
Timeout time.Duration `json:"timeout"`
RetryAttempts int `json:"retry_attempts"`
RetryInterval time.Duration `json:"retry_interval"`
RequireConsensus bool `json:"require_consensus"`
ForcedThreshold time.Duration `json:"forced_threshold"`
}
// RecoveryOperation represents an active recovery operation
type RecoveryOperation struct {
OperationID string `json:"operation_id"`
Strategy RecoveryStrategy `json:"strategy"`
StartedAt time.Time `json:"started_at"`
TargetNodes []string `json:"target_nodes"`
Status RecoveryStatus `json:"status"`
Progress float64 `json:"progress"`
CurrentPhase RecoveryPhase `json:"current_phase"`
Errors []string `json:"errors"`
LastUpdate time.Time `json:"last_update"`
OperationID string `json:"operation_id"`
Strategy RecoveryStrategy `json:"strategy"`
StartedAt time.Time `json:"started_at"`
TargetNodes []string `json:"target_nodes"`
Status RecoveryStatus `json:"status"`
Progress float64 `json:"progress"`
CurrentPhase RecoveryPhase `json:"current_phase"`
Errors []string `json:"errors"`
LastUpdate time.Time `json:"last_update"`
}
// RecoveryStatus represents the status of a recovery operation
@@ -213,12 +213,12 @@ const (
type RecoveryPhase string
const (
RecoveryPhaseAssessment RecoveryPhase = "assessment"
RecoveryPhasePreparation RecoveryPhase = "preparation"
RecoveryPhaseReconnection RecoveryPhase = "reconnection"
RecoveryPhaseAssessment RecoveryPhase = "assessment"
RecoveryPhasePreparation RecoveryPhase = "preparation"
RecoveryPhaseReconnection RecoveryPhase = "reconnection"
RecoveryPhaseSynchronization RecoveryPhase = "synchronization"
RecoveryPhaseValidation RecoveryPhase = "validation"
RecoveryPhaseCompletion RecoveryPhase = "completion"
RecoveryPhaseValidation RecoveryPhase = "validation"
RecoveryPhaseCompletion RecoveryPhase = "completion"
)
// NewNetworkManagerImpl creates a new network manager implementation
@@ -231,13 +231,13 @@ func NewNetworkManagerImpl(dht *dht.DHT, config *config.Config) (*NetworkManager
}
nm := &NetworkManagerImpl{
dht: dht,
config: config,
healthCheckInterval: 30 * time.Second,
partitionCheckInterval: 60 * time.Second,
connectivityTimeout: 10 * time.Second,
maxPartitionDuration: 10 * time.Minute,
connectivity: &ConnectivityMatrix{Matrix: make(map[string]map[string]*ConnectionInfo)},
dht: dht,
config: config,
healthCheckInterval: 30 * time.Second,
partitionCheckInterval: 60 * time.Second,
connectivityTimeout: 10 * time.Second,
maxPartitionDuration: 10 * time.Minute,
connectivity: &ConnectivityMatrix{Matrix: make(map[string]map[string]*ConnectionInfo)},
stats: &NetworkStatistics{
LastUpdated: time.Now(),
},
@@ -255,33 +255,33 @@ func NewNetworkManagerImpl(dht *dht.DHT, config *config.Config) (*NetworkManager
func (nm *NetworkManagerImpl) initializeComponents() error {
// Initialize topology
nm.topology = &NetworkTopology{
TotalNodes: 0,
Connections: make(map[string][]string),
Regions: make(map[string][]string),
TotalNodes: 0,
Connections: make(map[string][]string),
Regions: make(map[string][]string),
AvailabilityZones: make(map[string][]string),
UpdatedAt: time.Now(),
UpdatedAt: time.Now(),
}
// Initialize partition info
nm.partitionInfo = &PartitionInfo{
PartitionDetected: false,
PartitionCount: 1,
IsolatedNodes: []string{},
PartitionDetected: false,
PartitionCount: 1,
IsolatedNodes: []string{},
ConnectivityMatrix: make(map[string]map[string]bool),
DetectedAt: time.Now(),
DetectedAt: time.Now(),
}
// Initialize health checker
nm.healthChecker = &NetworkHealthChecker{
nodeHealth: make(map[string]*NodeHealth),
healthHistory: make(map[string][]*HealthCheckResult),
healthHistory: make(map[string][]*NetworkHealthCheckResult),
alertThresholds: &NetworkAlertThresholds{
LatencyWarning: 500 * time.Millisecond,
LatencyCritical: 2 * time.Second,
PacketLossWarning: 0.05, // 5%
PacketLossCritical: 0.15, // 15%
HealthScoreWarning: 0.7,
HealthScoreCritical: 0.4,
LatencyWarning: 500 * time.Millisecond,
LatencyCritical: 2 * time.Second,
PacketLossWarning: 0.05, // 5%
PacketLossCritical: 0.15, // 15%
HealthScoreWarning: 0.7,
HealthScoreCritical: 0.4,
},
}
@@ -307,20 +307,20 @@ func (nm *NetworkManagerImpl) initializeComponents() error {
nm.recoveryManager = &RecoveryManager{
recoveryStrategies: map[RecoveryStrategy]*RecoveryStrategyConfig{
RecoveryStrategyAutomatic: {
Strategy: RecoveryStrategyAutomatic,
Timeout: 5 * time.Minute,
RetryAttempts: 3,
RetryInterval: 30 * time.Second,
Strategy: RecoveryStrategyAutomatic,
Timeout: 5 * time.Minute,
RetryAttempts: 3,
RetryInterval: 30 * time.Second,
RequireConsensus: false,
ForcedThreshold: 10 * time.Minute,
ForcedThreshold: 10 * time.Minute,
},
RecoveryStrategyGraceful: {
Strategy: RecoveryStrategyGraceful,
Timeout: 10 * time.Minute,
RetryAttempts: 5,
RetryInterval: 60 * time.Second,
Strategy: RecoveryStrategyGraceful,
Timeout: 10 * time.Minute,
RetryAttempts: 5,
RetryInterval: 60 * time.Second,
RequireConsensus: true,
ForcedThreshold: 20 * time.Minute,
ForcedThreshold: 20 * time.Minute,
},
},
activeRecoveries: make(map[string]*RecoveryOperation),
@@ -628,10 +628,10 @@ func (nm *NetworkManagerImpl) connectivityChecker(ctx context.Context) {
func (nm *NetworkManagerImpl) updateTopology() {
peers := nm.dht.GetConnectedPeers()
nm.topology.TotalNodes = len(peers) + 1 // +1 for current node
nm.topology.Connections = make(map[string][]string)
// Build connection map
currentNodeID := nm.config.Agent.ID
peerConnections := make([]string, len(peers))
@@ -639,21 +639,21 @@ func (nm *NetworkManagerImpl) updateTopology() {
peerConnections[i] = peer.String()
}
nm.topology.Connections[currentNodeID] = peerConnections
// Calculate network metrics
nm.topology.ClusterDiameter = nm.calculateClusterDiameter()
nm.topology.ClusteringCoefficient = nm.calculateClusteringCoefficient()
nm.topology.UpdatedAt = time.Now()
nm.lastTopologyUpdate = time.Now()
}
func (nm *NetworkManagerImpl) performHealthChecks(ctx context.Context) {
peers := nm.dht.GetConnectedPeers()
for _, peer := range peers {
result := nm.performHealthCheck(ctx, peer.String())
// Update node health
nodeHealth := &NodeHealth{
NodeID: peer.String(),
@@ -664,7 +664,7 @@ func (nm *NetworkManagerImpl) performHealthChecks(ctx context.Context) {
PacketLossRate: 0.0, // Would be measured in real implementation
ErrorRate: 0.0, // Would be calculated from history
}
if result.Success {
nodeHealth.Status = NodeStatusHealthy
nodeHealth.HealthScore = 1.0
@@ -672,21 +672,21 @@ func (nm *NetworkManagerImpl) performHealthChecks(ctx context.Context) {
nodeHealth.Status = NodeStatusUnreachable
nodeHealth.HealthScore = 0.0
}
nm.healthChecker.nodeHealth[peer.String()] = nodeHealth
// Store health check history
if _, exists := nm.healthChecker.healthHistory[peer.String()]; !exists {
nm.healthChecker.healthHistory[peer.String()] = []*HealthCheckResult{}
nm.healthChecker.healthHistory[peer.String()] = []*NetworkHealthCheckResult{}
}
nm.healthChecker.healthHistory[peer.String()] = append(
nm.healthChecker.healthHistory[peer.String()],
nm.healthChecker.healthHistory[peer.String()],
result,
)
// Keep only recent history (last 100 checks)
if len(nm.healthChecker.healthHistory[peer.String()]) > 100 {
nm.healthChecker.healthHistory[peer.String()] =
nm.healthChecker.healthHistory[peer.String()] =
nm.healthChecker.healthHistory[peer.String()][1:]
}
}
@@ -694,31 +694,31 @@ func (nm *NetworkManagerImpl) performHealthChecks(ctx context.Context) {
func (nm *NetworkManagerImpl) updateConnectivityMatrix(ctx context.Context) {
peers := nm.dht.GetConnectedPeers()
nm.connectivity.mu.Lock()
defer nm.connectivity.mu.Unlock()
// Initialize matrix if needed
if nm.connectivity.Matrix == nil {
nm.connectivity.Matrix = make(map[string]map[string]*ConnectionInfo)
}
currentNodeID := nm.config.Agent.ID
// Ensure current node exists in matrix
if nm.connectivity.Matrix[currentNodeID] == nil {
nm.connectivity.Matrix[currentNodeID] = make(map[string]*ConnectionInfo)
}
// Test connectivity to all peers
for _, peer := range peers {
peerID := peer.String()
// Test connection
connInfo := nm.testConnection(ctx, peerID)
nm.connectivity.Matrix[currentNodeID][peerID] = connInfo
}
nm.connectivity.LastUpdated = time.Now()
}
@@ -741,7 +741,7 @@ func (nm *NetworkManagerImpl) detectPartitionByConnectivity() (bool, []string, f
// Simplified connectivity-based detection
peers := nm.dht.GetConnectedPeers()
knownPeers := nm.dht.GetKnownPeers()
// If we know more peers than we're connected to, might be partitioned
if len(knownPeers) > len(peers)+2 { // Allow some tolerance
isolatedNodes := []string{}
@@ -759,7 +759,7 @@ func (nm *NetworkManagerImpl) detectPartitionByConnectivity() (bool, []string, f
}
return true, isolatedNodes, 0.8
}
return false, []string{}, 0.0
}
@@ -767,18 +767,18 @@ func (nm *NetworkManagerImpl) detectPartitionByHeartbeat() (bool, []string, floa
// Simplified heartbeat-based detection
nm.healthChecker.mu.RLock()
defer nm.healthChecker.mu.RUnlock()
isolatedNodes := []string{}
for nodeID, health := range nm.healthChecker.nodeHealth {
if health.Status == NodeStatusUnreachable {
isolatedNodes = append(isolatedNodes, nodeID)
}
}
if len(isolatedNodes) > 0 {
return true, isolatedNodes, 0.7
}
return false, []string{}, 0.0
}
@@ -791,7 +791,7 @@ func (nm *NetworkManagerImpl) detectPartitionHybrid() (bool, []string, float64)
// Combine multiple detection methods
partitioned1, nodes1, conf1 := nm.detectPartitionByConnectivity()
partitioned2, nodes2, conf2 := nm.detectPartitionByHeartbeat()
if partitioned1 && partitioned2 {
// Both methods agree
combinedNodes := nm.combineNodeLists(nodes1, nodes2)
@@ -805,7 +805,7 @@ func (nm *NetworkManagerImpl) detectPartitionHybrid() (bool, []string, float64)
return true, nodes2, conf2 * 0.7
}
}
return false, []string{}, 0.0
}
@@ -878,11 +878,11 @@ func (nm *NetworkManagerImpl) completeRecovery(ctx context.Context, operation *R
func (nm *NetworkManagerImpl) testPeerConnectivity(ctx context.Context, peerID string) *ConnectivityResult {
start := time.Now()
// In a real implementation, this would test actual network connectivity
// For now, we'll simulate based on DHT connectivity
peers := nm.dht.GetConnectedPeers()
for _, peer := range peers {
if peer.String() == peerID {
return &ConnectivityResult{
@@ -895,7 +895,7 @@ func (nm *NetworkManagerImpl) testPeerConnectivity(ctx context.Context, peerID s
}
}
}
return &ConnectivityResult{
PeerID: peerID,
Reachable: false,
@@ -907,13 +907,13 @@ func (nm *NetworkManagerImpl) testPeerConnectivity(ctx context.Context, peerID s
}
}
func (nm *NetworkManagerImpl) performHealthCheck(ctx context.Context, nodeID string) *HealthCheckResult {
func (nm *NetworkManagerImpl) performHealthCheck(ctx context.Context, nodeID string) *NetworkHealthCheckResult {
start := time.Now()
// In a real implementation, this would perform actual health checks
// For now, simulate based on connectivity
peers := nm.dht.GetConnectedPeers()
for _, peer := range peers {
if peer.String() == nodeID {
return &HealthCheckResult{
@@ -924,7 +924,7 @@ func (nm *NetworkManagerImpl) performHealthCheck(ctx context.Context, nodeID str
}
}
}
return &HealthCheckResult{
NodeID: nodeID,
Timestamp: time.Now(),
@@ -938,7 +938,7 @@ func (nm *NetworkManagerImpl) testConnection(ctx context.Context, peerID string)
// Test connection to specific peer
connected := false
latency := time.Duration(0)
// Check if peer is in connected peers list
peers := nm.dht.GetConnectedPeers()
for _, peer := range peers {
@@ -948,28 +948,28 @@ func (nm *NetworkManagerImpl) testConnection(ctx context.Context, peerID string)
break
}
}
return &ConnectionInfo{
Connected: connected,
Latency: latency,
PacketLoss: 0.0,
Bandwidth: 1000000, // 1 Mbps placeholder
LastChecked: time.Now(),
ErrorCount: 0,
Connected: connected,
Latency: latency,
PacketLoss: 0.0,
Bandwidth: 1000000, // 1 Mbps placeholder
LastChecked: time.Now(),
ErrorCount: 0,
}
}
func (nm *NetworkManagerImpl) updateNetworkStatistics() {
peers := nm.dht.GetConnectedPeers()
nm.stats.TotalNodes = len(peers) + 1
nm.stats.ConnectedNodes = len(peers)
nm.stats.DisconnectedNodes = nm.stats.TotalNodes - nm.stats.ConnectedNodes
// Calculate average latency from connectivity matrix
totalLatency := time.Duration(0)
connectionCount := 0
nm.connectivity.mu.RLock()
for _, connections := range nm.connectivity.Matrix {
for _, conn := range connections {
@@ -980,11 +980,11 @@ func (nm *NetworkManagerImpl) updateNetworkStatistics() {
}
}
nm.connectivity.mu.RUnlock()
if connectionCount > 0 {
nm.stats.AverageLatency = totalLatency / time.Duration(connectionCount)
}
nm.stats.OverallHealth = nm.calculateOverallNetworkHealth()
nm.stats.LastUpdated = time.Now()
}
@@ -1024,14 +1024,14 @@ func (nm *NetworkManagerImpl) calculateOverallNetworkHealth() float64 {
return float64(nm.stats.ConnectedNodes) / float64(nm.stats.TotalNodes)
}
func (nm *NetworkManagerImpl) determineNodeStatus(result *HealthCheckResult) NodeStatus {
func (nm *NetworkManagerImpl) determineNodeStatus(result *NetworkHealthCheckResult) NodeStatus {
if result.Success {
return NodeStatusHealthy
}
return NodeStatusUnreachable
}
func (nm *NetworkManagerImpl) calculateHealthScore(result *HealthCheckResult) float64 {
func (nm *NetworkManagerImpl) calculateHealthScore(result *NetworkHealthCheckResult) float64 {
if result.Success {
return 1.0
}
@@ -1040,19 +1040,19 @@ func (nm *NetworkManagerImpl) calculateHealthScore(result *HealthCheckResult) fl
func (nm *NetworkManagerImpl) combineNodeLists(list1, list2 []string) []string {
nodeSet := make(map[string]bool)
for _, node := range list1 {
nodeSet[node] = true
}
for _, node := range list2 {
nodeSet[node] = true
}
result := make([]string, 0, len(nodeSet))
for node := range nodeSet {
result = append(result, node)
}
sort.Strings(result)
return result
}
@@ -1073,4 +1073,4 @@ func (nm *NetworkManagerImpl) generateEventID() string {
func (nm *NetworkManagerImpl) generateOperationID() string {
return fmt.Sprintf("op-%d", time.Now().UnixNano())
}
}

68
pkg/slurp/distribution/replication.go
View File

@@ -7,39 +7,39 @@ import (
"sync"
"time"
"chorus/pkg/dht"
"chorus/pkg/config"
"chorus/pkg/dht"
"chorus/pkg/ucxl"
"github.com/libp2p/go-libp2p/core/peer"
)
// ReplicationManagerImpl implements ReplicationManager interface
type ReplicationManagerImpl struct {
mu sync.RWMutex
dht *dht.DHT
config *config.Config
replicationMap map[string]*ReplicationStatus
repairQueue chan *RepairRequest
rebalanceQueue chan *RebalanceRequest
consistentHash ConsistentHashing
policy *ReplicationPolicy
stats *ReplicationStatistics
running bool
mu sync.RWMutex
dht *dht.DHT
config *config.Config
replicationMap map[string]*ReplicationStatus
repairQueue chan *RepairRequest
rebalanceQueue chan *RebalanceRequest
consistentHash ConsistentHashing
policy *ReplicationPolicy
stats *ReplicationStatistics
running bool
}
// RepairRequest represents a repair request
type RepairRequest struct {
Address ucxl.Address
RequestedBy string
Priority Priority
RequestTime time.Time
Address ucxl.Address
RequestedBy string
Priority Priority
RequestTime time.Time
}
// RebalanceRequest represents a rebalance request
type RebalanceRequest struct {
Reason string
RequestedBy string
RequestTime time.Time
Reason string
RequestedBy string
RequestTime time.Time
}
// NewReplicationManagerImpl creates a new replication manager implementation
@@ -220,10 +220,10 @@ func (rm *ReplicationManagerImpl) BalanceReplicas(ctx context.Context) (*Rebalan
start := time.Now()
result := &RebalanceResult{
RebalanceTime: 0,
RebalanceTime: 0,
RebalanceSuccessful: false,
Errors: []string{},
RebalancedAt: time.Now(),
Errors: []string{},
RebalancedAt: time.Now(),
}
// Get current cluster topology
@@ -462,9 +462,9 @@ func (rm *ReplicationManagerImpl) discoverReplicas(ctx context.Context, address
// For now, we'll simulate some replicas
peers := rm.dht.GetConnectedPeers()
if len(peers) > 0 {
status.CurrentReplicas = min(len(peers), rm.policy.DefaultFactor)
status.CurrentReplicas = minInt(len(peers), rm.policy.DefaultFactor)
status.HealthyReplicas = status.CurrentReplicas
for i, peer := range peers {
if i >= status.CurrentReplicas {
break
@@ -478,9 +478,9 @@ func (rm *ReplicationManagerImpl) determineOverallHealth(status *ReplicationStat
if status.HealthyReplicas == 0 {
return HealthFailed
}
healthRatio := float64(status.HealthyReplicas) / float64(status.DesiredReplicas)
if healthRatio >= 1.0 {
return HealthHealthy
} else if healthRatio >= 0.7 {
@@ -579,7 +579,7 @@ func (rm *ReplicationManagerImpl) calculateIdealDistribution(peers []peer.ID) ma
func (rm *ReplicationManagerImpl) getCurrentDistribution(ctx context.Context) map[string]map[string]int {
// Returns current distribution: address -> node -> replica count
distribution := make(map[string]map[string]int)
rm.mu.RLock()
for addr, status := range rm.replicationMap {
distribution[addr] = make(map[string]int)
@@ -588,7 +588,7 @@ func (rm *ReplicationManagerImpl) getCurrentDistribution(ctx context.Context) ma
}
}
rm.mu.RUnlock()
return distribution
}
@@ -630,17 +630,17 @@ func (rm *ReplicationManagerImpl) isNodeOverloaded(nodeID string) bool {
// RebalanceMove represents a replica move operation
type RebalanceMove struct {
Address ucxl.Address `json:"address"`
FromNode string `json:"from_node"`
ToNode string `json:"to_node"`
Priority Priority `json:"priority"`
Reason string `json:"reason"`
Address ucxl.Address `json:"address"`
FromNode string `json:"from_node"`
ToNode string `json:"to_node"`
Priority Priority `json:"priority"`
Reason string `json:"reason"`
}
// Utility functions
func min(a, b int) int {
func minInt(a, b int) int {
if a < b {
return a
}
return b
}
}

340
pkg/slurp/distribution/security.go
View File

@@ -20,22 +20,22 @@ import (
// SecurityManager handles all security aspects of the distributed system
type SecurityManager struct {
mu sync.RWMutex
config *config.Config
tlsConfig *TLSConfig
authManager *AuthenticationManager
authzManager *AuthorizationManager
auditLogger *SecurityAuditLogger
nodeAuth *NodeAuthentication
encryption *DistributionEncryption
certificateAuth *CertificateAuthority
mu sync.RWMutex
config *config.Config
tlsConfig *TLSConfig
authManager *AuthenticationManager
authzManager *AuthorizationManager
auditLogger *SecurityAuditLogger
nodeAuth *NodeAuthentication
encryption *DistributionEncryption
certificateAuth *CertificateAuthority
// Security state
trustedNodes map[string]*TrustedNode
activeSessions map[string]*SecuritySession
securityPolicies map[string]*SecurityPolicy
threatDetector *ThreatDetector
trustedNodes map[string]*TrustedNode
activeSessions map[string]*SecuritySession
securityPolicies map[string]*SecurityPolicy
threatDetector *ThreatDetector
// Configuration
tlsEnabled bool
mutualTLSEnabled bool
@@ -45,28 +45,28 @@ type SecurityManager struct {
// TLSConfig manages TLS configuration for secure communications
type TLSConfig struct {
ServerConfig *tls.Config
ClientConfig *tls.Config
CertificatePath string
PrivateKeyPath string
CAPath string
MinTLSVersion uint16
CipherSuites []uint16
CurvePreferences []tls.CurveID
ClientAuth tls.ClientAuthType
VerifyConnection func(tls.ConnectionState) error
ServerConfig *tls.Config
ClientConfig *tls.Config
CertificatePath string
PrivateKeyPath string
CAPath string
MinTLSVersion uint16
CipherSuites []uint16
CurvePreferences []tls.CurveID
ClientAuth tls.ClientAuthType
VerifyConnection func(tls.ConnectionState) error
}
// AuthenticationManager handles node and user authentication
type AuthenticationManager struct {
mu sync.RWMutex
providers map[string]AuthProvider
tokenValidator TokenValidator
sessionManager *SessionManager
multiFactorAuth *MultiFactorAuth
credentialStore *CredentialStore
loginAttempts map[string]*LoginAttempts
authPolicies map[string]*AuthPolicy
mu sync.RWMutex
providers map[string]AuthProvider
tokenValidator TokenValidator
sessionManager *SessionManager
multiFactorAuth *MultiFactorAuth
credentialStore *CredentialStore
loginAttempts map[string]*LoginAttempts
authPolicies map[string]*AuthPolicy
}
// AuthProvider interface for different authentication methods
@@ -80,14 +80,14 @@ type AuthProvider interface {
// Credentials represents authentication credentials
type Credentials struct {
Type CredentialType `json:"type"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Token string `json:"token,omitempty"`
Certificate *x509.Certificate `json:"certificate,omitempty"`
Signature []byte `json:"signature,omitempty"`
Challenge string `json:"challenge,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
Type CredentialType `json:"type"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Token string `json:"token,omitempty"`
Certificate *x509.Certificate `json:"certificate,omitempty"`
Signature []byte `json:"signature,omitempty"`
Challenge string `json:"challenge,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
// CredentialType represents different types of credentials
@@ -104,15 +104,15 @@ const (
// AuthResult represents the result of authentication
type AuthResult struct {
Success bool `json:"success"`
UserID string `json:"user_id"`
Roles []string `json:"roles"`
Permissions []string `json:"permissions"`
TokenPair *TokenPair `json:"token_pair"`
SessionID string `json:"session_id"`
ExpiresAt time.Time `json:"expires_at"`
Metadata map[string]interface{} `json:"metadata"`
FailureReason string `json:"failure_reason,omitempty"`
Success bool `json:"success"`
UserID string `json:"user_id"`
Roles []string `json:"roles"`
Permissions []string `json:"permissions"`
TokenPair *TokenPair `json:"token_pair"`
SessionID string `json:"session_id"`
ExpiresAt time.Time `json:"expires_at"`
Metadata map[string]interface{} `json:"metadata"`
FailureReason string `json:"failure_reason,omitempty"`
}
// TokenPair represents access and refresh tokens
@@ -140,13 +140,13 @@ type TokenClaims struct {
// AuthorizationManager handles authorization and access control
type AuthorizationManager struct {
mu sync.RWMutex
policyEngine PolicyEngine
rbacManager *RBACManager
aclManager *ACLManager
resourceManager *ResourceManager
permissionCache *PermissionCache
authzPolicies map[string]*AuthorizationPolicy
mu sync.RWMutex
policyEngine PolicyEngine
rbacManager *RBACManager
aclManager *ACLManager
resourceManager *ResourceManager
permissionCache *PermissionCache
authzPolicies map[string]*AuthorizationPolicy
}
// PolicyEngine interface for policy evaluation
@@ -168,13 +168,13 @@ type AuthorizationRequest struct {
// AuthorizationResult represents the result of authorization
type AuthorizationResult struct {
Decision AuthorizationDecision `json:"decision"`
Reason string `json:"reason"`
Policies []string `json:"applied_policies"`
Conditions []string `json:"conditions"`
TTL time.Duration `json:"ttl"`
Metadata map[string]interface{} `json:"metadata"`
EvaluationTime time.Duration `json:"evaluation_time"`
Decision AuthorizationDecision `json:"decision"`
Reason string `json:"reason"`
Policies []string `json:"applied_policies"`
Conditions []string `json:"conditions"`
TTL time.Duration `json:"ttl"`
Metadata map[string]interface{} `json:"metadata"`
EvaluationTime time.Duration `json:"evaluation_time"`
}
// AuthorizationDecision represents authorization decisions
@@ -188,13 +188,13 @@ const (
// SecurityAuditLogger handles security event logging
type SecurityAuditLogger struct {
mu sync.RWMutex
loggers []SecurityLogger
eventBuffer []*SecurityEvent
alertManager *SecurityAlertManager
compliance *ComplianceManager
retention *AuditRetentionPolicy
enabled bool
mu sync.RWMutex
loggers []SecurityLogger
eventBuffer []*SecurityEvent
alertManager *SecurityAlertManager
compliance *ComplianceManager
retention *AuditRetentionPolicy
enabled bool
}
// SecurityLogger interface for security event logging
@@ -206,22 +206,22 @@ type SecurityLogger interface {
// SecurityEvent represents a security event
type SecurityEvent struct {
EventID string `json:"event_id"`
EventType SecurityEventType `json:"event_type"`
Severity SecuritySeverity `json:"severity"`
Timestamp time.Time `json:"timestamp"`
UserID string `json:"user_id,omitempty"`
NodeID string `json:"node_id,omitempty"`
Resource string `json:"resource,omitempty"`
Action string `json:"action,omitempty"`
Result string `json:"result"`
Message string `json:"message"`
Details map[string]interface{} `json:"details"`
IPAddress string `json:"ip_address,omitempty"`
UserAgent string `json:"user_agent,omitempty"`
SessionID string `json:"session_id,omitempty"`
RequestID string `json:"request_id,omitempty"`
Fingerprint string `json:"fingerprint"`
EventID string `json:"event_id"`
EventType SecurityEventType `json:"event_type"`
Severity SecuritySeverity `json:"severity"`
Timestamp time.Time `json:"timestamp"`
UserID string `json:"user_id,omitempty"`
NodeID string `json:"node_id,omitempty"`
Resource string `json:"resource,omitempty"`
Action string `json:"action,omitempty"`
Result string `json:"result"`
Message string `json:"message"`
Details map[string]interface{} `json:"details"`
IPAddress string `json:"ip_address,omitempty"`
UserAgent string `json:"user_agent,omitempty"`
SessionID string `json:"session_id,omitempty"`
RequestID string `json:"request_id,omitempty"`
Fingerprint string `json:"fingerprint"`
}
// SecurityEventType represents different types of security events
@@ -242,12 +242,12 @@ const (
type SecuritySeverity string
const (
SeverityDebug SecuritySeverity = "debug"
SeverityInfo SecuritySeverity = "info"
SeverityWarning SecuritySeverity = "warning"
SeverityError SecuritySeverity = "error"
SeverityCritical SecuritySeverity = "critical"
SeverityAlert SecuritySeverity = "alert"
SecuritySeverityDebug SecuritySeverity = "debug"
SecuritySeverityInfo SecuritySeverity = "info"
SecuritySeverityWarning SecuritySeverity = "warning"
SecuritySeverityError SecuritySeverity = "error"
SecuritySeverityCritical SecuritySeverity = "critical"
SecuritySeverityAlert SecuritySeverity = "alert"
)
// NodeAuthentication handles node-to-node authentication
@@ -262,16 +262,16 @@ type NodeAuthentication struct {
// TrustedNode represents a trusted node in the network
type TrustedNode struct {
NodeID string `json:"node_id"`
PublicKey []byte `json:"public_key"`
Certificate *x509.Certificate `json:"certificate"`
Roles []string `json:"roles"`
Capabilities []string `json:"capabilities"`
TrustLevel TrustLevel `json:"trust_level"`
LastSeen time.Time `json:"last_seen"`
VerifiedAt time.Time `json:"verified_at"`
Metadata map[string]interface{} `json:"metadata"`
Status NodeStatus `json:"status"`
NodeID string `json:"node_id"`
PublicKey []byte `json:"public_key"`
Certificate *x509.Certificate `json:"certificate"`
Roles []string `json:"roles"`
Capabilities []string `json:"capabilities"`
TrustLevel TrustLevel `json:"trust_level"`
LastSeen time.Time `json:"last_seen"`
VerifiedAt time.Time `json:"verified_at"`
Metadata map[string]interface{} `json:"metadata"`
Status NodeStatus `json:"status"`
}
// TrustLevel represents the trust level of a node
@@ -287,18 +287,18 @@ const (
// SecuritySession represents an active security session
type SecuritySession struct {
SessionID string `json:"session_id"`
UserID string `json:"user_id"`
NodeID string `json:"node_id"`
Roles []string `json:"roles"`
Permissions []string `json:"permissions"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
LastActivity time.Time `json:"last_activity"`
IPAddress string `json:"ip_address"`
UserAgent string `json:"user_agent"`
Metadata map[string]interface{} `json:"metadata"`
Status SessionStatus `json:"status"`
SessionID string `json:"session_id"`
UserID string `json:"user_id"`
NodeID string `json:"node_id"`
Roles []string `json:"roles"`
Permissions []string `json:"permissions"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
LastActivity time.Time `json:"last_activity"`
IPAddress string `json:"ip_address"`
UserAgent string `json:"user_agent"`
Metadata map[string]interface{} `json:"metadata"`
Status SessionStatus `json:"status"`
}
// SessionStatus represents session status
@@ -313,61 +313,61 @@ const (
// ThreatDetector detects security threats and anomalies
type ThreatDetector struct {
mu sync.RWMutex
detectionRules []*ThreatDetectionRule
behaviorAnalyzer *BehaviorAnalyzer
anomalyDetector *AnomalyDetector
threatIntelligence *ThreatIntelligence
activeThreats map[string]*ThreatEvent
mu sync.RWMutex
detectionRules []*ThreatDetectionRule
behaviorAnalyzer *BehaviorAnalyzer
anomalyDetector *AnomalyDetector
threatIntelligence *ThreatIntelligence
activeThreats map[string]*ThreatEvent
mitigationStrategies map[ThreatType]*MitigationStrategy
}
// ThreatDetectionRule represents a threat detection rule
type ThreatDetectionRule struct {
RuleID string `json:"rule_id"`
Name string `json:"name"`
Description string `json:"description"`
ThreatType ThreatType `json:"threat_type"`
Severity SecuritySeverity `json:"severity"`
Conditions []*ThreatCondition `json:"conditions"`
Actions []*ThreatAction `json:"actions"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Metadata map[string]interface{} `json:"metadata"`
RuleID string `json:"rule_id"`
Name string `json:"name"`
Description string `json:"description"`
ThreatType ThreatType `json:"threat_type"`
Severity SecuritySeverity `json:"severity"`
Conditions []*ThreatCondition `json:"conditions"`
Actions []*ThreatAction `json:"actions"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Metadata map[string]interface{} `json:"metadata"`
}
// ThreatType represents different types of threats
type ThreatType string
const (
ThreatTypeBruteForce ThreatType = "brute_force"
ThreatTypeUnauthorized ThreatType = "unauthorized_access"
ThreatTypeDataExfiltration ThreatType = "data_exfiltration"
ThreatTypeDoS ThreatType = "denial_of_service"
ThreatTypeBruteForce ThreatType = "brute_force"
ThreatTypeUnauthorized ThreatType = "unauthorized_access"
ThreatTypeDataExfiltration ThreatType = "data_exfiltration"
ThreatTypeDoS ThreatType = "denial_of_service"
ThreatTypePrivilegeEscalation ThreatType = "privilege_escalation"
ThreatTypeAnomalous ThreatType = "anomalous_behavior"
ThreatTypeMaliciousCode ThreatType = "malicious_code"
ThreatTypeInsiderThreat ThreatType = "insider_threat"
ThreatTypeAnomalous ThreatType = "anomalous_behavior"
ThreatTypeMaliciousCode ThreatType = "malicious_code"
ThreatTypeInsiderThreat ThreatType = "insider_threat"
)
// CertificateAuthority manages certificate generation and validation
type CertificateAuthority struct {
mu sync.RWMutex
rootCA *x509.Certificate
rootKey interface{}
intermediateCA *x509.Certificate
mu sync.RWMutex
rootCA *x509.Certificate
rootKey interface{}
intermediateCA *x509.Certificate
intermediateKey interface{}
certStore *CertificateStore
crlManager *CRLManager
ocspResponder *OCSPResponder
certStore *CertificateStore
crlManager *CRLManager
ocspResponder *OCSPResponder
}
// DistributionEncryption handles encryption for distributed communications
type DistributionEncryption struct {
mu sync.RWMutex
keyManager *DistributionKeyManager
encryptionSuite *EncryptionSuite
mu sync.RWMutex
keyManager *DistributionKeyManager
encryptionSuite *EncryptionSuite
keyRotationPolicy *KeyRotationPolicy
encryptionMetrics *EncryptionMetrics
}
@@ -379,13 +379,13 @@ func NewSecurityManager(config *config.Config) (*SecurityManager, error) {
}
sm := &SecurityManager{
config: config,
trustedNodes: make(map[string]*TrustedNode),
activeSessions: make(map[string]*SecuritySession),
securityPolicies: make(map[string]*SecurityPolicy),
tlsEnabled: true,
mutualTLSEnabled: true,
auditingEnabled: true,
config: config,
trustedNodes: make(map[string]*TrustedNode),
activeSessions: make(map[string]*SecuritySession),
securityPolicies: make(map[string]*SecurityPolicy),
tlsEnabled: true,
mutualTLSEnabled: true,
auditingEnabled: true,
encryptionEnabled: true,
}
@@ -508,12 +508,12 @@ func (sm *SecurityManager) Authenticate(ctx context.Context, credentials *Creden
// Log authentication attempt
sm.logSecurityEvent(ctx, &SecurityEvent{
EventType: EventTypeAuthentication,
Severity: SeverityInfo,
Severity: SecuritySeverityInfo,
Action: "authenticate",
Message: "Authentication attempt",
Details: map[string]interface{}{
"credential_type": credentials.Type,
"username": credentials.Username,
"username": credentials.Username,
},
})
@@ -525,7 +525,7 @@ func (sm *SecurityManager) Authorize(ctx context.Context, request *Authorization
// Log authorization attempt
sm.logSecurityEvent(ctx, &SecurityEvent{
EventType: EventTypeAuthorization,
Severity: SeverityInfo,
Severity: SecuritySeverityInfo,
UserID: request.UserID,
Resource: request.Resource,
Action: request.Action,
@@ -554,7 +554,7 @@ func (sm *SecurityManager) ValidateNodeIdentity(ctx context.Context, nodeID stri
// Log successful validation
sm.logSecurityEvent(ctx, &SecurityEvent{
EventType: EventTypeAuthentication,
Severity: SeverityInfo,
Severity: SecuritySeverityInfo,
NodeID: nodeID,
Action: "validate_node_identity",
Result: "success",
@@ -609,7 +609,7 @@ func (sm *SecurityManager) AddTrustedNode(ctx context.Context, node *TrustedNode
// Log node addition
sm.logSecurityEvent(ctx, &SecurityEvent{
EventType: EventTypeConfiguration,
Severity: SeverityInfo,
Severity: SecuritySeverityInfo,
NodeID: node.NodeID,
Action: "add_trusted_node",
Result: "success",
@@ -649,7 +649,7 @@ func (sm *SecurityManager) loadOrGenerateCertificate() (*tls.Certificate, error)
func (sm *SecurityManager) generateSelfSignedCertificate() ([]byte, []byte, error) {
// Generate a self-signed certificate for development/testing
// In production, use proper CA-signed certificates
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
@@ -660,11 +660,11 @@ func (sm *SecurityManager) generateSelfSignedCertificate() ([]byte, []byte, erro
StreetAddress: []string{""},
PostalCode: []string{""},
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(365 * 24 * time.Hour),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
NotBefore: time.Now(),
NotAfter: time.Now().Add(365 * 24 * time.Hour),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
}
// This is a simplified implementation
@@ -765,8 +765,8 @@ func NewDistributionEncryption(config *config.Config) (*DistributionEncryption,
func NewThreatDetector(config *config.Config) (*ThreatDetector, error) {
return &ThreatDetector{
detectionRules: []*ThreatDetectionRule{},
activeThreats: make(map[string]*ThreatEvent),
detectionRules: []*ThreatDetectionRule{},
activeThreats: make(map[string]*ThreatEvent),
mitigationStrategies: make(map[ThreatType]*MitigationStrategy),
}, nil
}
@@ -831,4 +831,4 @@ type OCSPResponder struct{}
type DistributionKeyManager struct{}
type EncryptionSuite struct{}
type KeyRotationPolicy struct{}
type EncryptionMetrics struct{}
type EncryptionMetrics struct{}