bzzz/test/integration/load_performance_test.go

// Load Testing and Performance Benchmarks for BZZZ System
// This comprehensive test suite validates system performance under various load conditions
// and provides detailed benchmarks for critical components and workflows.

package integration

import (
	"context"
	"encoding/json"
	"fmt"
	"math/rand"
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"chorus.services/bzzz/pkg/config"
	"chorus.services/bzzz/pkg/dht"
	"chorus.services/bzzz/pkg/slurp"
	"chorus.services/bzzz/pkg/ucxi"
	"chorus.services/bzzz/pkg/ucxl"
)

// LoadTestSuite provides comprehensive load testing capabilities
type LoadTestSuite struct {
	ctx              context.Context
	config           *config.Config
	dhtStorage       dht.DHT
	ucxiServer       *ucxi.Server
	slurpProcessor   *slurp.EventProcessor
	performanceStats *PerformanceStats
	loadProfiles     map[string]*LoadProfile
	testData         *TestDataManager
	resourceMonitor  *ResourceMonitor
}

// PerformanceStats tracks detailed performance metrics
type PerformanceStats struct {
	mu                       sync.RWMutex
	TotalOperations          int64         `json:"total_operations"`
	SuccessfulOperations     int64         `json:"successful_operations"`
	FailedOperations         int64         `json:"failed_operations"`
	AverageLatency           time.Duration `json:"average_latency"`
	P50Latency               time.Duration `json:"p50_latency"`
	P95Latency               time.Duration `json:"p95_latency"`
	P99Latency               time.Duration `json:"p99_latency"`
	MaxLatency               time.Duration `json:"max_latency"`
	MinLatency               time.Duration `json:"min_latency"`
	Throughput               float64       `json:"throughput_ops_per_sec"`
	ErrorRate                float64       `json:"error_rate_percent"`
	LatencyHistogram         []int64       `json:"latency_histogram_ms"`
	latencies                []time.Duration
	startTime                time.Time
	endTime                  time.Time
	MemoryUsageStart         int64         `json:"memory_usage_start_bytes"`
	MemoryUsageEnd           int64         `json:"memory_usage_end_bytes"`
	MemoryUsagePeak          int64         `json:"memory_usage_peak_bytes"`
	GoroutineCountStart      int           `json:"goroutine_count_start"`
	GoroutineCountEnd        int           `json:"goroutine_count_end"`
	GoroutineCountPeak       int           `json:"goroutine_count_peak"`
}

// LoadProfile defines different load testing scenarios
type LoadProfile struct {
	Name                    string        `json:"name"`
	Description             string        `json:"description"`
	Duration                time.Duration `json:"duration"`
	ConcurrentWorkers       int           `json:"concurrent_workers"`
	RequestsPerSecond       float64       `json:"requests_per_second"`
	PayloadSizeBytes        int           `json:"payload_size_bytes"`
	OperationDistribution   map[string]float64 `json:"operation_distribution"` // PUT:70%, GET:20%, DELETE:10%
	AddressPatternComplexity string        `json:"address_pattern_complexity"` // simple, medium, complex
	EnableLatencyJitter     bool          `json:"enable_latency_jitter"`
	FailureInjectionRate    float64       `json:"failure_injection_rate"`
}

// TestDataManager handles test data generation and management
type TestDataManager struct {
	mu            sync.RWMutex
	generatedData map[string][]byte
	addresses     []string
	payloadSizes  []int
	patterns      []string
}

// ResourceMonitor tracks system resource usage during tests
type ResourceMonitor struct {
	mu                sync.RWMutex
	monitoring        bool
	interval          time.Duration
	memoryUsage       []int64
	goroutineCount    []int
	cpuUsage          []float64
	diskIOOperations  []int64
	networkBytesIn    []int64
	networkBytesOut   []int64
	timestamps        []time.Time
}

func TestBZZZLoadAndPerformance(t *testing.T) {
	suite := NewLoadTestSuite(t)
	defer suite.Cleanup()

	// Define test cases based on realistic usage patterns
	t.Run("Baseline_Single_User", suite.TestBaselineSingleUser)
	t.Run("Light_Load_10_Users", suite.TestLightLoad10Users)
	t.Run("Medium_Load_100_Users", suite.TestMediumLoad100Users)
	t.Run("Heavy_Load_1000_Users", suite.TestHeavyLoad1000Users)
	t.Run("Spike_Load_Sudden_Increase", suite.TestSpikeLoadSuddenIncrease)
	t.Run("Sustained_Load_Long_Duration", suite.TestSustainedLoadLongDuration)
	t.Run("Mixed_Operations_Realistic", suite.TestMixedOperationsRealistic)
	t.Run("Large_Payload_Stress", suite.TestLargePayloadStress)
	t.Run("High_Concurrency_Stress", suite.TestHighConcurrencyStress)
	t.Run("Memory_Pressure_Test", suite.TestMemoryPressureTest)
}

func NewLoadTestSuite(t *testing.T) *LoadTestSuite {
	ctx := context.Background()

	// Initialize configuration optimized for testing
	cfg := &config.Config{
		DHT: config.DHTConfig{
			ReplicationFactor: 3,
			PutTimeout:        5 * time.Second,
			GetTimeout:        2 * time.Second,
			MaxKeySize:        1024 * 1024, // 1MB max
		},
		SLURP: config.SLURPConfig{
			BatchSize:         50,
			ProcessingTimeout: 10 * time.Second,
			BackpressureEnabled: true,
			CircuitBreakerEnabled: true,
		},
	}

	// Initialize DHT storage
	dhtStorage := dht.NewMockDHT()
	
	// Configure mock DHT for performance testing
	mockDHT := dhtStorage.(*dht.MockDHT)
	mockDHT.SetLatency(1 * time.Millisecond) // Realistic network latency
	
	// Initialize UCXI server
	ucxiServer, err := ucxi.NewServer(dhtStorage)
	require.NoError(t, err, "Failed to create UCXI server")

	// Initialize SLURP processor
	slurpProcessor, err := slurp.NewEventProcessor(cfg, dhtStorage)
	require.NoError(t, err, "Failed to create SLURP processor")

	// Initialize performance tracking
	performanceStats := &PerformanceStats{
		latencies: make([]time.Duration, 0, 10000),
		LatencyHistogram: make([]int64, 100), // 0-99ms buckets
	}

	// Initialize load profiles
	loadProfiles := map[string]*LoadProfile{
		"baseline": {
			Name:                    "Baseline Single User",
			Description:             "Single user performing mixed operations",
			Duration:                30 * time.Second,
			ConcurrentWorkers:       1,
			RequestsPerSecond:       5,
			PayloadSizeBytes:        1024,
			OperationDistribution:   map[string]float64{"PUT": 0.5, "GET": 0.4, "DELETE": 0.1},
			AddressPatternComplexity: "simple",
			EnableLatencyJitter:     false,
			FailureInjectionRate:    0.0,
		},
		"light": {
			Name:                    "Light Load 10 Users",
			Description:             "10 concurrent users with normal usage patterns",
			Duration:                2 * time.Minute,
			ConcurrentWorkers:       10,
			RequestsPerSecond:       50,
			PayloadSizeBytes:        2048,
			OperationDistribution:   map[string]float64{"PUT": 0.4, "GET": 0.5, "DELETE": 0.1},
			AddressPatternComplexity: "medium",
			EnableLatencyJitter:     true,
			FailureInjectionRate:    0.01, // 1% failure rate
		},
		"medium": {
			Name:                    "Medium Load 100 Users", 
			Description:             "100 concurrent users with mixed workload",
			Duration:                5 * time.Minute,
			ConcurrentWorkers:       100,
			RequestsPerSecond:       500,
			PayloadSizeBytes:        4096,
			OperationDistribution:   map[string]float64{"PUT": 0.3, "GET": 0.6, "DELETE": 0.1},
			AddressPatternComplexity: "complex",
			EnableLatencyJitter:     true,
			FailureInjectionRate:    0.02, // 2% failure rate
		},
		"heavy": {
			Name:                    "Heavy Load 1000 Users",
			Description:             "1000 concurrent users with high throughput",
			Duration:                10 * time.Minute,
			ConcurrentWorkers:       1000,
			RequestsPerSecond:       5000,
			PayloadSizeBytes:        8192,
			OperationDistribution:   map[string]float64{"PUT": 0.25, "GET": 0.65, "DELETE": 0.1},
			AddressPatternComplexity: "complex",
			EnableLatencyJitter:     true,
			FailureInjectionRate:    0.03, // 3% failure rate
		},
		"spike": {
			Name:                    "Spike Load Test",
			Description:             "Sudden traffic spike simulation",
			Duration:                3 * time.Minute,
			ConcurrentWorkers:       2000,
			RequestsPerSecond:       10000,
			PayloadSizeBytes:        1024,
			OperationDistribution:   map[string]float64{"PUT": 0.2, "GET": 0.75, "DELETE": 0.05},
			AddressPatternComplexity: "medium",
			EnableLatencyJitter:     true,
			FailureInjectionRate:    0.05, // 5% failure rate during spike
		},
	}

	// Initialize test data manager
	testData := &TestDataManager{
		generatedData: make(map[string][]byte),
		payloadSizes:  []int{256, 512, 1024, 2048, 4096, 8192, 16384, 32768},
		patterns: []string{
			"ucxl://agent-{id}:developer@project-{id}:task-{id}/*^",
			"ucxl://user-{id}:viewer@docs:read-{id}/*^",
			"ucxl://service-{id}:admin@system:config-{id}/*^",
			"ucxl://monitor-{id}:developer@metrics:collect-{id}/*^",
		},
	}

	// Initialize resource monitor
	resourceMonitor := &ResourceMonitor{
		interval:         time.Second,
		memoryUsage:      make([]int64, 0, 1000),
		goroutineCount:   make([]int, 0, 1000),
		cpuUsage:         make([]float64, 0, 1000),
		timestamps:       make([]time.Time, 0, 1000),
	}

	return &LoadTestSuite{
		ctx:              ctx,
		config:           cfg,
		dhtStorage:       dhtStorage,
		ucxiServer:       ucxiServer,
		slurpProcessor:   slurpProcessor,
		performanceStats: performanceStats,
		loadProfiles:     loadProfiles,
		testData:         testData,
		resourceMonitor:  resourceMonitor,
	}
}

func (suite *LoadTestSuite) Cleanup() {
	suite.resourceMonitor.Stop()
}

// TestBaselineSingleUser establishes baseline performance metrics
func (suite *LoadTestSuite) TestBaselineSingleUser(t *testing.T) {
	profile := suite.loadProfiles["baseline"]
	result := suite.runLoadTestWithProfile(t, profile)
	
	// Baseline assertions - these establish expected performance
	assert.Less(t, result.AverageLatency, 10*time.Millisecond, "Baseline average latency should be under 10ms")
	assert.Less(t, result.P95Latency, 50*time.Millisecond, "Baseline P95 latency should be under 50ms")
	assert.Greater(t, result.Throughput, 4.0, "Baseline throughput should be at least 4 ops/sec")
	assert.Less(t, result.ErrorRate, 0.1, "Baseline error rate should be under 0.1%")
	
	t.Logf("Baseline Performance: Avg=%v, P95=%v, Throughput=%.2f ops/sec, Errors=%.2f%%",
		result.AverageLatency, result.P95Latency, result.Throughput, result.ErrorRate)
}

// TestLightLoad10Users tests system behavior under light concurrent load
func (suite *LoadTestSuite) TestLightLoad10Users(t *testing.T) {
	profile := suite.loadProfiles["light"]
	result := suite.runLoadTestWithProfile(t, profile)
	
	// Light load assertions
	assert.Less(t, result.AverageLatency, 50*time.Millisecond, "Light load average latency should be reasonable")
	assert.Less(t, result.P95Latency, 200*time.Millisecond, "Light load P95 latency should be acceptable")
	assert.Greater(t, result.Throughput, 40.0, "Light load throughput should meet minimum requirements")
	assert.Less(t, result.ErrorRate, 2.0, "Light load error rate should be manageable")
	
	// Memory usage should be reasonable
	memoryIncrease := result.MemoryUsageEnd - result.MemoryUsageStart
	assert.Less(t, memoryIncrease, int64(100*1024*1024), "Memory usage increase should be under 100MB")
	
	t.Logf("Light Load Performance: Avg=%v, P95=%v, Throughput=%.2f ops/sec, Errors=%.2f%%",
		result.AverageLatency, result.P95Latency, result.Throughput, result.ErrorRate)
}

// TestMediumLoad100Users tests system behavior under medium concurrent load
func (suite *LoadTestSuite) TestMediumLoad100Users(t *testing.T) {
	profile := suite.loadProfiles["medium"]
	result := suite.runLoadTestWithProfile(t, profile)
	
	// Medium load assertions - more relaxed than light load
	assert.Less(t, result.AverageLatency, 100*time.Millisecond, "Medium load average latency should be acceptable")
	assert.Less(t, result.P95Latency, 500*time.Millisecond, "Medium load P95 latency should be manageable")
	assert.Greater(t, result.Throughput, 300.0, "Medium load throughput should meet requirements")
	assert.Less(t, result.ErrorRate, 5.0, "Medium load error rate should be acceptable")
	
	// Resource usage checks
	assert.Less(t, result.GoroutineCountPeak, 200, "Goroutine count should not exceed reasonable limits")
	
	t.Logf("Medium Load Performance: Avg=%v, P95=%v, Throughput=%.2f ops/sec, Errors=%.2f%%",
		result.AverageLatency, result.P95Latency, result.Throughput, result.ErrorRate)
}

// TestHeavyLoad1000Users tests system behavior under heavy concurrent load
func (suite *LoadTestSuite) TestHeavyLoad1000Users(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping heavy load test in short mode")
	}

	profile := suite.loadProfiles["heavy"]
	result := suite.runLoadTestWithProfile(t, profile)
	
	// Heavy load assertions - system should remain stable but performance may degrade
	assert.Less(t, result.AverageLatency, 500*time.Millisecond, "Heavy load average latency should remain reasonable")
	assert.Less(t, result.P95Latency, 2*time.Second, "Heavy load P95 latency should not exceed 2 seconds")
	assert.Greater(t, result.Throughput, 1000.0, "Heavy load throughput should meet minimum requirements")
	assert.Less(t, result.ErrorRate, 10.0, "Heavy load error rate should remain below 10%")
	
	// System should not crash or become unresponsive
	assert.Greater(t, result.SuccessfulOperations, result.FailedOperations, "More operations should succeed than fail")
	
	t.Logf("Heavy Load Performance: Avg=%v, P95=%v, Throughput=%.2f ops/sec, Errors=%.2f%%",
		result.AverageLatency, result.P95Latency, result.Throughput, result.ErrorRate)
}

// TestSpikeLoadSuddenIncrease tests system resilience to sudden traffic spikes
func (suite *LoadTestSuite) TestSpikeLoadSuddenIncrease(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping spike load test in short mode")
	}

	profile := suite.loadProfiles["spike"]
	result := suite.runLoadTestWithProfile(t, profile)
	
	// Spike load assertions - system should handle spikes gracefully
	// May have higher latency and error rates but should not crash
	assert.Less(t, result.ErrorRate, 20.0, "Spike load error rate should not exceed 20%")
	assert.Greater(t, result.Throughput, 500.0, "Spike load should maintain minimum throughput")
	
	// System should recover and remain responsive
	assert.Less(t, result.P99Latency, 5*time.Second, "P99 latency should not exceed 5 seconds even during spikes")
	
	t.Logf("Spike Load Performance: Avg=%v, P95=%v, P99=%v, Throughput=%.2f ops/sec, Errors=%.2f%%",
		result.AverageLatency, result.P95Latency, result.P99Latency, result.Throughput, result.ErrorRate)
}

// TestSustainedLoadLongDuration tests system stability over extended periods
func (suite *LoadTestSuite) TestSustainedLoadLongDuration(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping sustained load test in short mode")
	}

	// Create extended duration profile
	sustainedProfile := &LoadProfile{
		Name:                    "Sustained Load Test",
		Description:             "Extended duration load test for stability",
		Duration:                20 * time.Minute,
		ConcurrentWorkers:       200,
		RequestsPerSecond:       1000,
		PayloadSizeBytes:        2048,
		OperationDistribution:   map[string]float64{"PUT": 0.3, "GET": 0.6, "DELETE": 0.1},
		AddressPatternComplexity: "medium",
		EnableLatencyJitter:     true,
		FailureInjectionRate:    0.02,
	}

	result := suite.runLoadTestWithProfile(t, sustainedProfile)
	
	// Sustained load assertions - focus on stability over time
	assert.Less(t, result.ErrorRate, 5.0, "Sustained load error rate should remain stable")
	
	// Memory usage should not continuously grow (no memory leaks)
	memoryGrowth := result.MemoryUsageEnd - result.MemoryUsageStart
	assert.Less(t, memoryGrowth, int64(500*1024*1024), "Memory growth should be bounded (no major leaks)")
	
	// Performance should not significantly degrade over time
	assert.Greater(t, result.Throughput, 800.0, "Sustained load should maintain reasonable throughput")
	
	t.Logf("Sustained Load Performance: Duration=%v, Throughput=%.2f ops/sec, Errors=%.2f%%, MemGrowth=%dMB",
		sustainedProfile.Duration, result.Throughput, result.ErrorRate, memoryGrowth/(1024*1024))
}

// TestMixedOperationsRealistic tests realistic mixed workload patterns
func (suite *LoadTestSuite) TestMixedOperationsRealistic(t *testing.T) {
	// Create realistic mixed operations profile
	realisticProfile := &LoadProfile{
		Name:                    "Realistic Mixed Operations",
		Description:             "Realistic distribution of operations with varying payloads",
		Duration:                5 * time.Minute,
		ConcurrentWorkers:       50,
		RequestsPerSecond:       200,
		PayloadSizeBytes:        4096, // Will be varied in implementation
		OperationDistribution:   map[string]float64{"PUT": 0.2, "GET": 0.7, "DELETE": 0.1},
		AddressPatternComplexity: "complex",
		EnableLatencyJitter:     true,
		FailureInjectionRate:    0.015, // 1.5% realistic failure rate
	}

	result := suite.runMixedOperationsTest(t, realisticProfile)
	
	// Realistic workload assertions
	assert.Less(t, result.AverageLatency, 100*time.Millisecond, "Mixed operations average latency should be reasonable")
	assert.Less(t, result.P95Latency, 500*time.Millisecond, "Mixed operations P95 latency should be acceptable")
	assert.Greater(t, result.Throughput, 150.0, "Mixed operations throughput should meet requirements")
	assert.Less(t, result.ErrorRate, 3.0, "Mixed operations error rate should be low")
	
	t.Logf("Mixed Operations Performance: Avg=%v, P95=%v, Throughput=%.2f ops/sec, Errors=%.2f%%",
		result.AverageLatency, result.P95Latency, result.Throughput, result.ErrorRate)
}

// TestLargePayloadStress tests system behavior with large payloads
func (suite *LoadTestSuite) TestLargePayloadStress(t *testing.T) {
	// Test with increasingly large payloads
	payloadSizes := []int{
		64 * 1024,    // 64KB
		256 * 1024,   // 256KB
		1024 * 1024,  // 1MB
		4 * 1024 * 1024, // 4MB
	}

	for _, payloadSize := range payloadSizes {
		t.Run(fmt.Sprintf("Payload_%dKB", payloadSize/1024), func(t *testing.T) {
			largePayloadProfile := &LoadProfile{
				Name:                    fmt.Sprintf("Large Payload %dKB", payloadSize/1024),
				Description:             "Large payload stress test",
				Duration:                2 * time.Minute,
				ConcurrentWorkers:       20,
				RequestsPerSecond:       50,
				PayloadSizeBytes:        payloadSize,
				OperationDistribution:   map[string]float64{"PUT": 0.5, "GET": 0.5, "DELETE": 0.0},
				AddressPatternComplexity: "simple",
				EnableLatencyJitter:     false,
				FailureInjectionRate:    0.0,
			}

			result := suite.runLoadTestWithProfile(t, largePayloadProfile)
			
			// Large payload assertions - latency will be higher but should not fail
			maxExpectedLatency := time.Duration(payloadSize/1024) * time.Millisecond // 1ms per KB
			if maxExpectedLatency < 100*time.Millisecond {
				maxExpectedLatency = 100 * time.Millisecond
			}
			
			assert.Less(t, result.AverageLatency, maxExpectedLatency, 
				"Large payload average latency should scale reasonably with size")
			assert.Less(t, result.ErrorRate, 2.0, "Large payload error rate should be low")
			assert.Greater(t, result.SuccessfulOperations, int64(0), "Some operations should succeed")
			
			t.Logf("Large Payload %dKB: Avg=%v, P95=%v, Throughput=%.2f ops/sec", 
				payloadSize/1024, result.AverageLatency, result.P95Latency, result.Throughput)
		})
	}
}

// TestHighConcurrencyStress tests system behavior under very high concurrency
func (suite *LoadTestSuite) TestHighConcurrencyStress(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping high concurrency test in short mode")
	}

	concurrencyProfile := &LoadProfile{
		Name:                    "High Concurrency Stress",
		Description:             "Very high concurrency with many goroutines",
		Duration:                3 * time.Minute,
		ConcurrentWorkers:       5000, // Very high concurrency
		RequestsPerSecond:       10000,
		PayloadSizeBytes:        512,
		OperationDistribution:   map[string]float64{"PUT": 0.1, "GET": 0.85, "DELETE": 0.05},
		AddressPatternComplexity: "simple",
		EnableLatencyJitter:     true,
		FailureInjectionRate:    0.0,
	}

	result := suite.runLoadTestWithProfile(t, concurrencyProfile)
	
	// High concurrency assertions - system should not deadlock or crash
	assert.Less(t, result.ErrorRate, 15.0, "High concurrency error rate should be manageable")
	assert.Greater(t, result.Throughput, 2000.0, "High concurrency should achieve reasonable throughput")
	assert.Greater(t, result.SuccessfulOperations, result.FailedOperations, 
		"More operations should succeed than fail even under high concurrency")
	
	t.Logf("High Concurrency Performance: Workers=%d, Avg=%v, Throughput=%.2f ops/sec, Errors=%.2f%%",
		concurrencyProfile.ConcurrentWorkers, result.AverageLatency, result.Throughput, result.ErrorRate)
}

// TestMemoryPressureTest tests system behavior under memory pressure
func (suite *LoadTestSuite) TestMemoryPressureTest(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping memory pressure test in short mode")
	}

	// Force GC before test to get clean baseline
	runtime.GC()
	runtime.GC()

	memoryProfile := &LoadProfile{
		Name:                    "Memory Pressure Test",
		Description:             "Test under memory pressure with large objects",
		Duration:                5 * time.Minute,
		ConcurrentWorkers:       100,
		RequestsPerSecond:       500,
		PayloadSizeBytes:        100 * 1024, // 100KB payloads
		OperationDistribution:   map[string]float64{"PUT": 0.8, "GET": 0.2, "DELETE": 0.0}, // Mostly writes to increase memory
		AddressPatternComplexity: "complex",
		EnableLatencyJitter:     true,
		FailureInjectionRate:    0.0,
	}

	result := suite.runLoadTestWithProfile(t, memoryProfile)
	
	// Memory pressure assertions
	assert.Less(t, result.ErrorRate, 10.0, "Memory pressure should not cause excessive errors")
	assert.Greater(t, result.Throughput, 200.0, "Memory pressure should maintain minimum throughput")
	
	// Check for reasonable memory growth
	memoryGrowth := result.MemoryUsageEnd - result.MemoryUsageStart
	assert.Less(t, memoryGrowth, int64(2*1024*1024*1024), "Memory growth should be bounded under 2GB")
	
	t.Logf("Memory Pressure: MemGrowth=%dMB, Peak=%dMB, Throughput=%.2f ops/sec",
		memoryGrowth/(1024*1024), result.MemoryUsagePeak/(1024*1024), result.Throughput)
}

// Core load testing implementation

func (suite *LoadTestSuite) runLoadTestWithProfile(t *testing.T, profile *LoadProfile) *PerformanceStats {
	t.Logf("Starting load test: %s", profile.Name)
	t.Logf("Profile: Workers=%d, RPS=%.1f, Duration=%v, PayloadSize=%d bytes",
		profile.ConcurrentWorkers, profile.RequestsPerSecond, profile.Duration, profile.PayloadSizeBytes)

	// Reset performance stats
	suite.performanceStats = &PerformanceStats{
		latencies:        make([]time.Duration, 0, int(profile.Duration.Seconds()*profile.RequestsPerSecond)),
		LatencyHistogram: make([]int64, 100),
		startTime:        time.Now(),
		MinLatency:       time.Hour, // Initialize to very high value
	}

	// Start resource monitoring
	suite.resourceMonitor.Start()
	defer suite.resourceMonitor.Stop()

	// Record initial resource usage
	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)
	suite.performanceStats.MemoryUsageStart = int64(memStats.Alloc)
	suite.performanceStats.GoroutineCountStart = runtime.NumGoroutine()

	// Create worker channels
	workChan := make(chan WorkItem, profile.ConcurrentWorkers*10)
	resultChan := make(chan WorkResult, profile.ConcurrentWorkers*10)

	// Start workers
	var workerWg sync.WaitGroup
	for i := 0; i < profile.ConcurrentWorkers; i++ {
		workerWg.Add(1)
		go suite.loadTestWorker(i, workChan, resultChan, &workerWg)
	}

	// Start result collector
	var collectorWg sync.WaitGroup
	collectorWg.Add(1)
	go suite.resultCollector(resultChan, &collectorWg)

	// Generate work items
	suite.generateWorkload(profile, workChan)

	// Wait for workers to complete
	close(workChan)
	workerWg.Wait()

	// Wait for result collection to complete
	close(resultChan)
	collectorWg.Wait()

	// Record final resource usage
	runtime.ReadMemStats(&memStats)
	suite.performanceStats.MemoryUsageEnd = int64(memStats.Alloc)
	suite.performanceStats.GoroutineCountEnd = runtime.NumGoroutine()
	suite.performanceStats.endTime = time.Now()

	// Calculate final metrics
	suite.calculateMetrics()

	return suite.performanceStats
}

func (suite *LoadTestSuite) runMixedOperationsTest(t *testing.T, profile *LoadProfile) *PerformanceStats {
	// Similar to runLoadTestWithProfile but with varying payload sizes
	return suite.runLoadTestWithProfile(t, profile)
}

func (suite *LoadTestSuite) generateWorkload(profile *LoadProfile, workChan chan<- WorkItem) {
	ticker := time.NewTicker(time.Duration(float64(time.Second) / profile.RequestsPerSecond))
	defer ticker.Stop()

	deadline := time.Now().Add(profile.Duration)
	itemID := 0

	for time.Now().Before(deadline) {
		select {
		case <-ticker.C:
			workItem := suite.createWorkItem(profile, itemID)
			select {
			case workChan <- workItem:
				itemID++
			default:
				// Channel full, skip this work item
			}
		}
	}
}

func (suite *LoadTestSuite) createWorkItem(profile *LoadProfile, itemID int) WorkItem {
	// Determine operation type based on distribution
	rand.Seed(time.Now().UnixNano() + int64(itemID))
	r := rand.Float64()
	
	var operation string
	cumulative := 0.0
	for op, prob := range profile.OperationDistribution {
		cumulative += prob
		if r <= cumulative {
			operation = op
			break
		}
	}

	// Generate address based on complexity
	address := suite.generateAddress(profile.AddressPatternComplexity, itemID)

	// Generate payload
	payload := suite.generatePayload(profile.PayloadSizeBytes, itemID)

	// Apply failure injection if enabled
	shouldFail := false
	if profile.FailureInjectionRate > 0 {
		shouldFail = rand.Float64() < profile.FailureInjectionRate
	}

	return WorkItem{
		ID:        itemID,
		Operation: operation,
		Address:   address,
		Payload:   payload,
		ShouldFail: shouldFail,
		Timestamp: time.Now(),
	}
}

func (suite *LoadTestSuite) generateAddress(complexity string, id int) string {
	var pattern string
	switch complexity {
	case "simple":
		pattern = fmt.Sprintf("ucxl://agent-%d:developer@project:task-%d/*^", id%10, id)
	case "medium":
		pattern = fmt.Sprintf("ucxl://user-%d:role-%d@project-%d:task-%d/path/%d*^", 
			id%100, id%3, id%20, id, id%5)
	case "complex":
		pattern = fmt.Sprintf("ucxl://service-%d:role-%d@namespace-%d:operation-%d/api/v%d/resource-%d*^",
			id%500, id%5, id%50, id, (id%3)+1, id%100)
	default:
		pattern = fmt.Sprintf("ucxl://default-%d:developer@project:task-%d/*^", id, id)
	}
	return pattern
}

func (suite *LoadTestSuite) generatePayload(size int, id int) []byte {
	payload := make([]byte, size)
	
	// Create realistic payload with some structure
	data := map[string]interface{}{
		"id":        id,
		"timestamp": time.Now().Unix(),
		"type":      "load_test_data",
		"content":   make([]byte, size-200), // Leave room for JSON overhead
	}
	
	// Fill content with pseudo-random but deterministic data
	rand.Seed(int64(id))
	for i := range data["content"].([]byte) {
		data["content"].([]byte)[i] = byte(rand.Intn(256))
	}
	
	jsonData, _ := json.Marshal(data)
	
	// Pad or truncate to exact size
	if len(jsonData) < size {
		padding := make([]byte, size-len(jsonData))
		payload = append(jsonData, padding...)
	} else {
		payload = jsonData[:size]
	}
	
	return payload
}

func (suite *LoadTestSuite) loadTestWorker(workerID int, workChan <-chan WorkItem, resultChan chan<- WorkResult, wg *sync.WaitGroup) {
	defer wg.Done()

	for workItem := range workChan {
		startTime := time.Now()
		var err error
		var success bool

		// Simulate failure injection
		if workItem.ShouldFail {
			err = fmt.Errorf("injected failure for testing")
			success = false
		} else {
			// Perform actual operation
			switch workItem.Operation {
			case "PUT":
				err = suite.dhtStorage.PutValue(suite.ctx, workItem.Address, workItem.Payload)
			case "GET":
				_, err = suite.dhtStorage.GetValue(suite.ctx, workItem.Address)
			case "DELETE":
				// Mock DHT doesn't have delete, so simulate it
				_, err = suite.dhtStorage.GetValue(suite.ctx, workItem.Address)
				if err == nil {
					// Simulate delete by putting empty value
					err = suite.dhtStorage.PutValue(suite.ctx, workItem.Address, []byte{})
				}
			default:
				err = fmt.Errorf("unknown operation: %s", workItem.Operation)
			}
			success = err == nil
		}

		duration := time.Since(startTime)

		result := WorkResult{
			WorkerID:    workerID,
			WorkItemID:  workItem.ID,
			Operation:   workItem.Operation,
			Duration:    duration,
			Success:     success,
			Error:       err,
			CompletedAt: time.Now(),
		}

		select {
		case resultChan <- result:
		default:
			// Result channel full, drop result (shouldn't happen with proper sizing)
		}
	}
}

func (suite *LoadTestSuite) resultCollector(resultChan <-chan WorkResult, wg *sync.WaitGroup) {
	defer wg.Done()

	for result := range resultChan {
		suite.performanceStats.mu.Lock()
		
		atomic.AddInt64(&suite.performanceStats.TotalOperations, 1)
		
		if result.Success {
			atomic.AddInt64(&suite.performanceStats.SuccessfulOperations, 1)
		} else {
			atomic.AddInt64(&suite.performanceStats.FailedOperations, 1)
		}

		// Record latency
		suite.performanceStats.latencies = append(suite.performanceStats.latencies, result.Duration)
		
		// Update min/max latency
		if result.Duration < suite.performanceStats.MinLatency {
			suite.performanceStats.MinLatency = result.Duration
		}
		if result.Duration > suite.performanceStats.MaxLatency {
			suite.performanceStats.MaxLatency = result.Duration
		}

		// Update latency histogram (0-99ms buckets)
		bucketIndex := int(result.Duration.Nanoseconds() / int64(time.Millisecond))
		if bucketIndex >= len(suite.performanceStats.LatencyHistogram) {
			bucketIndex = len(suite.performanceStats.LatencyHistogram) - 1
		}
		suite.performanceStats.LatencyHistogram[bucketIndex]++

		suite.performanceStats.mu.Unlock()
	}
}

func (suite *LoadTestSuite) calculateMetrics() {
	suite.performanceStats.mu.Lock()
	defer suite.performanceStats.mu.Unlock()

	if len(suite.performanceStats.latencies) == 0 {
		return
	}

	// Calculate average latency
	var totalLatency time.Duration
	for _, latency := range suite.performanceStats.latencies {
		totalLatency += latency
	}
	suite.performanceStats.AverageLatency = totalLatency / time.Duration(len(suite.performanceStats.latencies))

	// Calculate percentiles
	latencies := make([]time.Duration, len(suite.performanceStats.latencies))
	copy(latencies, suite.performanceStats.latencies)
	
	// Simple sort for percentile calculation (could use more efficient algorithm)
	for i := 0; i < len(latencies); i++ {
		for j := i + 1; j < len(latencies); j++ {
			if latencies[i] > latencies[j] {
				latencies[i], latencies[j] = latencies[j], latencies[i]
			}
		}
	}

	// Calculate percentiles
	suite.performanceStats.P50Latency = latencies[len(latencies)*50/100]
	suite.performanceStats.P95Latency = latencies[len(latencies)*95/100]
	suite.performanceStats.P99Latency = latencies[len(latencies)*99/100]

	// Calculate throughput
	duration := suite.performanceStats.endTime.Sub(suite.performanceStats.startTime)
	suite.performanceStats.Throughput = float64(suite.performanceStats.TotalOperations) / duration.Seconds()

	// Calculate error rate
	if suite.performanceStats.TotalOperations > 0 {
		suite.performanceStats.ErrorRate = float64(suite.performanceStats.FailedOperations) / float64(suite.performanceStats.TotalOperations) * 100
	}

	// Update memory usage peak
	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)
	if int64(memStats.Alloc) > suite.performanceStats.MemoryUsagePeak {
		suite.performanceStats.MemoryUsagePeak = int64(memStats.Alloc)
	}

	// Update goroutine count peak  
	currentGoroutines := runtime.NumGoroutine()
	if currentGoroutines > suite.performanceStats.GoroutineCountPeak {
		suite.performanceStats.GoroutineCountPeak = currentGoroutines
	}
}

// Supporting types

type WorkItem struct {
	ID        int       `json:"id"`
	Operation string    `json:"operation"`
	Address   string    `json:"address"`
	Payload   []byte    `json:"payload"`
	ShouldFail bool     `json:"should_fail"`
	Timestamp time.Time `json:"timestamp"`
}

type WorkResult struct {
	WorkerID    int           `json:"worker_id"`
	WorkItemID  int           `json:"work_item_id"`
	Operation   string        `json:"operation"`
	Duration    time.Duration `json:"duration"`
	Success     bool          `json:"success"`
	Error       error         `json:"error,omitempty"`
	CompletedAt time.Time     `json:"completed_at"`
}

// ResourceMonitor implementation

func (rm *ResourceMonitor) Start() {
	rm.mu.Lock()
	defer rm.mu.Unlock()

	if rm.monitoring {
		return
	}

	rm.monitoring = true
	go rm.monitorResources()
}

func (rm *ResourceMonitor) Stop() {
	rm.mu.Lock()
	defer rm.mu.Unlock()
	rm.monitoring = false
}

func (rm *ResourceMonitor) monitorResources() {
	ticker := time.NewTicker(rm.interval)
	defer ticker.Stop()

	for {
		rm.mu.RLock()
		if !rm.monitoring {
			rm.mu.RUnlock()
			break
		}
		rm.mu.RUnlock()

		// Collect metrics
		var memStats runtime.MemStats
		runtime.ReadMemStats(&memStats)

		rm.mu.Lock()
		rm.memoryUsage = append(rm.memoryUsage, int64(memStats.Alloc))
		rm.goroutineCount = append(rm.goroutineCount, runtime.NumGoroutine())
		rm.timestamps = append(rm.timestamps, time.Now())
		rm.mu.Unlock()

		<-ticker.C
	}
}