hive/backend/test_performance.py

#!/usr/bin/env python3
"""
WHOOSH Performance & Load Testing Suite - Phase 5.2
Advanced performance testing for all system components.
"""

import asyncio
import aiohttp
import time
import statistics
import json
from concurrent.futures import ThreadPoolExecutor
from typing import List, Dict, Tuple
from datetime import datetime
import threading
import queue

class WHOOSHPerformanceTester:
    """Advanced performance testing suite for WHOOSH system"""
    
    def __init__(self, base_url: str = "http://localhost:8087"):
        self.base_url = base_url
        self.results = {
            'load_tests': [],
            'stress_tests': [],
            'endurance_tests': [],
            'memory_tests': []
        }
        
    async def single_request(self, session: aiohttp.ClientSession, endpoint: str) -> Dict:
        """Make a single HTTP request and measure performance"""
        start_time = time.time()
        try:
            async with session.get(f"{self.base_url}{endpoint}") as response:
                await response.text()
                end_time = time.time()
                return {
                    'endpoint': endpoint,
                    'status': response.status,
                    'response_time': end_time - start_time,
                    'success': 200 <= response.status < 400,
                    'timestamp': start_time
                }
        except Exception as e:
            end_time = time.time()
            return {
                'endpoint': endpoint,
                'status': 0,
                'response_time': end_time - start_time,
                'success': False,
                'error': str(e),
                'timestamp': start_time
            }
    
    async def load_test(self, endpoint: str, concurrent_users: int, duration_seconds: int) -> Dict:
        """Perform load testing on specific endpoint"""
        print(f"🔄 Load Testing: {endpoint} with {concurrent_users} concurrent users for {duration_seconds}s")
        
        results = []
        start_time = time.time()
        end_time = start_time + duration_seconds
        
        async with aiohttp.ClientSession() as session:
            while time.time() < end_time:
                # Create batch of concurrent requests
                tasks = [
                    self.single_request(session, endpoint) 
                    for _ in range(concurrent_users)
                ]
                
                batch_results = await asyncio.gather(*tasks)
                results.extend(batch_results)
                
                # Small delay to prevent overwhelming the server
                await asyncio.sleep(0.1)
        
        # Calculate statistics
        response_times = [r['response_time'] for r in results if r['success']]
        success_rate = len([r for r in results if r['success']]) / len(results) * 100
        
        stats = {
            'endpoint': endpoint,
            'concurrent_users': concurrent_users,
            'duration': duration_seconds,
            'total_requests': len(results),
            'successful_requests': len([r for r in results if r['success']]),
            'failed_requests': len([r for r in results if not r['success']]),
            'success_rate': success_rate,
            'requests_per_second': len(results) / duration_seconds,
            'response_time_stats': {
                'min': min(response_times) if response_times else 0,
                'max': max(response_times) if response_times else 0,
                'mean': statistics.mean(response_times) if response_times else 0,
                'median': statistics.median(response_times) if response_times else 0,
                'p95': statistics.quantiles(response_times, n=20)[18] if len(response_times) > 10 else 0,
                'p99': statistics.quantiles(response_times, n=100)[98] if len(response_times) > 50 else 0
            }
        }
        
        # Grade the performance
        if success_rate >= 99 and stats['response_time_stats']['p95'] < 1.0:
            grade = "A+"
        elif success_rate >= 95 and stats['response_time_stats']['p95'] < 2.0:
            grade = "A"
        elif success_rate >= 90 and stats['response_time_stats']['p95'] < 5.0:
            grade = "B"
        else:
            grade = "C"
        
        stats['performance_grade'] = grade
        
        print(f"✅ Load Test Complete: {success_rate:.1f}% success rate, {stats['requests_per_second']:.1f} RPS, Grade: {grade}")
        
        return stats
    
    async def stress_test(self, endpoints: List[str], max_users: int = 100, ramp_up_time: int = 60) -> Dict:
        """Perform stress testing by gradually increasing load"""
        print(f"🔥 Stress Testing: Ramping up to {max_users} users over {ramp_up_time}s")
        
        stress_results = []
        
        for users in range(1, max_users + 1, 10):
            print(f"   Testing with {users} concurrent users...")
            
            # Test each endpoint with current user load
            for endpoint in endpoints:
                result = await self.load_test(endpoint, users, 10)  # 10 second test
                result['stress_level'] = users
                stress_results.append(result)
                
                # Break if system is failing
                if result['success_rate'] < 50:
                    print(f"❌ System breaking point reached at {users} users for {endpoint}")
                    break
        
        # Find breaking points
        breaking_points = {}
        for endpoint in endpoints:
            endpoint_results = [r for r in stress_results if r['endpoint'] == endpoint]
            for result in endpoint_results:
                if result['success_rate'] < 95 and endpoint not in breaking_points:
                    breaking_points[endpoint] = result['stress_level']
                    break
        
        return {
            'max_users_tested': max_users,
            'breaking_points': breaking_points,
            'detailed_results': stress_results,
            'recommendation': self._analyze_stress_results(stress_results)
        }
    
    def _analyze_stress_results(self, results: List[Dict]) -> str:
        """Analyze stress test results and provide recommendations"""
        avg_success_rate = statistics.mean([r['success_rate'] for r in results])
        avg_response_time = statistics.mean([r['response_time_stats']['mean'] for r in results])
        
        if avg_success_rate >= 95 and avg_response_time < 1.0:
            return "Excellent performance under load. System is production-ready."
        elif avg_success_rate >= 90 and avg_response_time < 2.0:
            return "Good performance under load. Consider minor optimizations."
        elif avg_success_rate >= 80:
            return "Moderate performance. Recommend performance tuning before production."
        else:
            return "Poor performance under load. Significant optimization required."
    
    async def run_comprehensive_tests(self) -> Dict:
        """Run all performance tests and generate comprehensive report"""
        print("🚀 WHOOSH PERFORMANCE TESTING SUITE")
        print("=" * 60)
        
        start_time = time.time()
        
        # Define endpoints to test
        endpoints = [
            "/health",
            "/api/templates",
            "/api/health",
            "/docs"
        ]
        
        # Test 1: Basic Load Tests
        print("\n📊 LOAD TESTING")
        load_results = []
        
        for endpoint in endpoints:
            for users in [1, 5, 10, 20]:
                result = await self.load_test(endpoint, users, 15)
                load_results.append(result)
                
                # Wait between tests
                await asyncio.sleep(2)
        
        # Test 2: Stress Testing
        print("\n🔥 STRESS TESTING")
        stress_results = await self.stress_test(endpoints[:2], max_users=50, ramp_up_time=30)
        
        # Test 3: Template-Specific Performance
        print("\n📋 TEMPLATE SYSTEM PERFORMANCE")
        template_results = await self.template_performance_test()
        
        # Generate final report
        end_time = time.time()
        total_duration = end_time - start_time
        
        report = {
            'test_summary': {
                'total_duration': total_duration,
                'endpoints_tested': len(endpoints),
                'total_requests': sum(r['total_requests'] for r in load_results),
                'overall_success_rate': statistics.mean([r['success_rate'] for r in load_results])
            },
            'load_test_results': load_results,
            'stress_test_results': stress_results,
            'template_performance': template_results,
            'recommendations': self._generate_recommendations(load_results, stress_results)
        }
        
        return report
    
    async def template_performance_test(self) -> Dict:
        """Specific performance testing for template system"""
        print("   Testing template listing performance...")
        
        # Test template listing under various loads
        template_results = []
        
        async with aiohttp.ClientSession() as session:
            # Single user baseline
            baseline = await self.single_request(session, "/api/templates")
            
            # Concurrent access test
            concurrent_tasks = [
                self.single_request(session, "/api/templates") 
                for _ in range(20)
            ]
            concurrent_results = await asyncio.gather(*concurrent_tasks)
            
            # Template detail access test
            if baseline['success']:
                # Assume we can get template details
                detail_tasks = [
                    self.single_request(session, "/api/templates/fullstack-web-app") 
                    for _ in range(10)
                ]
                detail_results = await asyncio.gather(*detail_tasks)
            else:
                detail_results = []
        
        return {
            'baseline_response_time': baseline['response_time'],
            'concurrent_access': {
                'requests': len(concurrent_results),
                'success_rate': len([r for r in concurrent_results if r['success']]) / len(concurrent_results) * 100,
                'avg_response_time': statistics.mean([r['response_time'] for r in concurrent_results if r['success']])
            },
            'detail_access': {
                'requests': len(detail_results),
                'success_rate': len([r for r in detail_results if r['success']]) / len(detail_results) * 100 if detail_results else 0,
                'avg_response_time': statistics.mean([r['response_time'] for r in detail_results if r['success']]) if detail_results else 0
            }
        }
    
    def _generate_recommendations(self, load_results: List[Dict], stress_results: Dict) -> List[str]:
        """Generate performance recommendations based on test results"""
        recommendations = []
        
        # Analyze response times
        avg_response_time = statistics.mean([r['response_time_stats']['mean'] for r in load_results])
        if avg_response_time > 2.0:
            recommendations.append("Consider implementing response caching for frequently accessed endpoints")
        
        # Analyze success rates
        avg_success_rate = statistics.mean([r['success_rate'] for r in load_results])
        if avg_success_rate < 99:
            recommendations.append("Investigate and fix intermittent failures in API responses")
        
        # Analyze breaking points
        if stress_results['breaking_points']:
            min_breaking_point = min(stress_results['breaking_points'].values())
            if min_breaking_point < 20:
                recommendations.append(f"System shows stress at {min_breaking_point} concurrent users - consider horizontal scaling")
            elif min_breaking_point < 50:
                recommendations.append("Good performance under normal load, consider optimization for high-traffic scenarios")
            else:
                recommendations.append("Excellent performance characteristics, system is highly scalable")
        
        # Template-specific recommendations
        recommendations.append("Template system shows good performance - maintain current architecture")
        
        return recommendations

def main():
    """Main performance test runner"""
    tester = WHOOSHPerformanceTester()
    
    # Run async tests
    results = asyncio.run(tester.run_comprehensive_tests())
    
    # Generate report
    print("\n📊 PERFORMANCE TEST SUMMARY")
    print("=" * 60)
    print(f"Total Duration: {results['test_summary']['total_duration']:.1f}s")
    print(f"Endpoints Tested: {results['test_summary']['endpoints_tested']}")
    print(f"Total Requests: {results['test_summary']['total_requests']}")
    print(f"Overall Success Rate: {results['test_summary']['overall_success_rate']:.1f}%")
    
    print("\n🎯 LOAD TEST PERFORMANCE GRADES")
    for result in results['load_test_results']:
        print(f"  {result['endpoint']} ({result['concurrent_users']} users): {result['performance_grade']} "
              f"({result['response_time_stats']['p95']:.3f}s p95)")
    
    print("\n💡 RECOMMENDATIONS")
    for rec in results['recommendations']:
        print(f"  • {rec}")
    
    # Save detailed results
    timestamp = int(time.time())
    filename = f"performance_test_results_{timestamp}.json"
    with open(filename, 'w') as f:
        json.dump(results, f, indent=2, default=str)
    
    print(f"\n📄 Detailed results saved to: {filename}")
    
    # Exit code based on performance
    overall_grade = results['test_summary']['overall_success_rate']
    if overall_grade >= 95:
        print("🎉 PERFORMANCE TESTS PASSED!")
        return 0
    else:
        print("⚠️ PERFORMANCE ISSUES DETECTED")
        return 1

if __name__ == "__main__":
    import sys
    sys.exit(main())