Pre-cleanup snapshot - all current files

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

modules/shhh/api/__init__.py

@@ -0,0 +1,4 @@
+# SHHH API Module
+"""
+FastAPI dashboard and API endpoints for SHHH Secrets Sentinel.
+"""

modules/shhh/api/main.py

@@ -0,0 +1,374 @@
+"""
+FastAPI Dashboard Backend for SHHH Secrets Sentinel
+Provides REST API endpoints for quarantine management and system monitoring.
+"""
+
+import asyncio
+from datetime import datetime
+from typing import List, Optional
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException, Depends, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+import structlog
+
+from .models import (
+    QuarantineEntryResponse, QuarantineReviewRequest, RevocationEventResponse,
+    PatternResponse, PatternUpdateRequest, StatsResponse, SystemHealthResponse,
+    ProcessingStatsResponse, AlertRequest, WebhookTestRequest, WebhookTestResponse,
+    PatternTestRequest, PatternTestResponse, SearchRequest, PaginatedResponse
+)
+from ..core.quarantine import QuarantineManager, QuarantineEntry
+from ..core.detector import SecretDetector
+from ..automation.revocation import SecretRevoker
+from ..pipeline.processor import MessageProcessor
+
+logger = structlog.get_logger()
+
+# Global components (initialized in lifespan)
+quarantine_manager: Optional[QuarantineManager] = None
+detector: Optional[SecretDetector] = None
+revoker: Optional[SecretRevoker] = None
+processor: Optional[MessageProcessor] = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan manager"""
+    global quarantine_manager, detector, revoker, processor
+    
+    try:
+        # Initialize components
+        logger.info("Initializing SHHH API components")
+        
+        # These would normally come from configuration
+        config = {
+            'database_url': 'postgresql://shhh:password@localhost:5432/shhh_sentinel',
+            'patterns_file': 'patterns.yaml',
+            'revocation_webhooks': {
+                'AWS_ACCESS_KEY': 'https://security.chorus.services/hooks/aws-revoke',
+                'GITHUB_TOKEN': 'https://security.chorus.services/hooks/github-revoke',
+                'SLACK_TOKEN': 'https://security.chorus.services/hooks/slack-revoke'
+            }
+        }
+        
+        # Initialize quarantine manager
+        quarantine_manager = QuarantineManager(config['database_url'])
+        await quarantine_manager.initialize()
+        
+        # Initialize detector
+        detector = SecretDetector(config['patterns_file'])
+        
+        # Initialize revoker
+        revoker = SecretRevoker(quarantine_manager, config['revocation_webhooks'])
+        
+        logger.info("SHHH API components initialized successfully")
+        
+        yield
+        
+    except Exception as e:
+        logger.error(f"Failed to initialize SHHH API: {e}")
+        raise
+    finally:
+        # Cleanup
+        if quarantine_manager:
+            await quarantine_manager.close()
+        logger.info("SHHH API components shut down")
+
+
+app = FastAPI(
+    title="SHHH Secrets Sentinel API",
+    description="REST API for managing secrets detection, quarantine, and response",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Configure appropriately for production
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# Dependency functions
+async def get_quarantine_manager() -> QuarantineManager:
+    if not quarantine_manager:
+        raise HTTPException(status_code=503, detail="Quarantine manager not available")
+    return quarantine_manager
+
+
+async def get_detector() -> SecretDetector:
+    if not detector:
+        raise HTTPException(status_code=503, detail="Secret detector not available")
+    return detector
+
+
+async def get_revoker() -> SecretRevoker:
+    if not revoker:
+        raise HTTPException(status_code=503, detail="Secret revoker not available")
+    return revoker
+
+
+# Health and status endpoints
+@app.get("/health", response_model=SystemHealthResponse)
+async def get_health():
+    """Get system health status"""
+    health = {
+        'status': 'healthy',
+        'timestamp': datetime.now(),
+        'components': {
+            'quarantine_manager': {
+                'initialized': quarantine_manager is not None,
+                'database_connected': quarantine_manager.pool is not None if quarantine_manager else False
+            },
+            'detector': {
+                'initialized': detector is not None,
+                'patterns_loaded': len(detector.patterns) if detector else 0
+            },
+            'revoker': {
+                'initialized': revoker is not None,
+                'webhooks_configured': len(revoker.webhook_config) if revoker else 0
+            }
+        }
+    }
+    
+    return health
+
+
+@app.get("/stats", response_model=StatsResponse)
+async def get_stats(qm: QuarantineManager = Depends(get_quarantine_manager)):
+    """Get quarantine statistics"""
+    stats = await qm.get_quarantine_stats()
+    return stats
+
+
+# Quarantine management endpoints
+@app.get("/quarantine", response_model=List[QuarantineEntryResponse])
+async def get_quarantine_entries(
+    limit: int = 100,
+    offset: int = 0,
+    severity: Optional[str] = None,
+    reviewed: Optional[bool] = None,
+    qm: QuarantineManager = Depends(get_quarantine_manager)
+):
+    """Get quarantine entries with optional filters"""
+    entries = await qm.get_quarantine_entries(
+        limit=limit,
+        offset=offset,
+        severity_filter=severity,
+        reviewed_filter=reviewed
+    )
+    
+    return [QuarantineEntryResponse(**entry.__dict__) for entry in entries]
+
+
+@app.post("/quarantine/search", response_model=PaginatedResponse)
+async def search_quarantine_entries(
+    search: SearchRequest,
+    qm: QuarantineManager = Depends(get_quarantine_manager)
+):
+    """Search quarantine entries with advanced filters"""
+    # This would implement more complex search logic
+    entries = await qm.get_quarantine_entries(
+        limit=search.limit,
+        offset=search.offset,
+        severity_filter=search.severity,
+        reviewed_filter=search.reviewed
+    )
+    
+    items = [QuarantineEntryResponse(**entry.__dict__) for entry in entries]
+    
+    return PaginatedResponse(
+        items=items,
+        total=len(items),  # This would be the actual total from a count query
+        limit=search.limit,
+        offset=search.offset,
+        has_more=len(items) == search.limit
+    )
+
+
+@app.post("/quarantine/{entry_id}/review")
+async def review_quarantine_entry(
+    entry_id: int,
+    review: QuarantineReviewRequest,
+    qm: QuarantineManager = Depends(get_quarantine_manager)
+):
+    """Mark a quarantine entry as reviewed"""
+    success = await qm.mark_reviewed(entry_id, review.action, review.reviewer)
+    
+    if not success:
+        raise HTTPException(status_code=404, detail="Quarantine entry not found")
+    
+    return {"status": "success", "message": f"Entry {entry_id} marked as {review.action}"}
+
+
+@app.get("/quarantine/{entry_id}")
+async def get_quarantine_entry(
+    entry_id: int,
+    qm: QuarantineManager = Depends(get_quarantine_manager)
+):
+    """Get a specific quarantine entry by ID"""
+    # This would need to be implemented in QuarantineManager
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+# Pattern management endpoints
+@app.get("/patterns", response_model=List[PatternResponse])
+async def get_patterns(detector: SecretDetector = Depends(get_detector)):
+    """Get all detection patterns"""
+    patterns = []
+    for name, config in detector.patterns.items():
+        patterns.append(PatternResponse(
+            name=name,
+            regex=config['regex'],
+            description=config.get('description', ''),
+            severity=config.get('severity', 'MEDIUM'),
+            confidence=config.get('confidence', 0.8),
+            active=config.get('active', True)
+        ))
+    
+    return patterns
+
+
+@app.post("/patterns/{pattern_name}")
+async def update_pattern(
+    pattern_name: str,
+    pattern: PatternUpdateRequest,
+    detector: SecretDetector = Depends(get_detector)
+):
+    """Update or create a detection pattern"""
+    # This would update the patterns.yaml file
+    # For now, just update in memory
+    detector.patterns[pattern_name] = {
+        'regex': pattern.regex,
+        'description': pattern.description,
+        'severity': pattern.severity,
+        'confidence': pattern.confidence,
+        'active': pattern.active
+    }
+    
+    # Recompile regex
+    import re
+    try:
+        detector.patterns[pattern_name]['compiled_regex'] = re.compile(
+            pattern.regex, re.MULTILINE | re.DOTALL
+        )
+    except re.error as e:
+        raise HTTPException(status_code=400, detail=f"Invalid regex: {e}")
+    
+    return {"status": "success", "message": f"Pattern {pattern_name} updated"}
+
+
+@app.post("/patterns/{pattern_name}/test", response_model=PatternTestResponse)
+async def test_pattern(
+    pattern_name: str,
+    test_request: PatternTestRequest,
+    detector: SecretDetector = Depends(get_detector)
+):
+    """Test a detection pattern against sample text"""
+    try:
+        matches = detector.test_pattern(pattern_name, test_request.test_text)
+        return PatternTestResponse(
+            matches=[match.__dict__ for match in matches],
+            match_count=len(matches)
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+
+@app.get("/patterns/stats")
+async def get_pattern_stats(detector: SecretDetector = Depends(get_detector)):
+    """Get pattern statistics"""
+    return detector.get_pattern_stats()
+
+
+# Revocation management endpoints
+@app.get("/revocations", response_model=List[RevocationEventResponse])
+async def get_revocations(
+    limit: int = 100,
+    offset: int = 0,
+    qm: QuarantineManager = Depends(get_quarantine_manager)
+):
+    """Get revocation events"""
+    # This would need to be implemented in QuarantineManager
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@app.post("/revocations/test", response_model=WebhookTestResponse)
+async def test_webhook(
+    test_request: WebhookTestRequest,
+    revoker: SecretRevoker = Depends(get_revoker)
+):
+    """Test a webhook endpoint"""
+    result = await revoker.test_webhook_endpoint(test_request.secret_type)
+    return WebhookTestResponse(**result)
+
+
+@app.get("/revocations/stats")
+async def get_revocation_stats(revoker: SecretRevoker = Depends(get_revoker)):
+    """Get revocation statistics"""
+    return revoker.get_stats()
+
+
+# Administrative endpoints
+@app.post("/admin/cleanup")
+async def cleanup_old_entries(
+    qm: QuarantineManager = Depends(get_quarantine_manager)
+):
+    """Clean up old quarantine entries"""
+    deleted_count = await qm.cleanup_old_entries()
+    return {"status": "success", "deleted_entries": deleted_count}
+
+
+@app.post("/admin/reload-patterns")
+async def reload_patterns(detector: SecretDetector = Depends(get_detector)):
+    """Reload detection patterns from file"""
+    detector.load_patterns()
+    return {"status": "success", "message": "Patterns reloaded"}
+
+
+@app.post("/admin/reset-stats")
+async def reset_stats(revoker: SecretRevoker = Depends(get_revoker)):
+    """Reset revocation statistics"""
+    revoker.reset_stats()
+    return {"status": "success", "message": "Statistics reset"}
+
+
+# Monitoring endpoints
+@app.get("/metrics/prometheus")
+async def get_prometheus_metrics():
+    """Get metrics in Prometheus format"""
+    # This would generate Prometheus-formatted metrics
+    raise HTTPException(status_code=501, detail="Prometheus metrics not implemented yet")
+
+
+@app.get("/logs/recent")
+async def get_recent_logs(limit: int = 100):
+    """Get recent system logs"""
+    # This would return recent log entries
+    raise HTTPException(status_code=501, detail="Log endpoint not implemented yet")
+
+
+# Error handlers
+@app.exception_handler(Exception)
+async def general_exception_handler(request, exc):
+    logger.error(f"Unhandled exception: {exc}")
+    return JSONResponse(
+        status_code=500,
+        content={"detail": "Internal server error"}
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "api.main:app",
+        host="127.0.0.1",
+        port=8000,
+        reload=True,
+        log_level="info"
+    )

modules/shhh/api/models.py

@@ -0,0 +1,149 @@
+"""
+Pydantic models for SHHH API endpoints.
+"""
+
+from datetime import datetime
+from typing import List, Dict, Any, Optional
+from pydantic import BaseModel, Field
+
+
+class QuarantineEntryResponse(BaseModel):
+    """Response model for quarantine entries"""
+    id: int
+    timestamp: datetime
+    hypercore_position: int
+    bzzz_message_id: Optional[str] = None
+    secret_type: str
+    severity: str
+    confidence: float
+    redacted_content: str
+    content_hash: str
+    source_agent: str
+    match_count: int
+    reviewed: bool
+    review_action: Optional[str] = None
+    reviewer: Optional[str] = None
+    review_timestamp: Optional[datetime] = None
+    metadata: Dict[str, Any] = {}
+
+
+class QuarantineReviewRequest(BaseModel):
+    """Request model for reviewing quarantine entries"""
+    action: str = Field(..., description="Review action: 'false_positive', 'confirmed', 'uncertain'")
+    reviewer: str = Field(..., description="Name or ID of the reviewer")
+    notes: Optional[str] = Field(None, description="Optional review notes")
+
+
+class RevocationEventResponse(BaseModel):
+    """Response model for revocation events"""
+    id: int
+    quarantine_id: int
+    secret_type: str
+    revocation_method: str
+    status: str
+    response_data: Dict[str, Any] = {}
+    timestamp: datetime
+
+
+class PatternResponse(BaseModel):
+    """Response model for detection patterns"""
+    name: str
+    regex: str
+    description: str
+    severity: str
+    confidence: float
+    active: bool
+
+
+class PatternUpdateRequest(BaseModel):
+    """Request model for updating patterns"""
+    regex: str = Field(..., description="Regular expression pattern")
+    description: Optional[str] = Field(None, description="Pattern description")
+    severity: str = Field(..., description="Severity level: LOW, MEDIUM, HIGH, CRITICAL")
+    confidence: float = Field(..., ge=0.0, le=1.0, description="Confidence score")
+    active: bool = Field(True, description="Whether pattern is active")
+
+
+class StatsResponse(BaseModel):
+    """Response model for system statistics"""
+    total_entries: int
+    pending_review: int
+    critical_count: int
+    high_count: int
+    medium_count: int
+    low_count: int
+    last_24h: int
+    last_7d: int
+
+
+class SystemHealthResponse(BaseModel):
+    """Response model for system health"""
+    status: str
+    timestamp: datetime
+    components: Dict[str, Dict[str, Any]]
+
+
+class ProcessingStatsResponse(BaseModel):
+    """Response model for processing statistics"""
+    entries_processed: int
+    secrets_detected: int
+    messages_quarantined: int
+    revocations_triggered: int
+    processing_errors: int
+    uptime_hours: Optional[float] = None
+    entries_per_second: Optional[float] = None
+    secrets_per_hour: Optional[float] = None
+    is_running: bool
+
+
+class AlertRequest(BaseModel):
+    """Request model for manual alerts"""
+    message: str = Field(..., description="Alert message")
+    severity: str = Field(..., description="Alert severity")
+    source: str = Field(..., description="Alert source")
+
+
+class WebhookTestRequest(BaseModel):
+    """Request model for testing webhook endpoints"""
+    secret_type: str = Field(..., description="Secret type to test")
+
+
+class WebhookTestResponse(BaseModel):
+    """Response model for webhook tests"""
+    success: bool
+    method: Optional[str] = None
+    response_data: Dict[str, Any] = {}
+    error: Optional[str] = None
+
+
+class PatternTestRequest(BaseModel):
+    """Request model for testing detection patterns"""
+    pattern_name: str = Field(..., description="Name of pattern to test")
+    test_text: str = Field(..., description="Text to test against pattern")
+
+
+class PatternTestResponse(BaseModel):
+    """Response model for pattern testing"""
+    matches: List[Dict[str, Any]]
+    match_count: int
+
+
+class SearchRequest(BaseModel):
+    """Request model for searching quarantine entries"""
+    query: Optional[str] = Field(None, description="Search query")
+    secret_type: Optional[str] = Field(None, description="Filter by secret type")
+    severity: Optional[str] = Field(None, description="Filter by severity")
+    reviewed: Optional[bool] = Field(None, description="Filter by review status")
+    start_date: Optional[datetime] = Field(None, description="Start date filter")
+    end_date: Optional[datetime] = Field(None, description="End date filter")
+    limit: int = Field(100, ge=1, le=1000, description="Result limit")
+    offset: int = Field(0, ge=0, description="Result offset")
+
+
+class PaginatedResponse(BaseModel):
+    """Generic paginated response model"""
+    items: List[Any]
+    total: int
+    limit: int
+    offset: int
+    has_more: bool