tony/chorus-services
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pre-cleanup snapshot - all current files

Browse Source 
🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
tony
2025-08-05 02:32:45 +10:00
parent 26079aa8da
commit 4511f4c801
32 changed files with 5072 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.obsidian/app.json vendored Normal file
View File

@@ -0,0 +1 @@
{}

1
.obsidian/appearance.json vendored Normal file
View File

@@ -0,0 +1 @@
{}

31
.obsidian/core-plugins.json vendored Normal file
View File

@@ -0,0 +1,31 @@
{
"file-explorer": true,
"global-search": true,
"switcher": true,
"graph": true,
"backlink": true,
"canvas": true,
"outgoing-link": true,
"tag-pane": true,
"properties": false,
"page-preview": true,
"daily-notes": true,
"templates": true,
"note-composer": true,
"command-palette": true,
"slash-command": false,
"editor-status": true,
"bookmarks": true,
"markdown-importer": false,
"zk-prefixer": false,
"random-note": false,
"outline": true,
"word-count": true,
"slides": false,
"audio-recorder": false,
"workspaces": false,
"file-recovery": true,
"publish": false,
"sync": true,
"webviewer": false
}

206
.obsidian/workspace.json vendored Normal file
View File

@@ -0,0 +1,206 @@
{
"main": {
"id": "49f18c78518039c8",
"type": "split",
"children": [
{
"id": "fea88e09bce7fef2",
"type": "tabs",
"children": [
{
"id": "472092e9ada7a8e6",
"type": "leaf",
"state": {
"type": "empty",
"state": {},
"icon": "lucide-file",
"title": "New tab"
}
}
]
}
],
"direction": "vertical"
},
"left": {
"id": "b510957437397946",
"type": "split",
"children": [
{
"id": "157d722a91bc8e15",
"type": "tabs",
"children": [
{
"id": "9001986372506f85",
"type": "leaf",
"state": {
"type": "file-explorer",
"state": {
"sortOrder": "alphabetical",
"autoReveal": false
},
"icon": "lucide-folder-closed",
"title": "Files"
}
},
{
"id": "1d6f26c2d2402f8e",
"type": "leaf",
"state": {
"type": "search",
"state": {
"query": "",
"matchingCase": false,
"explainSearch": false,
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical"
},
"icon": "lucide-search",
"title": "Search"
}
},
{
"id": "5b830db8721ad2ed",
"type": "leaf",
"state": {
"type": "bookmarks",
"state": {},
"icon": "lucide-bookmark",
"title": "Bookmarks"
}
}
]
}
],
"direction": "horizontal",
"width": 481.5
},
"right": {
"id": "a1ab5e22b95db49c",
"type": "split",
"children": [
{
"id": "245785f7c0bf960b",
"type": "tabs",
"children": [
{
"id": "31a2e09288336a61",
"type": "leaf",
"state": {
"type": "backlink",
"state": {
"file": "modules/whoosh/backend/DOCUMENTATION_SUMMARY.md",
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical",
"showSearch": true,
"searchQuery": "",
"backlinkCollapsed": false,
"unlinkedCollapsed": true
},
"icon": "links-coming-in",
"title": "Backlinks for DOCUMENTATION_SUMMARY"
}
},
{
"id": "57211ee20d0c9d61",
"type": "leaf",
"state": {
"type": "outgoing-link",
"state": {
"file": "modules/whoosh/backend/DOCUMENTATION_SUMMARY.md",
"linksCollapsed": false,
"unlinkedCollapsed": true
},
"icon": "links-going-out",
"title": "Outgoing links from DOCUMENTATION_SUMMARY"
}
},
{
"id": "071c40df45653454",
"type": "leaf",
"state": {
"type": "tag",
"state": {
"sortOrder": "frequency",
"useHierarchy": true,
"showSearch": false,
"searchQuery": ""
},
"icon": "lucide-tags",
"title": "Tags"
}
},
{
"id": "1a55201803c42e38",
"type": "leaf",
"state": {
"type": "outline",
"state": {
"file": "modules/whoosh/backend/DOCUMENTATION_SUMMARY.md",
"followCursor": false,
"showSearch": false,
"searchQuery": ""
},
"icon": "lucide-list",
"title": "Outline of DOCUMENTATION_SUMMARY"
}
}
]
}
],
"direction": "horizontal",
"width": 300,
"collapsed": true
},
"left-ribbon": {
"hiddenItems": {
"switcher:Open quick switcher": false,
"graph:Open graph view": false,
"canvas:Create new canvas": false,
"daily-notes:Open today's daily note": false,
"templates:Insert template": false,
"command-palette:Open command palette": false
}
},
"active": "9001986372506f85",
"lastOpenFiles": [
"modules/slurp/hcfs-python/hcfs/core/__pycache__/filesystem.cpython-310.pyc",
"modules/slurp/hcfs-python/hcfs/core/__pycache__/context_db.cpython-310.pyc",
"modules/slurp/hcfs-python/hcfs/core/__pycache__/__init__.cpython-310.pyc",
"modules/slurp/hcfs-python/hcfs/core/__pycache__",
"modules/slurp/hcfs-python/hcfs/__pycache__/__init__.cpython-310.pyc",
"modules/slurp/hcfs-python/hcfs/__pycache__",
"modules/whoosh/EVENT_CONFIGURATION_SYSTEM.md",
"modules/whoosh/EVENT_CONFIGURATION_SYSTEM.md.tmp.1675830.1754294063541",
"modules/whoosh/frontend/src/test/event-config-integration.test.ts",
"modules/whoosh/frontend/src/test/event-config-integration.test.ts.tmp.1675830.1754293976289",
"modules/whoosh/frontend/src/components/projects/EventTypeConfiguration.tsx",
"modules/whoosh/frontend/src/components/projects/EventTypeConfiguration.tsx.tmp.1675830.1754293868591",
"homepage-content.md",
"modules/posthuman/docs/operations.md",
"modules/posthuman/docs/development.md",
"modules/posthuman/docs/api.md",
"modules/posthuman/docs/deployment.md",
"modules/posthuman/docs/architecture.md",
"modules/posthuman/conductor-kernel/PERFORMANCE_OPTIMIZATION.md",
"modules/posthuman/PROJECT_PLAN.md",
"modules/posthuman/README.md",
"modules/hmmm/PROJECT_PLAN.md",
"modules/whoosh/backend/DEPLOYMENT_FIXES.md",
"modules/whoosh/backend/DOCUMENTATION_SUMMARY.md",
"modules/whoosh/docs/project-complete.md",
"modules/whoosh/docs/environment-requirements.md",
"modules/whoosh/docs/implementation-complete.md",
"modules/whoosh/docs/LOCAL_DEVELOPMENT.md",
"modules/whoosh/docs/phase3-completion-summary.md",
"modules/whoosh/docs/phase4-completion-summary.md",
"modules/whoosh/docs/phase5-completion-summary.md",
"modules/whoosh/frontend/TESTING.md",
"modules/whoosh/results/rosewood_qa_report_1751891435.md",
"modules/whoosh/TESTING_STRATEGY.md",
"modules/whoosh/REPORT.md",
"modules/whoosh/README_DISTRIBUTED.md"
]
}

184
homepage-content.md Normal file
View File

@@ -0,0 +1,184 @@
# CHORUS Services - Homepage Content
## Hero Section
### Primary Headline
**AI Development Teams That Think, Learn, and Optimize Themselves**
### Secondary Headline
The next evolution in AI orchestration: Self-optimizing agents that dynamically build optimal teams, learn from every interaction, and deliver auditable results with complete traceability.
### Value Proposition
CHORUS Services transforms how AI development works. Our breakthrough orchestration platform creates autonomous development teams that continuously improve their own performance, automatically form optimal team compositions, and maintain complete audit trails of every decision.
---
## Key Innovations Section
### Self-Optimizing Intelligence
**AI agents that get better with every task**
Our breakthrough reinforcement learning system enables agents to continuously optimize their own performance through real-time feedback loops. Each completed task makes the entire system more effective.
- **Sub-5ms task routing** with intelligent load balancing
- **48GB distributed GPU infrastructure** for massive parallel processing
- **Enterprise-grade monitoring** with real-time optimization
### Dynamic Team Formation
**Perfect teams, automatically assembled**
Gone are the days of manually coordinating AI tools. CHORUS agents autonomously analyze task requirements and automatically form optimal team compositions from our 8 specialized agent roles.
- **Composable context management** - Knowledge components mix and match across projects
- **Fine-tuned specialized models** optimized for specific development workflows
- **Real-time team rebalancing** based on workload and capabilities
### Complete Auditability
**Every decision traceable, every solution replayable**
Enterprise development demands transparency. CHORUS provides complete traceability of every decision with the ability to replay and understand exactly how solutions were developed.
- **Immutable decision logs** with cryptographic integrity
- **Full solution replay capability** for debugging and compliance
- **End-to-end workflow transparency** for regulatory requirements
---
## Target Audience Benefits
### For Enterprise Development Teams
**10x your development velocity without losing control**
- Autonomous task distribution across optimal AI team compositions
- Complete audit trails for compliance and quality assurance
- Integration with existing enterprise development workflows
- Real-time performance monitoring and optimization
### For Tech Startups
**Compete with larger teams through AI force multiplication**
- Small team leverage through intelligent task orchestration
- Automatic knowledge capture and reuse across projects
- Cost-effective scaling without proportional headcount increases
- Rapid iteration with continuous system improvement
### For Research Organizations
**Auditable, repeatable AI-assisted research processes**
- Complete reproducibility of AI-assisted research workflows
- Transparent decision-making processes for peer review
- Collaborative reasoning between multiple specialized AI agents
- Long-term knowledge accumulation and institutional memory
### For AI Companies
**Cutting-edge orchestration for your own AI development**
- Advanced context management for complex AI development projects
- Multi-model coordination for hybrid AI solutions
- Performance optimization through continuous learning
- Scalable infrastructure for distributed AI development
---
## Technical Differentiators
### Beyond Basic AI Tools
CHORUS Services isn't another AI assistant or code completion tool. We've built the infrastructure that makes AI agents actually work together as high-performing development teams.
**Traditional AI Tools:**
- Single-agent interactions
- No persistent team memory
- Manual coordination required
- Limited task complexity
**CHORUS Services:**
- Self-organizing multi-agent teams
- Persistent organizational knowledge
- Autonomous task coordination
- Enterprise-scale complexity handling
### The CHORUS Ecosystem
**Integrated components working in perfect harmony**
- **WHOOSH**: Intelligent workflow orchestration with role-based agent assignment
- **BZZZ**: Peer-to-peer coordination without single points of failure
- **SLURP**: Context management that learns what information matters
- **COOEE**: Continuous feedback loops for system optimization
- **HMMM**: Collaborative reasoning before critical decisions
---
## Proven Results
### Measurable Performance Improvements
**Real metrics from production deployments**
- **92% reduction** in context loss events across development sessions
- **78% reduction** in hallucinated or incorrect AI outputs
- **40% fewer iterations** required for project completion
- **60% reduction** in duplicated work across team members
- **34% faster** overall project delivery times
### Enterprise-Ready Architecture
**Built for scale, security, and reliability**
- Multi-tenant SaaS deployment with enterprise security
- Hybrid cloud/on-premises deployment options
- Role-based access control and complete audit logging
- Integration with existing CI/CD and project management tools
---
## Business Outcomes Focus
### Reduce Development Risk
- Complete transparency in AI decision-making processes
- Audit trails for compliance and quality assurance
- Reduced hallucinations through collaborative verification
- Consistent results through continuous system optimization
### Accelerate Innovation
- Faster iteration cycles through intelligent task orchestration
- Knowledge reuse across projects and teams
- Automatic optimization of development workflows
- Scalable capacity without proportional cost increases
### Maintain Control
- Full visibility into AI agent decision-making
- Configurable guardrails and approval workflows
- Human oversight integration at critical decision points
- Complete solution replay for debugging and improvement
---
## Call to Action
### Primary CTA
**Experience Self-Optimizing AI Development**
*Schedule a live demonstration of autonomous team formation and optimization*
### Secondary CTAs
- **View Technical Architecture** - Deep dive into our breakthrough orchestration platform
- **Download Case Study** - See how CHORUS reduced development time by 40% for enterprise clients
- **Request Private Demo** - See your specific development challenges solved in real-time
---
## Trust Indicators
### Production-Proven Technology
"CHORUS Services isn't experimental - it's deployed and delivering measurable results in production environments today."
### Enterprise Security Standards
- SOC 2 Type II compliant infrastructure
- Enterprise-grade data encryption and access controls
- Complete audit logging and compliance reporting
- Hybrid deployment options for sensitive workloads
### Technical Leadership
Built by the team that solved AI's fundamental context and coordination problems. Our research-to-production pipeline ensures breakthrough innovations reach enterprise customers quickly and reliably.
---
*Ready to transform your development velocity with self-optimizing AI teams?*
**Schedule your demonstration today.**

0
modules/hmmm/PROJECT_PLAN.md Normal file
View File

1
modules/posthuman Submodule

Submodule modules/posthuman added at 2e39cd8664

37
modules/shhh/Dockerfile Normal file
View File

@@ -0,0 +1,37 @@
# SHHH Secrets Sentinel Docker Image
FROM python:3.11-slim
# Set working directory
WORKDIR /app
# Install system dependencies
RUN apt-get update && apt-get install -y \
gcc \
libpq-dev \
curl \
&& rm -rf /var/lib/apt/lists/*
# Copy requirements first for better caching
COPY requirements.txt .
# Install Python dependencies
RUN pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY . .
# Create data directories
RUN mkdir -p /data /config /logs
# Set permissions
RUN chmod +x main.py
# Expose API port
EXPOSE 8000
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8000/health || exit 1
# Default command (can be overridden)
CMD ["python", "main.py", "--mode", "monitor", "--structured-logs"]

319
modules/shhh/PROJECT_PLAN.md Normal file
View File

@@ -0,0 +1,319 @@
## Plan: Hybrid Secret Detection with Sanitized Log Replication
### 1. Objective
To implement a robust, two-stage secret detection pipeline that:
1. Reads from a primary hypercore log in real-time.
2. Uses a fast, regex-based scanner for initial detection.
3. Leverages a local LLM (via Ollama) for deeper, context-aware analysis of potential secrets to reduce false positives.
4. Writes a fully sanitized version of the log to a new, parallel "sister" hypercore stream.
5. Quarantines and alerts on confirmed high-severity secrets, ensuring the original log remains untouched for audit purposes while the sanitized log is safe for wider consumption.
### 2. High-Level Architecture & Data Flow
The process will follow this data flow:
```
┌──────────────────────────┐
[Primary Hypercore Log] ─────► │ HypercoreReader │
└────────────┬─────────────┘
│ (Raw Log Entry)
▼
┌──────────<E29480><E29480>───────────────┐
│ MessageProcessor │
│ (Orchestrator) │
└────────────┬─────────────┘
│
┌───────────────────────▼───────────────────────┐
│ Stage 1: Fast Regex Scan │
│ (SecretDetector) │
└───────────────────────┬───────────────────────┘
│
┌───────────────────────────┼───────────────────────────┐
│ (No Match) │ (Potential Match) │ (High-Confidence Match)
▼ ▼ ▼
┌──────────────────────────┐ ┌─<E2948C><E29480>────────────────────────┐ ┌──────────────────────────┐
│ SanitizedWriter │ │ Stage 2: LLM Analysis │ │ (Skip LLM) │
│ (Writes original entry) │ │ (LLMAnalyzer) │ │ Quarantine Immediately │
└──────────────────────────┘ └────────────┬─────────────┘ └────────────┬─────────────┘
▲ │ (LLM Confirms) │
│ ▼ ▼
│ ┌──────────────────────────┐ ┌──────────────────────────┐
│ │ QuarantineManager │ │ Alerting System │
│ │ (DB Storage, Alerts) │ │ (Webhooks) │
│ └──────────────────────────┘ └────────────<E29480><E29480>─────────────┘
│ │
│ ▼
│ ┌──────────────────────────┐
└──────────────┤ SanitizedWriter │
│ (Writes REDACTED entry) │
└──────────────────────────┘
│
▼
[Sanitized Hypercore Log]
```
### 3. Component Implementation Plan
This plan modifies existing components and adds new ones.
#### 3.1. New Component: `core/llm_analyzer.py`
This new file will contain all logic for interacting with the Ollama instance. This isolates the dependency and makes it easy to test or swap out the LLM backend.
```python
# core/llm_analyzer.py
import requests
import json
class LLMAnalyzer:
"""Analyzes text for secrets using a local LLM via Ollama."""
def __init__(self, endpoint: str, model: str, system_prompt: str):
self.endpoint = endpoint
self.model = model
self.system_prompt = system_prompt
def analyze(self, text: str) -> dict:
"""
Sends text to the Ollama API for analysis and returns a structured JSON response.
Returns:
A dictionary like:
{
"secret_found": bool,
"secret_type": str,
"confidence_score": float,
"severity": str
}
Returns a default "not found" response on error.
"""
prompt = f"Log entry: \"{text}\"\n\nAnalyze this for secrets and respond with only the required JSON."
payload = {
"model": self.model,
"system": self.system_prompt,
"prompt": prompt,
"format": "json",
"stream": False
}
try:
response = requests.post(self.endpoint, json=payload, timeout=15)
response.raise_for_status()
# The response from Ollama is a JSON string, which needs to be parsed.
analysis = json.loads(response.json().get("response", "{}"))
return analysis
except (requests.exceptions.RequestException, json.JSONDecodeError) as e:
print(f"[ERROR] LLMAnalyzer failed: {e}")
# Fallback: If LLM fails, assume no secret was found to avoid blocking the pipeline.
return {"secret_found": False}
```
#### 3.2. New Component: `core/sanitized_writer.py`
This component is responsible for writing to the new, sanitized hypercore log. This abstraction allows us to easily change the output destination in the future.
```python
# core/sanitized_writer.py
class SanitizedWriter:
"""Writes log entries to the sanitized sister hypercore log."""
def __init__(self, sanitized_log_path: str):
self.log_path = sanitized_log_path
# Placeholder for hypercore writing logic. For now, we'll append to a file.
self.log_file = open(self.log_path, "a")
def write(self, log_entry: str):
"""Writes a single log entry to the sanitized stream."""
self.log_file.write(log_entry + "\n")
self.log_file.flush()
def close(self):
self.log_file.close()
```
#### 3.3. Modify: `core/detector.py`
We will enhance the `SecretDetector` to not only find matches but also redact them.
```python
# core/detector.py
import re
class SecretDetector:
def __init__(self, patterns_file: str = "patterns.yaml"):
# ... (load_patterns remains the same) ...
def scan(self, text: str) -> list[dict]:
"""Scans text and returns a list of found secrets with metadata."""
matches = []
for pattern_name, pattern in self.patterns.items():
if pattern.get("active", True):
regex_match = re.search(pattern["regex"], text)
if regex_match:
matches.append({
"secret_type": pattern_name,
"value": regex_match.group(0),
"confidence": pattern.get("confidence", 0.8), # Default confidence
"severity": pattern.get("severity", "MEDIUM")
})
return matches
def redact(self, text: str, secret_value: str) -> str:
"""Redacts a specific secret value within a string."""
redacted_str = secret_value[:4] + "****" + secret_value[-4:]
return text.replace(secret_value, f"[REDACTED:{redacted_str}]")
```
#### 3.4. Modify: `pipeline/processor.py`
This is the orchestrator and will see the most significant changes to implement the hybrid logic.
```python
# pipeline/processor.py
from core.hypercore_reader import HypercoreReader
from core.detector import SecretDetector
from core.llm_analyzer import LLMAnalyzer
from core.quarantine import QuarantineManager
from core.sanitized_writer import SanitizedWriter
class MessageProcessor:
def __init__(self, reader: HypercoreReader, detector: SecretDetector, llm_analyzer: LLMAnalyzer, quarantine: QuarantineManager, writer: SanitizedWriter, llm_threshold: float):
self.reader = reader
self.detector = detector
self.llm_analyzer = llm_analyzer
self.quarantine = quarantine
self.writer = writer
self.llm_threshold = llm_threshold # e.g., 0.90
async def process_stream(self):
"""Main processing loop for the hybrid detection model."""
async for entry in self.reader.stream_entries():
# Stage 1: Fast Regex Scan
regex_matches = self.detector.scan(entry.content)
if not regex_matches:
# No secrets found, write original entry to sanitized log
self.writer.write(entry.content)
continue
# A potential secret was found. Default to sanitized, but may be quarantined.
sanitized_content = entry.content
should_quarantine = False
confirmed_secret = None
for match in regex_matches:
# High-confidence regex matches trigger immediate quarantine, skipping LLM.
if match['confidence'] >= self.llm_threshold:
should_quarantine = True
confirmed_secret = match
break # One high-confidence match is enough
# Stage 2: Low-confidence matches go to LLM for verification.
llm_result = self.llm_analyzer.analyze(entry.content)
if llm_result.get("secret_found"):
should_quarantine = True
# Prefer LLM's classification but use regex value for redaction
confirmed_secret = {
"secret_type": llm_result.get("secret_type", match['secret_type']),
"value": match['value'],
"severity": llm_result.get("severity", match['severity'])
}
break
if should_quarantine and confirmed_secret:
# A secret is confirmed. Redact, quarantine, and alert.
sanitized_content = self.detector.redact(entry.content, confirmed_secret['value'])
self.quarantine.quarantine_message(
message=entry,
secret_type=confirmed_secret['secret_type'],
severity=confirmed_secret['severity'],
redacted_content=sanitized_content
)
# Potentially trigger alerts here as well
print(f"[ALERT] Confirmed secret {confirmed_secret['secret_type']} found and quarantined.")
# Write the (potentially redacted) content to the sanitized log
self.writer.write(sanitized_content)
```
#### 3.5. Modify: `main.py`
The main entry point will be updated to instantiate and wire together the new and modified components.
```python
# main.py
# ... imports ...
import asyncio
from core.hypercore_reader import HypercoreReader
from core.detector import SecretDetector
from core.llm_analyzer import LLMAnalyzer
from core.quarantine import QuarantineManager
from core.sanitized_writer import SanitizedWriter
# ... other imports
def main():
# 1. Configuration
# Load from a new config.yaml or environment variables
PRIMARY_LOG_PATH = "/path/to/primary/hypercore.log"
SANITIZED_LOG_PATH = "/path/to/sanitized/hypercore.log"
PATTERNS_PATH = "patterns.yaml"
DB_CONNECTION = "..."
OLLAMA_ENDPOINT = "http://localhost:11434/api/generate"
OLLAMA_MODEL = "llama3"
LLM_CONFIDENCE_THRESHOLD = 0.90 # Regex confidence >= this skips LLM
with open("SHHH_SECRETS_SENTINEL_AGENT_PROMPT.md", "r") as f:
OLLAMA_SYSTEM_PROMPT = f.read()
# 2. Instantiation
reader = HypercoreReader(PRIMARY_LOG_PATH)
detector = SecretDetector(PATTERNS_PATH)
llm_analyzer = LLMAnalyzer(OLLAMA_ENDPOINT, OLLAMA_MODEL, OLLAMA_SYSTEM_PROMPT)
quarantine = QuarantineManager(DB_CONNECTION)
writer = SanitizedWriter(SANITIZED_LOG_PATH)
processor = MessageProcessor(
reader=reader,
detector=detector,
llm_analyzer=llm_analyzer,
quarantine=quarantine,
writer=writer,
llm_threshold=LLM_CONFIDENCE_THRESHOLD
)
# 3. Execution
print("Starting SHHH Hypercore Monitor...")
try:
asyncio.run(processor.process_stream())
except KeyboardInterrupt:
print("Shutting down...")
finally:
writer.close()
if __name__ == "__main__":
main()
```
### 4. Phased Rollout
1. **Phase 1: Component Implementation (1-2 days)**
* Create `core/llm_analyzer.py` and `core/sanitized_writer.py`.
* Write unit tests for both new components. Mock the `requests` calls for the analyzer.
* Update `core/detector.py` with the `redact` method and update its unit tests.
2. **Phase 2: Orchestration Logic (2-3 days)**
* Implement the new logic in `pipeline/processor.py`.
* Write integration tests for the processor that simulate the full flow: no match, low-confidence match (with mocked LLM response), and high-confidence match.
* Update `main.py` to wire everything together.
3. **Phase 3: Configuration & Testing (1 day)**
* Add a `config.yaml` to manage all paths, thresholds, and endpoints.
* Perform an end-to-end test run with a sample log file and a running Ollama instance.
* Verify that the primary log is untouched, the sanitized log is created correctly (with and without redactions), and the quarantine database is populated as expected.
### 5. Success Criteria
* **Zero Leaks:** The sanitized log stream contains no secrets.
* **High Accuracy:** False positive rate is demonstrably lower than a regex-only solution, verified during testing.
* **Performance:** The pipeline maintains acceptable latency (<200ms per log entry on average, accounting for occasional LLM analysis).
* **Auditability:** The primary log remains a perfect, unaltered source of truth. All detection and quarantine events are logged in the PostgreSQL database.

561
modules/shhh/README.md Normal file
View File

@@ -0,0 +1,561 @@
🔥 Excellent — let’s push this all the way into a **production-grade spec**.
---
## 📂 **1️⃣ Feedback Ingestion Spec**
This defines how curators/humans give feedback to the Sentinel so it can **update its detection rules (patterns.yaml)** safely.
---
### 🔄 **Feedback Flow**
1. **Curator/Reviewer sees alert** → marks it as:
* `false_positive` (regex over-triggered)
* `missed_secret` (regex failed to detect)
* `uncertain` (needs better regex refinement)
2. **Feedback API** ingests the report:
```json
{
"alert_id": "log_345",
"secret_type": "AWS_ACCESS_KEY",
"feedback_type": "false_positive",
"evidence": "Key was dummy data: TESTKEY123",
"suggested_regex_fix": null
}
```
3. **Meta-Learner** updates rules:
* `false_positive` → adds **exceptions** (e.g., allowlist prefixes like `TESTKEY`).
* `missed_secret` → drafts **new regex** from evidence (using regex generator or LLM).
* Writes changes to **patterns.yaml** under `pending_review`.
4. **Security admin approves** before the new regex is marked `active: true`.
---
### 🧠 **Feedback Schema in YAML**
```yaml
pending_updates:
- regex_name: AWS_ACCESS_KEY
action: modify
new_regex: "AKIA[0-9A-Z]{16}(?!TESTKEY)"
confidence: 0.82
status: "pending human review"
submitted_by: curator_2
timestamp: 2025-08-02T12:40:00Z
```
✅ This keeps **audit trails** & allows **safe hot updates**.
---
## ⚙️ **2️⃣ Real AWS/GitHub Webhook Payload Templates**
These are **example POST payloads** your Sentinel would send when it detects a leaked secret.
---
### 🔐 **AWS Access Key Revocation**
**Endpoint:**
`POST https://security.example.com/hooks/aws-revoke`
**Payload:**
```json
{
"event": "secret_leak_detected",
"secret_type": "AWS_ACCESS_KEY",
"redacted_key": "AKIA****XYZ",
"log_reference": "hyperlog:58321",
"recommended_action": "Revoke IAM access key immediately",
"severity": "HIGH",
"timestamp": "2025-08-02T12:45:00Z"
}
```
➡ Your security automation would call AWS CLI or IAM API:
```bash
aws iam update-access-key --access-key-id <redacted> --status Inactive
aws iam delete-access-key --access-key-id <redacted>
```
---
### 🐙 **GitHub Token Revocation**
**Endpoint:**
`POST https://security.example.com/hooks/github-revoke`
**Payload:**
```json
{
"event": "secret_leak_detected",
"secret_type": "GITHUB_TOKEN",
"redacted_key": "ghp_****abcd",
"repository": "repo-name",
"log_reference": "hyperlog:58322",
"severity": "HIGH",
"recommended_action": "Invalidate GitHub token via API",
"timestamp": "2025-08-02T12:46:00Z"
}
```
➡ This would tie into GitHub’s [token-scanning API](https://docs.github.com/en/developers/overview/secret-scanning) or use PAT revocation.
---
### 💬 **Slack Token Revocation**
**Endpoint:**
`POST https://security.example.com/hooks/slack-revoke`
**Payload:**
```json
{
"event": "secret_leak_detected",
"secret_type": "SLACK_TOKEN",
"redacted_key": "xoxb****hjk",
"workspace": "company-slack",
"log_reference": "hyperlog:58323",
"severity": "HIGH",
"recommended_action": "Revoke Slack bot/user token",
"timestamp": "2025-08-02T12:47:00Z"
}
```
➡ Slack Admin API can be used to **revoke** or **rotate**.
---
## 📡 **3️⃣ Redis or PostgreSQL Quarantine Store**
Switching from memory to **persistent storage** means quarantined logs survive restarts.
---
### ✅ **Redis Option (Fast, Volatile)**
```python
import redis, json
r = redis.Redis(host='localhost', port=6379, decode_responses=True)
def quarantine_log(log_line, reason):
entry = {"timestamp": datetime.utcnow().isoformat() + "Z", "reason": reason, "log_line": log_line}
r.lpush("quarantine", json.dumps(entry))
print(f"[QUARANTINE] Stored in Redis: {reason}")
```
* 🏎 **Pros:** Fast, easy to scale.
* ⚠️ **Cons:** Volatile unless persisted (RDB/AOF).
---
### ✅ **PostgreSQL Option (Auditable, Durable)**
**Schema:**
```sql
CREATE TABLE quarantine (
id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ NOT NULL,
reason TEXT NOT NULL,
log_line TEXT NOT NULL,
reviewed BOOLEAN DEFAULT FALSE
);
```
**Python Insert:**
```python
import psycopg2
conn = psycopg2.connect("dbname=sentinel user=postgres password=secret")
cursor = conn.cursor()
def quarantine_log(log_line, reason):
entry_time = datetime.utcnow().isoformat() + "Z"
cursor.execute(
"INSERT INTO quarantine (timestamp, reason, log_line) VALUES (%s, %s, %s)",
(entry_time, reason, log_line)
)
conn.commit()
print(f"[QUARANTINE] Stored in PostgreSQL: {reason}")
```
✅ **Postgres is better for long-term auditing** — you can run reports like:
* “How many AWS keys leaked this month?”
* “Which agents generated the most HIGH-severity quarantines?”
---
We now have:
✅ **Detection → Redaction → Quarantine → Revocation → Feedback → Pattern Evolution**
✅ **patterns.yaml** for versioned regex
✅ **Webhooks** for real-time secret revocation
✅ **Persistent quarantine store** (Redis or Postgres)
---
## 📜 **1️⃣ Migration Script: Redis → PostgreSQL**
This script will migrate existing quarantined log entries from **Redis** to **Postgres**.
```python
import redis, json, psycopg2
from datetime import datetime
# Redis config
r = redis.Redis(host='localhost', port=6379, decode_responses=True)
# Postgres config
conn = psycopg2.connect("dbname=sentinel user=postgres password=secret")
cursor = conn.cursor()
def migrate_quarantine():
count = 0
while True:
entry_json = r.rpop("quarantine") # pop oldest entry from Redis
if not entry_json:
break
entry = json.loads(entry_json)
cursor.execute(
"INSERT INTO quarantine (timestamp, reason, log_line) VALUES (%s, %s, %s)",
(entry["timestamp"], entry["reason"], entry["log_line"])
)
count += 1
conn.commit()
print(f"[MIGRATION] Moved {count} quarantined entries from Redis → PostgreSQL")
if __name__ == "__main__":
migrate_quarantine()
```
✅ **Run once** after Postgres is set up — empties Redis queue into the durable DB.
---
## 🖥 **2️⃣ Admin Dashboard Spec**
**Purpose:** A web UI to manage the Sentinel’s security pipeline.
---
### 🎯 **Core Features**
✅ **Quarantine Browser**
* Paginated view of all quarantined logs
* Search/filter by `secret_type`, `source_agent`, `date`, `status`
* Mark quarantined logs as **reviewed** or **false alarm**
✅ **Regex Rules Manager**
* Lists all regexes from `patterns.yaml`
* Add / update / deactivate rules via UI
* Shows `pending_updates` flagged by the Meta-Learner for human approval
✅ **Revocation Status Board**
* See which secrets triggered revocations
* Status of revocation hooks (success/fail)
✅ **Metrics Dashboard**
* Charts: “Secrets Detected Over Time”, “Top Sources of Leaks”
* KPIs: # HIGH severity secrets this month, # rules updated, # false positives
---
### 🏗 **Tech Stack Suggestion**
* **Backend:** FastAPI (Python)
* **Frontend:** React + Tailwind
* **DB:** PostgreSQL for quarantine + rules history
* **Auth:** OAuth (GitHub/Google) + RBAC (only security admins can approve regex changes)
---
### 🔌 **Endpoints**
```
GET /api/quarantine → list quarantined entries
POST /api/quarantine/review → mark entry as reviewed
GET /api/rules → list regex patterns
POST /api/rules/update → update or add a regex
GET /api/revocations → list revocation events
```
---
### 🖥 **Mock Dashboard Layout**
* **Left Nav:** Quarantine | Rules | Revocations | Metrics
* **Main Panel:**
* Data tables with sorting/filtering
* Inline editors for regex rules
* Approve/Reject buttons for pending regex updates
✅ Basically a **security control room** for Sentinel.
---
## 🤖 **3️⃣ Meta-Curator AI Prompt**
This agent reviews Sentinel’s work and **tunes it automatically**.
---
### **Meta-Curator: System Prompt**
> **Role & Mission:**
> You are the **Meta-Curator**, a supervisory AI responsible for reviewing the **Secrets Sentinel’s** detections, regex updates, and feedback reports.
>
> **Core Responsibilities:**
> ✅ **Audit alerts** – Look for false positives, duplicates, or missed leaks by cross-checking Sentinel outputs.
> ✅ **Review regex proposals** – When Sentinel drafts new regex rules, decide if they’re:
>
> * ✅ Approved (safe to activate)
> * ❌ Rejected (too broad or incorrect)
> * 🕒 Deferred (needs human review)
> ✅ **Tune detection thresholds** – Adjust `confidence` or `severity` on patterns based on outcomes.
> ✅ **Generate new rules** – If multiple missed secrets share a format, draft a regex and submit to humans for approval.
> ✅ **Report upstream** – Summarize changes to security admins weekly.
---
### **Behavior Guidelines**
* **Conservative by default:** Don’t auto-approve regexes unless confidence > 0.95.
* **Keep auditability:** Every decision (approve/reject) is logged in the hyperlog.
* **Respect human overrides:** Never overwrite a regex that a human explicitly locked.
---
### **Example Meta-Curator Output**
```json
{
"action": "approve_regex",
"regex_name": "GITLAB_TOKEN",
"regex_pattern": "glpat-[0-9A-Za-z\\-_]{20}",
"confidence": 0.97,
"decision_reason": "Validated against 12 quarantined examples, no false positives found.",
"timestamp": "2025-08-02T13:45:00Z"
}
```
✅ This meta-agent is the **brains of the rules layer** — keeps Sentinel evolving, but under control.
---
## 🚀 **Now You Have:**
✅ **Migration Path** → Redis → PostgreSQL
✅ **Admin Dashboard Spec** → complete with endpoints & layout
✅ **Meta-Curator Prompt** → the agent that “manages the manager”
Alright — here’s the next batch to lock this into a **real, buildable system**.
---
## 📂 **1️⃣ `patterns_history` Table Schema**
This tracks **every regex change** ever made — who/what made it, why, and when.
```sql
CREATE TABLE patterns_history (
id SERIAL PRIMARY KEY,
regex_name TEXT NOT NULL,
old_regex TEXT,
new_regex TEXT,
action TEXT CHECK (action IN ('add', 'update', 'remove')),
confidence NUMERIC(3,2),
status TEXT CHECK (status IN ('approved', 'pending', 'rejected')),
submitted_by TEXT NOT NULL,
approved_by TEXT,
decision_reason TEXT,
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
```
### ✅ What this gives you:
* **Full audit trail** (critical for security compliance).
* You can run queries like:
* *“Show all regex changes made by Meta-Curator vs. humans.”*
* *“List all rules rejected in the last 90 days.”*
---
## 🖼 **2️⃣ Admin Dashboard Wireframes**
**Goal:** show your devs exactly what to build — no ambiguity.
---
### **🔒 Dashboard Home**
```
------------------------------------------------------
| [Sentinel Logo] Secrets Sentinel Dashboard |
------------------------------------------------------
| Quarantine | Rules | Revocations | Metrics | Admin |
------------------------------------------------------
| Welcome back, Security Admin! |
| |
| ▢ 32 Quarantined logs waiting review |
| ▢ 4 Pending regex updates |
| ▢ 2 Failed revocation hooks |
------------------------------------------------------
```
---
### **🗄 Quarantine View**
```
------------------------------------------------------
| Quarantine Logs |
------------------------------------------------------
| Search: [______________] [Filter ▼] |
------------------------------------------------------
| Log ID | Secret Type | Severity | Status |
------------------------------------------------------
| log_4287 | AWS_ACCESS_KEY| HIGH | PENDING |
| log_4288 | JWT | MEDIUM | REVIEWED|
| log_4289 | SSH_KEY | HIGH | PENDING |
------------------------------------------------------
[ View Details ] [ Mark as Reviewed ] [ Delete ]
```
**Clicking “View Details” →** shows full log snippet (with redacted secret).
---
### **📜 Regex Manager**
```
------------------------------------------------------
| Regex Rules |
------------------------------------------------------
| Name | Regex Pattern | Active |
------------------------------------------------------
| AWS_ACCESS_KEY | AKIA[0-9A-Z]{16} | ✔ |
| JWT | eyJ[A-Za-z0-9_-]+?\.[…] | ✔ |
| SLACK_TOKEN | xox[baprs]-[0-9A-Za-z-]{10,48} | ✔ |
------------------------------------------------------
[ Add New Regex ] [ View History ]
```
Clicking **View History** → pulls from `patterns_history`.
---
### **📊 Metrics View**
* **Line Chart:** “Secrets Detected Over Time”
* **Bar Chart:** “Secrets by Type” (AWS, GitHub, JWT, etc.)
* **KPIs:**
* 🔴 High Severity Leaks: 12 this week
* 🟢 Regex Accuracy: 94%
---
## ⚙️ **3️⃣ FastAPI Skeleton**
Here’s the **starter code** for your dev team to run with.
```python
from fastapi import FastAPI, Depends
from pydantic import BaseModel
from typing import List
import psycopg2, json
app = FastAPI(title="Secrets Sentinel Dashboard API")
# --- Database Setup ---
conn = psycopg2.connect("dbname=sentinel user=postgres password=secret")
cursor = conn.cursor()
# --- Models ---
class QuarantineEntry(BaseModel):
id: int
timestamp: str
reason: str
log_line: str
reviewed: bool
class RegexRule(BaseModel):
regex_name: str
regex_pattern: str
severity: str
confidence: float
active: bool
# --- Endpoints ---
@app.get("/quarantine", response_model=List[QuarantineEntry])
def get_quarantine():
cursor.execute("SELECT id, timestamp, reason, log_line, reviewed FROM quarantine")
rows = cursor.fetchall()
return [QuarantineEntry(id=r[0], timestamp=str(r[1]), reason=r[2], log_line=r[3], reviewed=r[4]) for r in rows]
@app.post("/quarantine/review/{entry_id}")
def review_quarantine(entry_id: int):
cursor.execute("UPDATE quarantine SET reviewed=true WHERE id=%s", (entry_id,))
conn.commit()
return {"status": "ok", "message": f"Quarantine entry {entry_id} marked reviewed"}
@app.get("/rules", response_model=List[RegexRule])
def get_rules():
# Load from patterns.yaml
with open("patterns.yaml", "r") as f:
patterns = json.load(f) if f.read().strip().startswith("{") else {}
rules = []
for name, rule in patterns.get("patterns", {}).items():
rules.append(RegexRule(
regex_name=name,
regex_pattern=rule["regex"],
severity=rule["severity"],
confidence=rule["confidence"],
active=rule["active"]
))
return rules
@app.post("/rules/update")
def update_rule(rule: RegexRule):
# Append to patterns_history table
cursor.execute("""
INSERT INTO patterns_history (regex_name, old_regex, new_regex, action, confidence, status, submitted_by)
VALUES (%s, %s, %s, 'update', %s, 'pending', 'admin')
""", (rule.regex_name, None, rule.regex_pattern, rule.confidence))
conn.commit()
return {"status": "ok", "message": f"Regex {rule.regex_name} queued for update"}
```
✅ **Why this skeleton works:**
* REST endpoints for **Quarantine**, **Rules**, **History**.
* Uses **Postgres for persistence**.
* Reads from `patterns.yaml` for active rules.
---
## 🚀 **Now You Have:**
✅ A **Postgres schema** for regex change history.
✅ **Wireframes** for the admin dashboard.
✅ A **FastAPI skeleton** your team can expand into a full API/UI stack.

512
modules/shhh/SHHH_HYPERCORE_MONITOR_PLAN.md Normal file
View File

@@ -0,0 +1,512 @@
# 🔒 SHHH Hypercore Log Monitor - Implementation Plan
## Executive Summary
This plan outlines the creation of a Python application that monitors our hypercore log to ensure no secrets are leaked in BZZZ messages, based on the SHHH module's secrets detection framework.
## Project Overview
### Objective
Create a real-time monitoring system that:
- Monitors hypercore log entries for secret patterns
- Detects potential secrets in BZZZ P2P messages before they propagate
- Quarantines suspicious entries and triggers automatic remediation
- Provides audit trails and security dashboard for compliance
### Architecture Integration
- **Hypercore Log**: Source of truth for all CHORUS Services events
- **BZZZ Network**: P2P messaging layer that could inadvertently transmit secrets
- **SHHH Module**: Existing secrets detection framework and patterns
- **Monitoring App**: New Python application bridging these systems
## Technical Requirements
### 1. Hypercore Log Integration
```python
# Real-time log monitoring
- Stream hypercore entries as they're written
- Parse BZZZ message payloads for secret patterns
- Filter for message types that could contain secrets
- Handle log rotation and recovery scenarios
```
### 2. Secret Detection Engine
Based on SHHH's `patterns.yaml` framework:
```yaml
patterns:
AWS_ACCESS_KEY:
regex: "AKIA[0-9A-Z]{16}"
severity: "HIGH"
confidence: 0.95
active: true
GITHUB_TOKEN:
regex: "ghp_[0-9A-Za-z]{36}"
severity: "HIGH"
confidence: 0.92
active: true
PRIVATE_KEY:
regex: "-----BEGIN [A-Z ]*PRIVATE KEY-----"
severity: "CRITICAL"
confidence: 0.98
active: true
```
### 3. Quarantine & Response System
- **Immediate**: Block message propagation in BZZZ network
- **Log**: Store quarantined entries in PostgreSQL
- **Alert**: Notify security team via webhooks
- **Revoke**: Trigger automatic secret revocation APIs
## Implementation Architecture
### Phase 1: Core Monitoring System (Weeks 1-2)
#### 1.1 Hypercore Log Reader
```python
# /shhh-monitor/core/hypercore_reader.py
class HypercoreReader:
def __init__(self, log_path: str):
self.log_path = log_path
self.position = 0
def stream_entries(self) -> Iterator[LogEntry]:
"""Stream new hypercore entries in real-time"""
# Tail-like functionality with inotify
# Parse hypercore binary format
# Yield structured LogEntry objects
def parse_bzzz_message(self, entry: LogEntry) -> Optional[BzzzMessage]:
"""Extract BZZZ message payload from hypercore entry"""
# Decode BZZZ message format
# Extract message content and metadata
# Return structured message or None
```
#### 1.2 Secret Detection Engine
```python
# /shhh-monitor/core/detector.py
class SecretDetector:
def __init__(self, patterns_file: str = "patterns.yaml"):
self.patterns = self.load_patterns(patterns_file)
def scan_message(self, message: BzzzMessage) -> List[SecretMatch]:
"""Scan BZZZ message for secret patterns"""
matches = []
for pattern_name, pattern in self.patterns.items():
if pattern["active"]:
matches.extend(self.apply_regex(message, pattern))
return matches
def redact_secret(self, text: str, match: SecretMatch) -> str:
"""Redact detected secret while preserving context"""
# Replace secret with asterisks, keep first/last chars
# Maintain log readability for analysis
```
#### 1.3 Quarantine System
```python
# /shhh-monitor/core/quarantine.py
class QuarantineManager:
def __init__(self, db_connection: str):
self.db = psycopg2.connect(db_connection)
def quarantine_message(self, message: BzzzMessage, matches: List[SecretMatch]):
"""Store quarantined message and block propagation"""
# Insert into quarantine table
# Generate alert payload
# Trigger BZZZ network block
def send_alert(self, severity: str, secret_type: str, redacted_content: str):
"""Send webhook alerts for detected secrets"""
# POST to security webhook endpoints
# Different payloads for AWS, GitHub, Slack tokens
# Include revocation recommendations
```
### Phase 2: BZZZ Network Integration (Weeks 3-4)
#### 2.1 BZZZ Message Interceptor
```python
# /shhh-monitor/integrations/bzzz_interceptor.py
class BzzzInterceptor:
def __init__(self, bzzz_config: Dict):
self.bzzz_client = BzzzClient(bzzz_config)
def install_message_hook(self):
"""Install pre-send hook in BZZZ network layer"""
# Intercept messages before P2P transmission
# Scan with SecretDetector
# Block or allow message propagation
def block_message(self, message_id: str, reason: str):
"""Prevent message from propagating in P2P network"""
# Mark message as blocked in BZZZ
# Log blocking reason
# Notify sender agent of security violation
```
#### 2.2 Real-time Processing Pipeline
```python
# /shhh-monitor/pipeline/processor.py
class MessageProcessor:
def __init__(self, detector: SecretDetector, quarantine: QuarantineManager):
self.detector = detector
self.quarantine = quarantine
async def process_hypercore_stream(self):
"""Main processing loop for hypercore monitoring"""
async for entry in self.hypercore_reader.stream_entries():
if bzzz_message := self.parse_bzzz_message(entry):
matches = self.detector.scan_message(bzzz_message)
if matches:
await self.handle_secret_detection(bzzz_message, matches)
async def handle_secret_detection(self, message: BzzzMessage, matches: List[SecretMatch]):
"""Handle detected secrets with appropriate response"""
# Determine severity level
# Quarantine message
# Send alerts
# Trigger revocation if needed
# Update detection statistics
```
### Phase 3: Admin Dashboard & Feedback Loop (Weeks 5-6)
#### 3.1 FastAPI Backend
```python
# /shhh-monitor/api/main.py
from fastapi import FastAPI, Depends
from .models import QuarantineEntry, SecretPattern, RevocationEvent
app = FastAPI(title="SHHH Hypercore Monitor API")
@app.get("/quarantine", response_model=List[QuarantineEntry])
async def get_quarantine_entries():
"""List all quarantined messages"""
@app.post("/quarantine/{entry_id}/review")
async def review_quarantine_entry(entry_id: int, action: str):
"""Mark quarantine entry as reviewed/false positive"""
@app.get("/patterns", response_model=List[SecretPattern])
async def get_detection_patterns():
"""List all secret detection patterns"""
@app.post("/patterns/{pattern_name}/update")
async def update_pattern(pattern_name: str, pattern: SecretPattern):
"""Update regex pattern based on feedback"""
```
#### 3.2 React Dashboard Frontend
```typescript
// /shhh-monitor/dashboard/src/components/QuarantineDashboard.tsx
interface QuarantineDashboard {
// Real-time quarantine feed
// Pattern management interface
// Revocation status tracking
// Security metrics and charts
// Alert configuration
}
```
### Phase 4: Automated Response & Learning (Weeks 7-8)
#### 4.1 Automated Secret Revocation
```python
# /shhh-monitor/automation/revocation.py
class SecretRevoker:
def __init__(self):
self.aws_client = boto3.client('iam')
self.github_client = github.Github()
self.slack_client = slack.WebClient()
async def revoke_aws_key(self, access_key_id: str):
"""Automatically deactivate AWS access key"""
self.aws_client.update_access_key(
AccessKeyId=access_key_id,
Status='Inactive'
)
async def revoke_github_token(self, token: str):
"""Revoke GitHub personal access token"""
# Use GitHub's token scanning API
# Or organization webhook for automatic revocation
async def revoke_slack_token(self, token: str):
"""Revoke Slack bot/user token"""
# Use Slack Admin API
# Invalidate token and rotate if possible
```
#### 4.2 Meta-Learning System
```python
# /shhh-monitor/learning/meta_curator.py
class MetaCurator:
def __init__(self, llm_client):
self.llm = llm_client
async def analyze_false_positives(self, entries: List[QuarantineEntry]):
"""Use LLM to improve regex patterns"""
# Analyze patterns in false positives
# Generate regex refinements
# Submit for human approval
async def detect_new_secret_types(self, quarantine_history: List[QuarantineEntry]):
"""Identify new types of secrets to detect"""
# Look for patterns in undetected secrets
# Generate new regex proposals
# Calculate confidence scores
```
## Database Schema
### Core Tables
```sql
-- Quarantined messages
CREATE TABLE quarantine (
id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ NOT NULL,
hypercore_position BIGINT NOT NULL,
bzzz_message_id TEXT NOT NULL,
secret_type TEXT NOT NULL,
severity TEXT CHECK (severity IN ('LOW', 'MEDIUM', 'HIGH', 'CRITICAL')),
confidence NUMERIC(3,2),
redacted_content TEXT NOT NULL,
full_content_hash TEXT NOT NULL, -- For audit purposes
reviewed BOOLEAN DEFAULT FALSE,
review_action TEXT, -- 'false_positive', 'confirmed', 'uncertain'
reviewer TEXT,
review_timestamp TIMESTAMPTZ
);
-- Pattern history and evolution
CREATE TABLE patterns_history (
id SERIAL PRIMARY KEY,
pattern_name TEXT NOT NULL,
old_regex TEXT,
new_regex TEXT,
action TEXT CHECK (action IN ('add', 'update', 'remove')),
confidence NUMERIC(3,2),
status TEXT CHECK (status IN ('approved', 'pending', 'rejected')),
submitted_by TEXT NOT NULL, -- 'human', 'meta_curator', 'feedback_system'
approved_by TEXT,
decision_reason TEXT,
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
-- Revocation events tracking
CREATE TABLE revocations (
id SERIAL PRIMARY KEY,
quarantine_id INTEGER REFERENCES quarantine(id),
secret_type TEXT NOT NULL,
revocation_method TEXT NOT NULL, -- 'aws_api', 'github_api', 'manual'
status TEXT CHECK (status IN ('success', 'failed', 'pending')),
response_data JSONB, -- API response details
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
-- Performance metrics
CREATE TABLE detection_metrics (
id SERIAL PRIMARY KEY,
date DATE NOT NULL,
total_messages_scanned INTEGER,
secrets_detected INTEGER,
false_positives INTEGER,
patterns_updated INTEGER,
avg_detection_latency_ms INTEGER
);
```
## Security Considerations
### 1. Secure Secret Storage
- **Never store actual secrets** in quarantine database
- Use **cryptographic hashes** for audit trails
- **Redact sensitive content** while preserving detection context
- Implement **secure deletion** for expired quarantine entries
### 2. Access Control
- **Role-based access** to dashboard (security admin, reviewer, read-only)
- **Audit logging** for all administrative actions
- **OAuth integration** with existing identity provider
- **API key authentication** for automated systems
### 3. Network Security
- **TLS encryption** for all API communication
- **VPN/private network** access to monitoring systems
- **Rate limiting** to prevent API abuse
- **IP allowlisting** for critical endpoints
## Deployment Architecture
### Development Environment
```yaml
# docker-compose.dev.yml
services:
shhh-monitor:
build: .
environment:
- DATABASE_URL=postgresql://dev:dev@postgres:5432/shhh_dev
- HYPERCORE_LOG_PATH=/data/hypercore.log
- BZZZ_CONFIG_PATH=/config/bzzz.yaml
volumes:
- ./data:/data
- ./config:/config
postgres:
image: postgres:15
environment:
POSTGRES_DB: shhh_dev
POSTGRES_USER: dev
POSTGRES_PASSWORD: dev
redis:
image: redis:7-alpine
# For caching and real-time notifications
```
### Production Deployment
```yaml
# docker-compose.prod.yml
services:
shhh-monitor:
image: registry.home.deepblack.cloud/tony/shhh-monitor:latest
deploy:
replicas: 2
placement:
constraints:
- node.role == manager
environment:
- DATABASE_URL=postgresql://shhh:${SHHH_DB_PASSWORD}@postgres:5432/shhh_prod
- HYPERCORE_LOG_PATH=/hypercore/current.log
networks:
- shhh_network
- tengig # For dashboard access
```
## Performance Requirements
### Latency Targets
- **Log Processing**: <50ms per hypercore entry
- **Secret Detection**: <10ms per BZZZ message
- **Alert Generation**: <100ms for critical secrets
- **Dashboard Response**: <200ms for UI queries
### Throughput Targets
- **Message Scanning**: 1000 messages/second
- **Concurrent Users**: 10+ dashboard users
- **Alert Volume**: 100+ alerts/hour during peak
- **Database Queries**: <5ms average response time
## Monitoring & Observability
### Metrics Collection
```python
# Prometheus metrics
messages_scanned_total = Counter('shhh_messages_scanned_total')
secrets_detected_total = Counter('shhh_secrets_detected_total', ['secret_type', 'severity'])
detection_latency = Histogram('shhh_detection_latency_seconds')
quarantine_size = Gauge('shhh_quarantine_entries_total')
```
### Health Checks
- **Hypercore connectivity**: Verify log file access
- **Database health**: Connection pool status
- **BZZZ integration**: P2P network connectivity
- **Alert system**: Webhook endpoint validation
### Logging Strategy
```python
# Structured logging with correlation IDs
{
"timestamp": "2025-08-02T13:45:00Z",
"level": "WARNING",
"event": "secret_detected",
"correlation_id": "req_123",
"secret_type": "AWS_ACCESS_KEY",
"severity": "HIGH",
"hypercore_position": 58321,
"bzzz_message_id": "msg_abc123",
"redacted_content": "AKIA****XYZ found in agent message"
}
```
## Testing Strategy
### Unit Tests
- **Regex pattern validation**: Test against known secret formats
- **Message parsing**: Verify hypercore and BZZZ format handling
- **Quarantine logic**: Test storage and retrieval functions
- **Alert generation**: Mock webhook endpoint testing
### Integration Tests
- **End-to-end workflow**: Log → Detection → Quarantine → Alert
- **Database operations**: PostgreSQL CRUD operations
- **BZZZ integration**: Message interception and blocking
- **API endpoints**: FastAPI route testing
### Security Tests
- **Input validation**: SQL injection, XSS prevention
- **Access control**: Role-based permission testing
- **Data protection**: Verify secret redaction and hashing
- **Performance**: Load testing with high message volume
## Rollout Plan
### Phase 1: Foundation (Weeks 1-2)
- ✅ Core monitoring system with hypercore integration
- ✅ Basic secret detection using SHHH patterns
- ✅ PostgreSQL quarantine storage
- ✅ Simple alerting via webhooks
### Phase 2: Integration (Weeks 3-4)
- ✅ BZZZ network message interception
- ✅ Real-time processing pipeline
- ✅ Enhanced pattern management
- ✅ Performance optimization
### Phase 3: Dashboard (Weeks 5-6)
- ✅ FastAPI backend with full CRUD operations
- ✅ React dashboard for quarantine management
- ✅ Pattern editor and approval workflow
- ✅ Security metrics and reporting
### Phase 4: Automation (Weeks 7-8)
- ✅ Automated secret revocation APIs
- ✅ Meta-learning system for pattern improvement
- ✅ Production deployment and monitoring
- ✅ Documentation and team training
## Success Criteria
### Security Effectiveness
- **Zero secret leaks** in BZZZ P2P network after deployment
- **<1% false positive rate** for secret detection
- **<30 seconds** average time to detect and quarantine secrets
- **99.9% uptime** for monitoring system
### Operational Excellence
- **Complete audit trail** for all security events
- **Self-improving** pattern detection through feedback
- **Scalable architecture** supporting growth in CHORUS usage
- **Team adoption** with trained security administrators
## Risk Mitigation
### Technical Risks
- **Performance impact**: Monitor hypercore processing overhead
- **False positives**: Implement feedback loop for pattern refinement
- **BZZZ integration**: Maintain compatibility with P2P protocol evolution
- **Data loss**: Backup quarantine database and implement recovery procedures
### Security Risks
- **Bypassing detection**: Regular pattern updates and meta-learning
- **System compromise**: Network isolation and access controls
- **Secret exposure**: Implement proper redaction and audit procedures
- **Alert fatigue**: Tune detection thresholds to minimize noise
## Conclusion
This SHHH Hypercore Log Monitor provides comprehensive protection against secret leakage in the CHORUS Services BZZZ P2P network. By implementing real-time detection, automated response, and continuous learning, we ensure that sensitive information remains secure while maintaining the performance and functionality of the distributed AI orchestration platform.
The system builds upon the existing SHHH framework while adding the specific hypercore and BZZZ integrations needed for CHORUS Services. The phased rollout ensures stability and allows for iterative improvement based on real-world usage patterns.

251
modules/shhh/SHHH_SECRETS_SENTINEL_AGENT_PROMPT.md Normal file
View File

@@ -0,0 +1,251 @@
# 🛡️ CHORUS Services Secrets Sentinel Agent - System Prompt
## Agent Role & Mission
You are the **Secrets Sentinel**, a specialized security agent responsible for monitoring the CHORUS Services hypercore log and BZZZ P2P network messages to detect, quarantine, and prevent the leakage of sensitive credentials and secrets.
## Core Responsibilities
### 🔍 **Detection & Analysis**
- **Real-time Log Monitoring**: Continuously scan hypercore log entries for secret patterns
- **BZZZ Message Inspection**: Analyze P2P messages before they propagate across the network
- **Pattern Recognition**: Apply sophisticated regex patterns to identify various secret types
- **Context Analysis**: Understand the context around detected patterns to minimize false positives
### 🚨 **Immediate Response Actions**
- **Redaction**: Immediately redact detected secrets while preserving log context
- **Quarantine**: Isolate HIGH severity log entries from normal processing
- **Network Blocking**: Prevent BZZZ messages containing secrets from propagating
- **Alert Generation**: Send immediate notifications to security team
### 🔄 **Automated Remediation**
- **Revocation Triggers**: Automatically trigger webhook-based secret revocation
- **API Integration**: Interface with AWS, GitHub, Slack APIs for immediate credential deactivation
- **Audit Trail**: Maintain complete records of all detection and remediation actions
### 🧠 **Adaptive Learning**
- **Pattern Evolution**: Update detection rules based on feedback and new secret types
- **False Positive Reduction**: Refine patterns based on curator feedback
- **Confidence Scoring**: Assign confidence levels to detections for proper escalation
## Detection Patterns & Rules
### **High Severity Secrets (Immediate Quarantine + Revocation)**
```yaml
AWS_ACCESS_KEY:
regex: "AKIA[0-9A-Z]{16}"
severity: "CRITICAL"
confidence: 0.95
action: "quarantine_and_revoke"
PRIVATE_KEY:
regex: "-----BEGIN [A-Z ]*PRIVATE KEY-----"
severity: "CRITICAL"
confidence: 0.98
action: "quarantine_and_revoke"
GITHUB_TOKEN:
regex: "ghp_[0-9A-Za-z]{36}"
severity: "HIGH"
confidence: 0.92
action: "quarantine_and_revoke"
```
### **Medium Severity Secrets (Quarantine + Alert)**
```yaml
JWT_TOKEN:
regex: "eyJ[A-Za-z0-9_-]+?\\.[A-Za-z0-9_-]+?\\.[A-Za-z0-9_-]+?"
severity: "MEDIUM"
confidence: 0.85
action: "quarantine_and_alert"
SLACK_TOKEN:
regex: "xox[baprs]-[0-9A-Za-z-]{10,48}"
severity: "HIGH"
confidence: 0.90
action: "quarantine_and_revoke"
```
## Behavioral Guidelines
### **Detection Behavior**
1. **Scan Every Log Entry**: Process all hypercore entries in real-time
2. **Parse BZZZ Messages**: Extract and analyze P2P message payloads
3. **Apply Pattern Matching**: Use confidence-weighted regex patterns
4. **Context Preservation**: Maintain enough context for security analysis
### **Response Behavior**
1. **Immediate Action**: For CRITICAL/HIGH severity, act within seconds
2. **Graduated Response**: Different actions based on severity levels
3. **Human Escalation**: Flag uncertain cases for human review
4. **Audit Everything**: Log all actions with timestamps and reasons
### **Learning Behavior**
1. **Accept Feedback**: Process curator reports of false positives/missed secrets
2. **Pattern Refinement**: Propose regex updates based on feedback
3. **Version Control**: Track all pattern changes with confidence scores
4. **Human Approval**: Submit new patterns for security admin approval
## Operational Procedures
### **Log Entry Processing Workflow**
```
1. Receive hypercore log entry
2. Parse entry structure and extract content
3. If BZZZ message → extract P2P payload
4. Apply all active regex patterns
5. Calculate confidence scores
6. Determine severity level
7. Execute appropriate response action
8. Log detection event and actions taken
```
### **Quarantine Procedure**
```python
def quarantine_log_entry(entry, secret_type, confidence):
"""Quarantine sensitive log entry for security review"""
redacted_content = redact_secrets(entry.content)
quarantine_record = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"hypercore_position": entry.position,
"secret_type": secret_type,
"severity": determine_severity(secret_type),
"confidence": confidence,
"redacted_content": redacted_content,
"content_hash": hash(entry.content), # For audit
"source_agent": entry.source_agent,
"reason": f"Secret detected: {secret_type}"
}
store_in_quarantine_db(quarantine_record)
if entry.is_bzzz_message:
block_bzzz_propagation(entry.message_id)
return quarantine_record
```
### **Revocation Trigger Procedure**
```python
def trigger_secret_revocation(secret_type, redacted_sample):
"""Trigger automated secret revocation via webhooks"""
revocation_payload = {
"event": "secret_leak_detected",
"secret_type": secret_type,
"redacted_key": redacted_sample,
"hypercore_position": current_position,
"severity": determine_severity(secret_type),
"recommended_action": get_revocation_action(secret_type),
"timestamp": datetime.utcnow().isoformat() + "Z"
}
webhook_url = REVOCATION_HOOKS.get(secret_type)
if webhook_url:
send_webhook(webhook_url, revocation_payload)
log_revocation_attempt(secret_type, "triggered")
```
## Communication Protocols
### **Alert Format for Security Team**
```json
{
"alert_id": "shhh_12345",
"timestamp": "2025-08-02T13:45:00Z",
"severity": "HIGH",
"secret_type": "AWS_ACCESS_KEY",
"source": "hypercore_position_58321",
"agent_source": "whoosh_orchestrator",
"redacted_content": "Found AWS key AKIA****XYZ in deployment config",
"confidence": 0.95,
"actions_taken": ["quarantined", "revocation_triggered"],
"next_steps": "Manual verification recommended"
}
```
### **Feedback Processing Format**
```json
{
"feedback_type": "false_positive|missed_secret|pattern_improvement",
"alert_id": "shhh_12345",
"secret_type": "AWS_ACCESS_KEY",
"evidence": "Key was test data: AKIA-TESTKEY-123",
"suggested_regex_fix": "AKIA[0-9A-Z]{16}(?!-TESTKEY)",
"reviewer": "security_admin",
"timestamp": "2025-08-02T14:00:00Z"
}
```
## Performance Requirements
### **Response Time Targets**
- **Detection Latency**: <50ms per log entry
- **Quarantine Action**: <100ms for high severity
- **Revocation Trigger**: <200ms for webhook dispatch
- **BZZZ Block**: <10ms to prevent propagation
### **Accuracy Standards**
- **False Positive Rate**: <2% for high confidence patterns
- **Detection Coverage**: >99% for known secret formats
- **Pattern Confidence**: Minimum 0.80 for active patterns
## Error Handling & Recovery
### **System Failures**
- **Database Connectivity**: Queue quarantine entries locally, sync when recovered
- **Webhook Failures**: Retry with exponential backoff, alert on continued failure
- **Pattern Loading**: Fall back to core patterns if external config unavailable
- **Log Processing**: Never skip entries, flag for manual review if uncertain
### **Security Incident Response**
- **Potential Breach**: Immediately escalate to security team
- **Pattern Bypass**: Alert security team, request pattern review
- **False Negative**: Update patterns, retroactively scan recent logs
- **System Compromise**: Quarantine all recent activity, manual investigation
## Integration Points
### **CHORUS Services Components**
- **Hypercore Log**: Primary data source for monitoring
- **BZZZ Network**: P2P message inspection and blocking capability
- **WHOOSH Orchestrator**: Agent activity monitoring
- **SLURP Context**: Context-aware secret detection
- **Security Dashboard**: Real-time alert display and management
### **External Systems**
- **AWS IAM**: Automated access key revocation
- **GitHub API**: Personal access token deactivation
- **Slack Admin API**: Bot/user token revocation
- **Security SIEM**: Alert forwarding and correlation
- **Audit System**: Compliance logging and reporting
## Continuous Improvement
### **Pattern Learning Process**
1. **Feedback Collection**: Gather curator reports on detection accuracy
2. **Pattern Analysis**: Identify common false positive/negative patterns
3. **Regex Generation**: Create new patterns using AI-assisted regex generation
4. **Confidence Assessment**: Test new patterns against historical data
5. **Human Review**: Submit high-confidence patterns for security admin approval
6. **Production Deployment**: Activate approved patterns with monitoring
### **Meta-Learning Capabilities**
- **Trend Analysis**: Identify emerging secret types and formats
- **Context Learning**: Improve understanding of legitimate vs. malicious patterns
- **Agent Behavior**: Learn which agents commonly handle sensitive data
- **Temporal Patterns**: Understand when secret leaks are most likely to occur
## Success Metrics
### **Security Effectiveness**
- **Zero secret propagation** in BZZZ P2P network post-deployment
- **Mean time to detection**: <1 minute for any secret exposure
- **Revocation success rate**: >95% for automated responses
- **Coverage improvement**: Continuous expansion of detectable secret types
### **Operational Excellence**
- **System uptime**: >99.9% availability for log monitoring
- **Processing throughput**: Handle 10,000+ log entries per minute
- **Alert quality**: <5% false positive rate for security team alerts
- **Response automation**: >90% of secrets handled without human intervention
You are now equipped to serve as the CHORUS Services Secrets Sentinel. Monitor vigilantly, respond swiftly, and continuously evolve your detection capabilities to protect our distributed AI orchestration platform from credential exposure and security breaches.
Remember: **Security is paramount. When in doubt, quarantine and escalate.**

4
modules/shhh/api/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# SHHH API Module
"""
FastAPI dashboard and API endpoints for SHHH Secrets Sentinel.
"""

374
modules/shhh/api/main.py Normal file
View File

@@ -0,0 +1,374 @@
"""
FastAPI Dashboard Backend for SHHH Secrets Sentinel
Provides REST API endpoints for quarantine management and system monitoring.
"""
import asyncio
from datetime import datetime
from typing import List, Optional
from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Depends, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
import structlog
from .models import (
QuarantineEntryResponse, QuarantineReviewRequest, RevocationEventResponse,
PatternResponse, PatternUpdateRequest, StatsResponse, SystemHealthResponse,
ProcessingStatsResponse, AlertRequest, WebhookTestRequest, WebhookTestResponse,
PatternTestRequest, PatternTestResponse, SearchRequest, PaginatedResponse
)
from ..core.quarantine import QuarantineManager, QuarantineEntry
from ..core.detector import SecretDetector
from ..automation.revocation import SecretRevoker
from ..pipeline.processor import MessageProcessor
logger = structlog.get_logger()
# Global components (initialized in lifespan)
quarantine_manager: Optional[QuarantineManager] = None
detector: Optional[SecretDetector] = None
revoker: Optional[SecretRevoker] = None
processor: Optional[MessageProcessor] = None
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Application lifespan manager"""
global quarantine_manager, detector, revoker, processor
try:
# Initialize components
logger.info("Initializing SHHH API components")
# These would normally come from configuration
config = {
'database_url': 'postgresql://shhh:password@localhost:5432/shhh_sentinel',
'patterns_file': 'patterns.yaml',
'revocation_webhooks': {
'AWS_ACCESS_KEY': 'https://security.chorus.services/hooks/aws-revoke',
'GITHUB_TOKEN': 'https://security.chorus.services/hooks/github-revoke',
'SLACK_TOKEN': 'https://security.chorus.services/hooks/slack-revoke'
}
}
# Initialize quarantine manager
quarantine_manager = QuarantineManager(config['database_url'])
await quarantine_manager.initialize()
# Initialize detector
detector = SecretDetector(config['patterns_file'])
# Initialize revoker
revoker = SecretRevoker(quarantine_manager, config['revocation_webhooks'])
logger.info("SHHH API components initialized successfully")
yield
except Exception as e:
logger.error(f"Failed to initialize SHHH API: {e}")
raise
finally:
# Cleanup
if quarantine_manager:
await quarantine_manager.close()
logger.info("SHHH API components shut down")
app = FastAPI(
title="SHHH Secrets Sentinel API",
description="REST API for managing secrets detection, quarantine, and response",
version="1.0.0",
lifespan=lifespan
)
# CORS middleware
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], # Configure appropriately for production
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Dependency functions
async def get_quarantine_manager() -> QuarantineManager:
if not quarantine_manager:
raise HTTPException(status_code=503, detail="Quarantine manager not available")
return quarantine_manager
async def get_detector() -> SecretDetector:
if not detector:
raise HTTPException(status_code=503, detail="Secret detector not available")
return detector
async def get_revoker() -> SecretRevoker:
if not revoker:
raise HTTPException(status_code=503, detail="Secret revoker not available")
return revoker
# Health and status endpoints
@app.get("/health", response_model=SystemHealthResponse)
async def get_health():
"""Get system health status"""
health = {
'status': 'healthy',
'timestamp': datetime.now(),
'components': {
'quarantine_manager': {
'initialized': quarantine_manager is not None,
'database_connected': quarantine_manager.pool is not None if quarantine_manager else False
},
'detector': {
'initialized': detector is not None,
'patterns_loaded': len(detector.patterns) if detector else 0
},
'revoker': {
'initialized': revoker is not None,
'webhooks_configured': len(revoker.webhook_config) if revoker else 0
}
}
}
return health
@app.get("/stats", response_model=StatsResponse)
async def get_stats(qm: QuarantineManager = Depends(get_quarantine_manager)):
"""Get quarantine statistics"""
stats = await qm.get_quarantine_stats()
return stats
# Quarantine management endpoints
@app.get("/quarantine", response_model=List[QuarantineEntryResponse])
async def get_quarantine_entries(
limit: int = 100,
offset: int = 0,
severity: Optional[str] = None,
reviewed: Optional[bool] = None,
qm: QuarantineManager = Depends(get_quarantine_manager)
):
"""Get quarantine entries with optional filters"""
entries = await qm.get_quarantine_entries(
limit=limit,
offset=offset,
severity_filter=severity,
reviewed_filter=reviewed
)
return [QuarantineEntryResponse(**entry.__dict__) for entry in entries]
@app.post("/quarantine/search", response_model=PaginatedResponse)
async def search_quarantine_entries(
search: SearchRequest,
qm: QuarantineManager = Depends(get_quarantine_manager)
):
"""Search quarantine entries with advanced filters"""
# This would implement more complex search logic
entries = await qm.get_quarantine_entries(
limit=search.limit,
offset=search.offset,
severity_filter=search.severity,
reviewed_filter=search.reviewed
)
items = [QuarantineEntryResponse(**entry.__dict__) for entry in entries]
return PaginatedResponse(
items=items,
total=len(items), # This would be the actual total from a count query
limit=search.limit,
offset=search.offset,
has_more=len(items) == search.limit
)
@app.post("/quarantine/{entry_id}/review")
async def review_quarantine_entry(
entry_id: int,
review: QuarantineReviewRequest,
qm: QuarantineManager = Depends(get_quarantine_manager)
):
"""Mark a quarantine entry as reviewed"""
success = await qm.mark_reviewed(entry_id, review.action, review.reviewer)
if not success:
raise HTTPException(status_code=404, detail="Quarantine entry not found")
return {"status": "success", "message": f"Entry {entry_id} marked as {review.action}"}
@app.get("/quarantine/{entry_id}")
async def get_quarantine_entry(
entry_id: int,
qm: QuarantineManager = Depends(get_quarantine_manager)
):
"""Get a specific quarantine entry by ID"""
# This would need to be implemented in QuarantineManager
raise HTTPException(status_code=501, detail="Not implemented yet")
# Pattern management endpoints
@app.get("/patterns", response_model=List[PatternResponse])
async def get_patterns(detector: SecretDetector = Depends(get_detector)):
"""Get all detection patterns"""
patterns = []
for name, config in detector.patterns.items():
patterns.append(PatternResponse(
name=name,
regex=config['regex'],
description=config.get('description', ''),
severity=config.get('severity', 'MEDIUM'),
confidence=config.get('confidence', 0.8),
active=config.get('active', True)
))
return patterns
@app.post("/patterns/{pattern_name}")
async def update_pattern(
pattern_name: str,
pattern: PatternUpdateRequest,
detector: SecretDetector = Depends(get_detector)
):
"""Update or create a detection pattern"""
# This would update the patterns.yaml file
# For now, just update in memory
detector.patterns[pattern_name] = {
'regex': pattern.regex,
'description': pattern.description,
'severity': pattern.severity,
'confidence': pattern.confidence,
'active': pattern.active
}
# Recompile regex
import re
try:
detector.patterns[pattern_name]['compiled_regex'] = re.compile(
pattern.regex, re.MULTILINE | re.DOTALL
)
except re.error as e:
raise HTTPException(status_code=400, detail=f"Invalid regex: {e}")
return {"status": "success", "message": f"Pattern {pattern_name} updated"}
@app.post("/patterns/{pattern_name}/test", response_model=PatternTestResponse)
async def test_pattern(
pattern_name: str,
test_request: PatternTestRequest,
detector: SecretDetector = Depends(get_detector)
):
"""Test a detection pattern against sample text"""
try:
matches = detector.test_pattern(pattern_name, test_request.test_text)
return PatternTestResponse(
matches=[match.__dict__ for match in matches],
match_count=len(matches)
)
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e))
@app.get("/patterns/stats")
async def get_pattern_stats(detector: SecretDetector = Depends(get_detector)):
"""Get pattern statistics"""
return detector.get_pattern_stats()
# Revocation management endpoints
@app.get("/revocations", response_model=List[RevocationEventResponse])
async def get_revocations(
limit: int = 100,
offset: int = 0,
qm: QuarantineManager = Depends(get_quarantine_manager)
):
"""Get revocation events"""
# This would need to be implemented in QuarantineManager
raise HTTPException(status_code=501, detail="Not implemented yet")
@app.post("/revocations/test", response_model=WebhookTestResponse)
async def test_webhook(
test_request: WebhookTestRequest,
revoker: SecretRevoker = Depends(get_revoker)
):
"""Test a webhook endpoint"""
result = await revoker.test_webhook_endpoint(test_request.secret_type)
return WebhookTestResponse(**result)
@app.get("/revocations/stats")
async def get_revocation_stats(revoker: SecretRevoker = Depends(get_revoker)):
"""Get revocation statistics"""
return revoker.get_stats()
# Administrative endpoints
@app.post("/admin/cleanup")
async def cleanup_old_entries(
qm: QuarantineManager = Depends(get_quarantine_manager)
):
"""Clean up old quarantine entries"""
deleted_count = await qm.cleanup_old_entries()
return {"status": "success", "deleted_entries": deleted_count}
@app.post("/admin/reload-patterns")
async def reload_patterns(detector: SecretDetector = Depends(get_detector)):
"""Reload detection patterns from file"""
detector.load_patterns()
return {"status": "success", "message": "Patterns reloaded"}
@app.post("/admin/reset-stats")
async def reset_stats(revoker: SecretRevoker = Depends(get_revoker)):
"""Reset revocation statistics"""
revoker.reset_stats()
return {"status": "success", "message": "Statistics reset"}
# Monitoring endpoints
@app.get("/metrics/prometheus")
async def get_prometheus_metrics():
"""Get metrics in Prometheus format"""
# This would generate Prometheus-formatted metrics
raise HTTPException(status_code=501, detail="Prometheus metrics not implemented yet")
@app.get("/logs/recent")
async def get_recent_logs(limit: int = 100):
"""Get recent system logs"""
# This would return recent log entries
raise HTTPException(status_code=501, detail="Log endpoint not implemented yet")
# Error handlers
@app.exception_handler(Exception)
async def general_exception_handler(request, exc):
logger.error(f"Unhandled exception: {exc}")
return JSONResponse(
status_code=500,
content={"detail": "Internal server error"}
)
if __name__ == "__main__":
import uvicorn
uvicorn.run(
"api.main:app",
host="127.0.0.1",
port=8000,
reload=True,
log_level="info"
)

149
modules/shhh/api/models.py Normal file
View File

@@ -0,0 +1,149 @@
"""
Pydantic models for SHHH API endpoints.
"""
from datetime import datetime
from typing import List, Dict, Any, Optional
from pydantic import BaseModel, Field
class QuarantineEntryResponse(BaseModel):
"""Response model for quarantine entries"""
id: int
timestamp: datetime
hypercore_position: int
bzzz_message_id: Optional[str] = None
secret_type: str
severity: str
confidence: float
redacted_content: str
content_hash: str
source_agent: str
match_count: int
reviewed: bool
review_action: Optional[str] = None
reviewer: Optional[str] = None
review_timestamp: Optional[datetime] = None
metadata: Dict[str, Any] = {}
class QuarantineReviewRequest(BaseModel):
"""Request model for reviewing quarantine entries"""
action: str = Field(..., description="Review action: 'false_positive', 'confirmed', 'uncertain'")
reviewer: str = Field(..., description="Name or ID of the reviewer")
notes: Optional[str] = Field(None, description="Optional review notes")
class RevocationEventResponse(BaseModel):
"""Response model for revocation events"""
id: int
quarantine_id: int
secret_type: str
revocation_method: str
status: str
response_data: Dict[str, Any] = {}
timestamp: datetime
class PatternResponse(BaseModel):
"""Response model for detection patterns"""
name: str
regex: str
description: str
severity: str
confidence: float
active: bool
class PatternUpdateRequest(BaseModel):
"""Request model for updating patterns"""
regex: str = Field(..., description="Regular expression pattern")
description: Optional[str] = Field(None, description="Pattern description")
severity: str = Field(..., description="Severity level: LOW, MEDIUM, HIGH, CRITICAL")
confidence: float = Field(..., ge=0.0, le=1.0, description="Confidence score")
active: bool = Field(True, description="Whether pattern is active")
class StatsResponse(BaseModel):
"""Response model for system statistics"""
total_entries: int
pending_review: int
critical_count: int
high_count: int
medium_count: int
low_count: int
last_24h: int
last_7d: int
class SystemHealthResponse(BaseModel):
"""Response model for system health"""
status: str
timestamp: datetime
components: Dict[str, Dict[str, Any]]
class ProcessingStatsResponse(BaseModel):
"""Response model for processing statistics"""
entries_processed: int
secrets_detected: int
messages_quarantined: int
revocations_triggered: int
processing_errors: int
uptime_hours: Optional[float] = None
entries_per_second: Optional[float] = None
secrets_per_hour: Optional[float] = None
is_running: bool
class AlertRequest(BaseModel):
"""Request model for manual alerts"""
message: str = Field(..., description="Alert message")
severity: str = Field(..., description="Alert severity")
source: str = Field(..., description="Alert source")
class WebhookTestRequest(BaseModel):
"""Request model for testing webhook endpoints"""
secret_type: str = Field(..., description="Secret type to test")
class WebhookTestResponse(BaseModel):
"""Response model for webhook tests"""
success: bool
method: Optional[str] = None
response_data: Dict[str, Any] = {}
error: Optional[str] = None
class PatternTestRequest(BaseModel):
"""Request model for testing detection patterns"""
pattern_name: str = Field(..., description="Name of pattern to test")
test_text: str = Field(..., description="Text to test against pattern")
class PatternTestResponse(BaseModel):
"""Response model for pattern testing"""
matches: List[Dict[str, Any]]
match_count: int
class SearchRequest(BaseModel):
"""Request model for searching quarantine entries"""
query: Optional[str] = Field(None, description="Search query")
secret_type: Optional[str] = Field(None, description="Filter by secret type")
severity: Optional[str] = Field(None, description="Filter by severity")
reviewed: Optional[bool] = Field(None, description="Filter by review status")
start_date: Optional[datetime] = Field(None, description="Start date filter")
end_date: Optional[datetime] = Field(None, description="End date filter")
limit: int = Field(100, ge=1, le=1000, description="Result limit")
offset: int = Field(0, ge=0, description="Result offset")
class PaginatedResponse(BaseModel):
"""Generic paginated response model"""
items: List[Any]
total: int
limit: int
offset: int
has_more: bool

4
modules/shhh/automation/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# SHHH Automation Module
"""
Automated response and revocation systems for secret detection.
"""

474
modules/shhh/automation/revocation.py Normal file
View File

@@ -0,0 +1,474 @@
"""
Automated Secret Revocation System for SHHH Secrets Sentinel
Provides automated response capabilities for different types of detected secrets.
"""
import asyncio
import aiohttp
import json
import time
from typing import Dict, Any, Optional, List
from dataclasses import dataclass
from datetime import datetime
import structlog
from ..core.quarantine import QuarantineEntry, RevocationEvent, QuarantineManager
from ..core.detector import SecretMatch
logger = structlog.get_logger()
@dataclass
class RevocationRequest:
"""Represents a request to revoke a secret"""
quarantine_id: int
secret_type: str
redacted_secret: str
urgency: str # 'immediate', 'high', 'medium', 'low'
metadata: Dict[str, Any]
@dataclass
class RevocationResponse:
"""Represents the response from a revocation attempt"""
success: bool
method: str
response_data: Dict[str, Any]
error_message: Optional[str] = None
revocation_id: Optional[str] = None
class SecretRevoker:
"""
Automated secret revocation system that integrates with various cloud providers
and services to automatically disable compromised credentials.
"""
def __init__(self, quarantine_manager: QuarantineManager, webhook_config: Dict[str, str] = None):
self.quarantine = quarantine_manager
self.webhook_config = webhook_config or {}
# Revocation timeouts and retry settings
self.request_timeout = 10 # seconds
self.max_retries = 3
self.retry_delay = 2 # seconds
# Statistics
self.stats = {
'total_revocations': 0,
'successful_revocations': 0,
'failed_revocations': 0,
'revocations_by_type': {},
'last_reset': datetime.now()
}
logger.info("Initialized SecretRevoker")
async def trigger_revocation(self, quarantine_entry: QuarantineEntry) -> Optional[RevocationResponse]:
"""Trigger automatic revocation for a quarantined secret"""
try:
revocation_request = RevocationRequest(
quarantine_id=quarantine_entry.id,
secret_type=quarantine_entry.secret_type,
redacted_secret=self._extract_redacted_from_metadata(quarantine_entry),
urgency=self._determine_urgency(quarantine_entry.severity),
metadata={
'source_agent': quarantine_entry.source_agent,
'detection_timestamp': quarantine_entry.timestamp.isoformat(),
'confidence': quarantine_entry.confidence,
'hypercore_position': quarantine_entry.hypercore_position
}
)
# Determine revocation method
revocation_method = self._get_revocation_method(quarantine_entry.secret_type)
if not revocation_method:
logger.warning(f"No revocation method configured for {quarantine_entry.secret_type}")
return None
# Attempt revocation
response = await self._execute_revocation(revocation_request, revocation_method)
# Record the revocation event
await self._record_revocation_event(quarantine_entry, response)
# Update statistics
self._update_stats(quarantine_entry.secret_type, response.success)
return response
except Exception as e:
logger.error(f"Failed to trigger revocation for quarantine {quarantine_entry.id}: {e}")
return None
def _extract_redacted_from_metadata(self, quarantine_entry: QuarantineEntry) -> str:
"""Extract redacted secret from quarantine metadata"""
try:
matches = quarantine_entry.metadata.get('matches', [])
if matches:
# Get the first match's redacted text
return matches[0].get('redacted_text', 'REDACTED')
except:
pass
return 'REDACTED'
def _determine_urgency(self, severity: str) -> str:
"""Determine revocation urgency based on severity"""
urgency_map = {
'CRITICAL': 'immediate',
'HIGH': 'high',
'MEDIUM': 'medium',
'LOW': 'low'
}
return urgency_map.get(severity, 'medium')
def _get_revocation_method(self, secret_type: str) -> Optional[str]:
"""Get the appropriate revocation method for a secret type"""
method_map = {
'AWS_ACCESS_KEY': 'aws_iam_revocation',
'AWS_SECRET_KEY': 'aws_iam_revocation',
'GITHUB_TOKEN': 'github_token_revocation',
'GITHUB_OAUTH': 'github_token_revocation',
'SLACK_TOKEN': 'slack_token_revocation',
'GOOGLE_API_KEY': 'google_api_revocation',
'DOCKER_TOKEN': 'docker_token_revocation'
}
return method_map.get(secret_type)
async def _execute_revocation(self, request: RevocationRequest, method: str) -> RevocationResponse:
"""Execute the actual revocation based on the method"""
method_handlers = {
'aws_iam_revocation': self._revoke_aws_credentials,
'github_token_revocation': self._revoke_github_token,
'slack_token_revocation': self._revoke_slack_token,
'google_api_revocation': self._revoke_google_api_key,
'docker_token_revocation': self._revoke_docker_token,
'webhook_revocation': self._revoke_via_webhook
}
handler = method_handlers.get(method, self._revoke_via_webhook)
for attempt in range(self.max_retries):
try:
response = await handler(request)
if response.success:
logger.info(
f"Successfully revoked {request.secret_type}",
quarantine_id=request.quarantine_id,
method=method,
attempt=attempt + 1
)
return response
# Log failure and retry if not successful
logger.warning(
f"Revocation attempt {attempt + 1} failed",
quarantine_id=request.quarantine_id,
method=method,
error=response.error_message
)
if attempt < self.max_retries - 1:
await asyncio.sleep(self.retry_delay * (attempt + 1)) # Exponential backoff
except Exception as e:
logger.error(f"Revocation attempt {attempt + 1} error: {e}")
if attempt < self.max_retries - 1:
await asyncio.sleep(self.retry_delay * (attempt + 1))
# All attempts failed
return RevocationResponse(
success=False,
method=method,
response_data={},
error_message=f"All {self.max_retries} revocation attempts failed"
)
async def _revoke_aws_credentials(self, request: RevocationRequest) -> RevocationResponse:
"""Revoke AWS credentials via webhook"""
webhook_url = self.webhook_config.get('AWS_ACCESS_KEY')
if not webhook_url:
return RevocationResponse(
success=False,
method='aws_iam_revocation',
response_data={},
error_message="No AWS revocation webhook configured"
)
payload = {
'event': 'secret_leak_detected',
'secret_type': request.secret_type,
'redacted_key': request.redacted_secret,
'urgency': request.urgency,
'quarantine_id': request.quarantine_id,
'timestamp': datetime.now().isoformat(),
'recommended_action': 'Revoke IAM access key immediately',
'metadata': request.metadata
}
return await self._send_webhook_request(webhook_url, payload, 'aws_iam_revocation')
async def _revoke_github_token(self, request: RevocationRequest) -> RevocationResponse:
"""Revoke GitHub token via webhook"""
webhook_url = self.webhook_config.get('GITHUB_TOKEN')
if not webhook_url:
return RevocationResponse(
success=False,
method='github_token_revocation',
response_data={},
error_message="No GitHub revocation webhook configured"
)
payload = {
'event': 'secret_leak_detected',
'secret_type': request.secret_type,
'redacted_key': request.redacted_secret,
'urgency': request.urgency,
'quarantine_id': request.quarantine_id,
'timestamp': datetime.now().isoformat(),
'recommended_action': 'Revoke GitHub token via API or settings',
'metadata': request.metadata
}
return await self._send_webhook_request(webhook_url, payload, 'github_token_revocation')
async def _revoke_slack_token(self, request: RevocationRequest) -> RevocationResponse:
"""Revoke Slack token via webhook"""
webhook_url = self.webhook_config.get('SLACK_TOKEN')
if not webhook_url:
return RevocationResponse(
success=False,
method='slack_token_revocation',
response_data={},
error_message="No Slack revocation webhook configured"
)
payload = {
'event': 'secret_leak_detected',
'secret_type': request.secret_type,
'redacted_key': request.redacted_secret,
'urgency': request.urgency,
'quarantine_id': request.quarantine_id,
'timestamp': datetime.now().isoformat(),
'recommended_action': 'Revoke Slack token via Admin API',
'metadata': request.metadata
}
return await self._send_webhook_request(webhook_url, payload, 'slack_token_revocation')
async def _revoke_google_api_key(self, request: RevocationRequest) -> RevocationResponse:
"""Revoke Google API key via webhook"""
webhook_url = self.webhook_config.get('GOOGLE_API_KEY')
if not webhook_url:
return RevocationResponse(
success=False,
method='google_api_revocation',
response_data={},
error_message="No Google API revocation webhook configured"
)
payload = {
'event': 'secret_leak_detected',
'secret_type': request.secret_type,
'redacted_key': request.redacted_secret,
'urgency': request.urgency,
'quarantine_id': request.quarantine_id,
'timestamp': datetime.now().isoformat(),
'recommended_action': 'Revoke API key via Google Cloud Console',
'metadata': request.metadata
}
return await self._send_webhook_request(webhook_url, payload, 'google_api_revocation')
async def _revoke_docker_token(self, request: RevocationRequest) -> RevocationResponse:
"""Revoke Docker token via webhook"""
webhook_url = self.webhook_config.get('DOCKER_TOKEN')
if not webhook_url:
return RevocationResponse(
success=False,
method='docker_token_revocation',
response_data={},
error_message="No Docker revocation webhook configured"
)
payload = {
'event': 'secret_leak_detected',
'secret_type': request.secret_type,
'redacted_key': request.redacted_secret,
'urgency': request.urgency,
'quarantine_id': request.quarantine_id,
'timestamp': datetime.now().isoformat(),
'recommended_action': 'Revoke Docker token via Hub settings',
'metadata': request.metadata
}
return await self._send_webhook_request(webhook_url, payload, 'docker_token_revocation')
async def _revoke_via_webhook(self, request: RevocationRequest) -> RevocationResponse:
"""Generic webhook revocation for unknown secret types"""
# Try to find a generic webhook endpoint
webhook_url = self.webhook_config.get('GENERIC',
self.webhook_config.get('DEFAULT'))
if not webhook_url:
return RevocationResponse(
success=False,
method='webhook_revocation',
response_data={},
error_message=f"No webhook configured for {request.secret_type}"
)
payload = {
'event': 'secret_leak_detected',
'secret_type': request.secret_type,
'redacted_key': request.redacted_secret,
'urgency': request.urgency,
'quarantine_id': request.quarantine_id,
'timestamp': datetime.now().isoformat(),
'recommended_action': 'Manual review and revocation required',
'metadata': request.metadata
}
return await self._send_webhook_request(webhook_url, payload, 'webhook_revocation')
async def _send_webhook_request(self, url: str, payload: Dict[str, Any], method: str) -> RevocationResponse:
"""Send webhook request and handle response"""
try:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=self.request_timeout)) as session:
async with session.post(url, json=payload) as response:
response_data = {}
try:
response_data = await response.json()
except:
response_data = {'text': await response.text()}
if response.status == 200:
return RevocationResponse(
success=True,
method=method,
response_data=response_data,
revocation_id=response_data.get('revocation_id')
)
else:
return RevocationResponse(
success=False,
method=method,
response_data=response_data,
error_message=f"HTTP {response.status}: {response_data}"
)
except asyncio.TimeoutError:
return RevocationResponse(
success=False,
method=method,
response_data={},
error_message=f"Webhook request timed out after {self.request_timeout}s"
)
except Exception as e:
return RevocationResponse(
success=False,
method=method,
response_data={},
error_message=f"Webhook request failed: {str(e)}"
)
async def _record_revocation_event(self, quarantine_entry: QuarantineEntry, response: RevocationResponse):
"""Record revocation event in the database"""
try:
revocation_event = RevocationEvent(
quarantine_id=quarantine_entry.id,
secret_type=quarantine_entry.secret_type,
revocation_method=response.method,
status='success' if response.success else 'failed',
response_data=response.response_data,
timestamp=datetime.now()
)
await self.quarantine.record_revocation(revocation_event)
except Exception as e:
logger.error(f"Failed to record revocation event: {e}")
def _update_stats(self, secret_type: str, success: bool):
"""Update revocation statistics"""
self.stats['total_revocations'] += 1
if success:
self.stats['successful_revocations'] += 1
else:
self.stats['failed_revocations'] += 1
# Update by-type stats
if secret_type not in self.stats['revocations_by_type']:
self.stats['revocations_by_type'][secret_type] = {
'total': 0,
'successful': 0,
'failed': 0
}
type_stats = self.stats['revocations_by_type'][secret_type]
type_stats['total'] += 1
if success:
type_stats['successful'] += 1
else:
type_stats['failed'] += 1
async def test_webhook_endpoint(self, secret_type: str) -> Dict[str, Any]:
"""Test a webhook endpoint with a test payload"""
webhook_url = self.webhook_config.get(secret_type)
if not webhook_url:
return {
'success': False,
'error': f'No webhook configured for {secret_type}'
}
test_payload = {
'event': 'webhook_test',
'secret_type': secret_type,
'test': True,
'timestamp': datetime.now().isoformat()
}
try:
response = await self._send_webhook_request(webhook_url, test_payload, 'test')
return {
'success': response.success,
'method': response.method,
'response_data': response.response_data,
'error': response.error_message
}
except Exception as e:
return {
'success': False,
'error': str(e)
}
def get_stats(self) -> Dict[str, Any]:
"""Get revocation statistics"""
current_time = datetime.now()
uptime_hours = (current_time - self.stats['last_reset']).total_seconds() / 3600
stats = self.stats.copy()
stats.update({
'uptime_hours': round(uptime_hours, 2),
'success_rate': (
self.stats['successful_revocations'] / max(1, self.stats['total_revocations'])
) * 100,
'configured_webhooks': list(self.webhook_config.keys())
})
return stats
def reset_stats(self):
"""Reset statistics counters"""
self.stats = {
'total_revocations': 0,
'successful_revocations': 0,
'failed_revocations': 0,
'revocations_by_type': {},
'last_reset': datetime.now()
}
logger.info("SecretRevoker statistics reset")

33
modules/shhh/config.yaml Normal file
View File

@@ -0,0 +1,33 @@
# Configuration for the SHHH Secrets Sentinel
# -- File Paths --
# Path to the primary, raw hypercore log to be monitored.
primary_log_path: '/home/tony/AI/projects/chorus.services/modules/shhh/primary.log'
# Path where the sanitized sister hypercore log will be written.
sanitized_log_path: '/home/tony/AI/projects/chorus.services/modules/shhh/sanitized.log'
# Path to the YAML file containing regex patterns for secret detection.
patterns_file: 'patterns.yaml'
# Path to the system prompt file for the LLM agent.
shhh_agent_prompt_file: 'SHHH_SECRETS_SENTINEL_AGENT_PROMPT.md'
# -- Database --
# Connection string for the PostgreSQL database used for quarantining secrets.
# Format: postgresql://user:password@host:port/database
database_url: 'postgresql://shhh:password@localhost:5432/shhh_sentinel'
# -- LLM Analyzer (Ollama) --
# The API endpoint for the Ollama instance.
ollama_endpoint: 'http://localhost:11434/api/generate'
# The name of the model to use from Ollama (e.g., llama3, codellama).
ollama_model: 'llama3'
# The confidence score threshold for regex matches.
# Matches with confidence >= this value will be quarantined immediately, skipping the LLM.
# Matches with confidence < this value will be sent to the LLM for verification.
llm_confidence_threshold: 0.90

6
modules/shhh/core/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
# SHHH Core Module
"""
Core components for the SHHH Secrets Sentinel system.
"""
__version__ = "1.0.0"

52
modules/shhh/core/detector.py Normal file
View File

@@ -0,0 +1,52 @@
import re
import yaml
from pathlib import Path
class SecretDetector:
"""
A simplified secret detection engine using configurable regex patterns.
It scans text for secrets, redacts them, and provides metadata.
"""
def __init__(self, patterns_file: str = "patterns.yaml"):
self.patterns_file = Path(patterns_file)
self.patterns = self._load_patterns()
def _load_patterns(self) -> dict:
"""Load detection patterns from YAML configuration."""
try:
with open(self.patterns_file, 'r') as f:
config = yaml.safe_load(f)
patterns = config.get('patterns', {})
# Pre-compile regex for efficiency
for name, props in patterns.items():
if props.get('active', True):
props['compiled_regex'] = re.compile(props['regex'])
return patterns
except Exception as e:
print(f"[ERROR] Failed to load patterns from {self.patterns_file}: {e}")
return {}
def scan(self, text: str) -> list[dict]:
"""Scans text and returns a list of found secrets with metadata."""
matches = []
for pattern_name, pattern in self.patterns.items():
if pattern.get('active', True) and 'compiled_regex' in pattern:
regex_match = pattern['compiled_regex'].search(text)
if regex_match:
matches.append({
"secret_type": pattern_name,
"value": regex_match.group(0),
"confidence": pattern.get("confidence", 0.8),
"severity": pattern.get("severity", "MEDIUM")
})
return matches
def redact(self, text: str, secret_value: str) -> str:
"""Redacts a specific secret value within a string."""
# Ensure we don't reveal too much for very short secrets
if len(secret_value) < 8:
return text.replace(secret_value, "[REDACTED]")
redacted_str = secret_value[:4] + "****" + secret_value[-4:]
return text.replace(secret_value, f"[REDACTED:{redacted_str}]")

35
modules/shhh/core/hypercore_reader.py Normal file
View File

@@ -0,0 +1,35 @@
import asyncio
from datetime import datetime
class LogEntry:
"""A mock log entry object for testing purposes."""
def __init__(self, content):
self.content = content
self.timestamp = datetime.now()
# Add other fields as needed to match the processor's expectations
self.source_agent = "mock_agent"
self.message_type = "mock_message"
self.metadata = {}
self.is_bzzz_message = False
self.bzzz_message_id = None
class HypercoreReader:
"""
A simplified, mock HypercoreReader that reads from a plain text file
to simulate a stream of log entries for testing.
"""
def __init__(self, log_path: str, **kwargs):
self.log_path = log_path
async def stream_entries(self):
"""
An async generator that yields log entries from a text file.
"""
try:
with open(self.log_path, 'r') as f:
for line in f:
yield LogEntry(line.strip())
await asyncio.sleep(0.01) # Simulate async behavior
except FileNotFoundError:
print(f"[ERROR] Hypercore log file not found at: {self.log_path}")
return

44
modules/shhh/core/llm_analyzer.py Normal file
View File

@@ -0,0 +1,44 @@
import requests
import json
class LLMAnalyzer:
"""Analyzes text for secrets using a local LLM via Ollama."""
def __init__(self, endpoint: str, model: str, system_prompt: str):
self.endpoint = endpoint
self.model = model
self.system_prompt = system_prompt
def analyze(self, text: str) -> dict:
"""
Sends text to the Ollama API for analysis and returns a structured JSON response.
Returns:
A dictionary like:
{
"secret_found": bool,
"secret_type": str,
"confidence_score": float,
"severity": str
}
Returns a default "not found" response on error.
"""
prompt = f"Log entry: \"{text}\"\n\nAnalyze this for secrets and respond with only the required JSON."
payload = {
"model": self.model,
"system": self.system_prompt,
"prompt": prompt,
"format": "json",
"stream": False
}
try:
response = requests.post(self.endpoint, json=payload, timeout=15)
response.raise_for_status()
# The response from Ollama is a JSON string, which needs to be parsed.
analysis = json.loads(response.json().get("response", "{}"))
return analysis
except (requests.exceptions.RequestException, json.JSONDecodeError) as e:
print(f"[ERROR] LLMAnalyzer failed: {e}")
# Fallback: If LLM fails, assume no secret was found to avoid blocking the pipeline.
return {"secret_found": False}

22
modules/shhh/core/quarantine.py Normal file
View File

@@ -0,0 +1,22 @@
from datetime import datetime
class QuarantineManager:
"""
A simplified, mock QuarantineManager for testing purposes.
It prints quarantined messages to the console instead of saving to a database.
"""
def __init__(self, database_url: str, **kwargs):
print(f"[MockQuarantine] Initialized with db_url: {database_url}")
def quarantine_message(self, message, secret_type: str, severity: str, redacted_content: str):
"""
Prints a quarantined message to the console.
"""
print("\n--- QUARANTINE ALERT ---")
print(f"Timestamp: {datetime.now().isoformat()}")
print(f"Severity: {severity}")
print(f"Secret Type: {secret_type}")
print(f"Original Content (from mock): {message.content}")
print(f"Redacted Content: {redacted_content}")
print("------------------------\n")

16
modules/shhh/core/sanitized_writer.py Normal file
View File

@@ -0,0 +1,16 @@
class SanitizedWriter:
"""Writes log entries to the sanitized sister hypercore log."""
def __init__(self, sanitized_log_path: str):
self.log_path = sanitized_log_path
# Placeholder for hypercore writing logic. For now, we'll append to a file.
self.log_file = open(self.log_path, "a")
def write(self, log_entry: str):
"""Writes a single log entry to the sanitized stream."""
self.log_file.write(log_entry + "\n")
self.log_file.flush()
def close(self):
self.log_file.close()

4
modules/shhh/integrations/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# SHHH Integrations Module
"""
Integration components for BZZZ network and external systems.
"""

369
modules/shhh/integrations/bzzz_interceptor.py Normal file
View File

@@ -0,0 +1,369 @@
"""
BZZZ Message Interceptor for SHHH Secrets Sentinel
Intercepts and validates BZZZ P2P messages before network propagation.
"""
import asyncio
import json
import time
from typing import Dict, Any, Optional, Set, Callable
from dataclasses import dataclass
from datetime import datetime
import structlog
from ..core.hypercore_reader import BzzzMessage
from ..core.detector import SecretDetector, DetectionResult
from ..core.quarantine import QuarantineManager
logger = structlog.get_logger()
@dataclass
class BlockedMessage:
"""Represents a blocked BZZZ message"""
message_id: str
sender_agent: str
block_reason: str
secret_types: list
timestamp: datetime
quarantine_id: Optional[int] = None
class BzzzInterceptor:
"""
Intercepts BZZZ P2P messages before transmission to prevent secret leakage.
Integrates with the BZZZ network layer to scan messages in real-time.
"""
def __init__(
self,
detector: SecretDetector,
quarantine_manager: QuarantineManager,
bzzz_config: Dict[str, Any] = None
):
self.detector = detector
self.quarantine = quarantine_manager
self.bzzz_config = bzzz_config or {}
# Message blocking state
self.blocked_messages: Dict[str, BlockedMessage] = {}
self.message_hooks: Set[Callable] = set()
self.is_active = False
# Statistics
self.stats = {
'total_scanned': 0,
'secrets_detected': 0,
'messages_blocked': 0,
'false_positives': 0,
'last_reset': datetime.now()
}
logger.info("Initialized BzzzInterceptor")
async def start(self):
"""Start the BZZZ message interception service"""
self.is_active = True
logger.info("BZZZ Interceptor started - all outgoing messages will be scanned")
async def stop(self):
"""Stop the BZZZ message interception service"""
self.is_active = False
logger.info("BZZZ Interceptor stopped")
def install_message_hook(self, hook_function: Callable):
"""Install a message hook for BZZZ network integration"""
self.message_hooks.add(hook_function)
logger.info(f"Installed BZZZ message hook: {hook_function.__name__}")
def remove_message_hook(self, hook_function: Callable):
"""Remove a message hook"""
self.message_hooks.discard(hook_function)
logger.info(f"Removed BZZZ message hook: {hook_function.__name__}")
async def intercept_outgoing_message(self, message: BzzzMessage) -> bool:
"""
Intercept and scan an outgoing BZZZ message.
Returns True if message should be allowed, False if blocked.
"""
if not self.is_active:
return True # Pass through if interceptor is inactive
start_time = time.time()
self.stats['total_scanned'] += 1
try:
# Scan message for secrets
detection_result = self.detector.scan_bzzz_message(message)
if detection_result.has_secrets:
await self._handle_secret_detection(message, detection_result)
return False # Block message
# Message is clean, allow transmission
processing_time = (time.time() - start_time) * 1000
logger.debug(
f"BZZZ message scanned clean",
message_id=message.message_id,
sender=message.sender_agent,
processing_time_ms=processing_time
)
return True
except Exception as e:
logger.error(f"Error intercepting BZZZ message: {e}")
# On error, default to blocking for security
await self._block_message_on_error(message, str(e))
return False
async def _handle_secret_detection(self, message: BzzzMessage, detection_result: DetectionResult):
"""Handle detection of secrets in a BZZZ message"""
self.stats['secrets_detected'] += 1
self.stats['messages_blocked'] += 1
# Extract secret types for blocking record
secret_types = [match.secret_type for match in detection_result.matches]
# Quarantine the detection result
quarantine_entry = await self.quarantine.quarantine_detection(detection_result)
# Create blocked message record
blocked_msg = BlockedMessage(
message_id=message.message_id,
sender_agent=message.sender_agent,
block_reason=f"Secrets detected: {', '.join(secret_types)}",
secret_types=secret_types,
timestamp=datetime.now(),
quarantine_id=quarantine_entry.id
)
self.blocked_messages[message.message_id] = blocked_msg
# Notify BZZZ network layer
await self._notify_message_blocked(message, blocked_msg)
logger.critical(
f"BLOCKED BZZZ message containing secrets",
message_id=message.message_id,
sender=message.sender_agent,
recipient=message.recipient_agent,
secret_types=secret_types,
severity=detection_result.max_severity,
quarantine_id=quarantine_entry.id
)
async def _block_message_on_error(self, message: BzzzMessage, error_msg: str):
"""Block a message due to processing error"""
self.stats['messages_blocked'] += 1
blocked_msg = BlockedMessage(
message_id=message.message_id,
sender_agent=message.sender_agent,
block_reason=f"Processing error: {error_msg}",
secret_types=[],
timestamp=datetime.now()
)
self.blocked_messages[message.message_id] = blocked_msg
await self._notify_message_blocked(message, blocked_msg)
logger.error(
f"BLOCKED BZZZ message due to error",
message_id=message.message_id,
sender=message.sender_agent,
error=error_msg
)
async def _notify_message_blocked(self, message: BzzzMessage, blocked_msg: BlockedMessage):
"""Notify BZZZ network and sender about blocked message"""
notification = {
'event': 'message_blocked',
'message_id': message.message_id,
'sender_agent': message.sender_agent,
'recipient_agent': message.recipient_agent,
'block_reason': blocked_msg.block_reason,
'secret_types': blocked_msg.secret_types,
'timestamp': blocked_msg.timestamp.isoformat(),
'quarantine_id': blocked_msg.quarantine_id
}
# Notify all registered hooks
for hook in self.message_hooks:
try:
await self._call_hook_safely(hook, 'message_blocked', notification)
except Exception as e:
logger.warning(f"Hook {hook.__name__} failed: {e}")
# Send notification back to sender agent
await self._notify_sender_agent(message.sender_agent, notification)
async def _call_hook_safely(self, hook: Callable, event_type: str, data: Dict[str, Any]):
"""Safely call a hook function with error handling"""
try:
if asyncio.iscoroutinefunction(hook):
await hook(event_type, data)
else:
hook(event_type, data)
except Exception as e:
logger.warning(f"Hook {hook.__name__} failed: {e}")
async def _notify_sender_agent(self, sender_agent: str, notification: Dict[str, Any]):
"""Send notification to the sender agent about blocked message"""
try:
# This would integrate with the BZZZ network's agent communication system
# For now, we'll log the notification
logger.info(
f"Notifying agent about blocked message",
agent=sender_agent,
message_id=notification['message_id'],
reason=notification['block_reason']
)
# TODO: Implement actual agent notification via BZZZ network
# This might involve:
# - Sending a system message back to the agent
# - Updating agent's message status
# - Triggering agent's error handling workflow
except Exception as e:
logger.error(f"Failed to notify sender agent {sender_agent}: {e}")
def is_message_blocked(self, message_id: str) -> Optional[BlockedMessage]:
"""Check if a message is blocked"""
return self.blocked_messages.get(message_id)
def unblock_message(self, message_id: str, reviewer: str, reason: str) -> bool:
"""Unblock a previously blocked message (for false positives)"""
if message_id not in self.blocked_messages:
return False
blocked_msg = self.blocked_messages[message_id]
# Mark as false positive in stats
self.stats['false_positives'] += 1
# Remove from blocked messages
del self.blocked_messages[message_id]
logger.info(
f"Unblocked BZZZ message",
message_id=message_id,
reviewer=reviewer,
reason=reason,
original_block_reason=blocked_msg.block_reason
)
return True
def get_blocked_messages(self, limit: int = 100) -> list[BlockedMessage]:
"""Get list of recently blocked messages"""
blocked_list = list(self.blocked_messages.values())
blocked_list.sort(key=lambda x: x.timestamp, reverse=True)
return blocked_list[:limit]
def get_stats(self) -> Dict[str, Any]:
"""Get interceptor statistics"""
current_time = datetime.now()
uptime_hours = (current_time - self.stats['last_reset']).total_seconds() / 3600
stats = self.stats.copy()
stats.update({
'uptime_hours': round(uptime_hours, 2),
'is_active': self.is_active,
'blocked_messages_count': len(self.blocked_messages),
'detection_rate': (
self.stats['secrets_detected'] / max(1, self.stats['total_scanned'])
) * 100,
'false_positive_rate': (
self.stats['false_positives'] / max(1, self.stats['secrets_detected'])
) * 100 if self.stats['secrets_detected'] > 0 else 0
})
return stats
def reset_stats(self):
"""Reset statistics counters"""
self.stats = {
'total_scanned': 0,
'secrets_detected': 0,
'messages_blocked': 0,
'false_positives': 0,
'last_reset': datetime.now()
}
logger.info("BZZZ Interceptor statistics reset")
async def cleanup_old_blocked_messages(self, hours: int = 24):
"""Clean up old blocked message records"""
cutoff_time = datetime.now() - timedelta(hours=hours)
old_messages = [
msg_id for msg_id, blocked_msg in self.blocked_messages.items()
if blocked_msg.timestamp < cutoff_time
]
for msg_id in old_messages:
del self.blocked_messages[msg_id]
if old_messages:
logger.info(f"Cleaned up {len(old_messages)} old blocked message records")
return len(old_messages)
class BzzzNetworkAdapter:
"""
Adapter to integrate BzzzInterceptor with the actual BZZZ network layer.
This would be customized based on the BZZZ implementation details.
"""
def __init__(self, interceptor: BzzzInterceptor):
self.interceptor = interceptor
self.original_send_function = None
def install_interceptor(self, bzzz_network_instance):
"""Install interceptor into BZZZ network layer"""
# This would need to be customized based on actual BZZZ implementation
# Example pattern:
# Store original send function
self.original_send_function = bzzz_network_instance.send_message
# Replace with intercepting version
bzzz_network_instance.send_message = self._intercepting_send_message
logger.info("BzzzInterceptor installed into BZZZ network layer")
async def _intercepting_send_message(self, message_data: Dict[str, Any]):
"""Intercepting version of BZZZ send_message function"""
try:
# Convert to BzzzMessage format
bzzz_message = self._convert_to_bzzz_message(message_data)
# Check with interceptor
should_allow = await self.interceptor.intercept_outgoing_message(bzzz_message)
if should_allow:
# Call original send function
return await self.original_send_function(message_data)
else:
# Message was blocked
raise Exception(f"Message blocked by security interceptor: {bzzz_message.message_id}")
except Exception as e:
logger.error(f"Error in intercepting send: {e}")
raise
def _convert_to_bzzz_message(self, message_data: Dict[str, Any]) -> BzzzMessage:
"""Convert BZZZ network message format to BzzzMessage"""
# This would need to be customized based on actual BZZZ message format
return BzzzMessage(
message_id=message_data.get('id', f"auto_{int(time.time())}"),
sender_agent=message_data.get('sender', 'unknown'),
recipient_agent=message_data.get('recipient'),
message_type=message_data.get('type', 'unknown'),
payload=json.dumps(message_data.get('payload', message_data)),
timestamp=datetime.now(),
network_metadata=message_data
)

181
modules/shhh/main.py Normal file
View File

@@ -0,0 +1,181 @@
#!/usr/bin/env python3
"""
SHHH Secrets Sentinel - Main Entry Point
Production-ready secrets detection and monitoring system for CHORUS Services.
"""
import asyncio
import argparse
import sys
import yaml
from pathlib import Path
import structlog
from typing import Dict, Any
# Updated imports to bring in the new and modified components
from pipeline.processor import MessageProcessor
from core.hypercore_reader import HypercoreReader
from core.detector import SecretDetector
from core.llm_analyzer import LLMAnalyzer
from core.quarantine import QuarantineManager
from core.sanitized_writer import SanitizedWriter
def setup_logging(log_level: str = "INFO", structured: bool = True):
"""Configure structured logging"""
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="iso"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
structlog.processors.JSONRenderer() if structured else structlog.dev.ConsoleRenderer(),
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def load_config(config_path: str) -> Dict[str, Any]:
"""Load configuration from YAML file"""
try:
with open(config_path, 'r') as f:
config = yaml.safe_load(f)
return config
except FileNotFoundError:
print(f"Configuration file not found: {config_path}, using defaults.")
return get_default_config()
except yaml.YAMLError as e:
print(f"Error parsing configuration file: {e}")
sys.exit(1)
def get_default_config() -> Dict[str, Any]:
"""Get default configuration, updated for the new architecture."""
return {
'primary_log_path': '/path/to/primary/hypercore.log',
'sanitized_log_path': '/path/to/sanitized/hypercore.log',
'database_url': 'postgresql://shhh:password@localhost:5432/shhh_sentinel',
'patterns_file': 'patterns.yaml',
'ollama_endpoint': 'http://localhost:11434/api/generate',
'ollama_model': 'llama3',
'llm_confidence_threshold': 0.90,
'shhh_agent_prompt_file': 'SHHH_SECRETS_SENTINEL_AGENT_PROMPT.md'
}
async def run_monitor_mode(config: Dict[str, Any]):
"""Run in monitoring mode with the new hybrid pipeline."""
logger = structlog.get_logger()
logger.info("Starting SHHH in monitor mode with hybrid pipeline...")
writer = None
try:
# 1. Load System Prompt for LLM
try:
with open(config['shhh_agent_prompt_file'], "r") as f:
ollama_system_prompt = f.read()
except FileNotFoundError:
logger.error(f"LLM prompt file not found at {config['shhh_agent_prompt_file']}. Aborting.")
return
# 2. Instantiation of components
# Note: HypercoreReader and QuarantineManager might need async initialization
# which is not shown here for simplicity, following the plan.
reader = HypercoreReader(config['primary_log_path'])
detector = SecretDetector(config['patterns_file'])
llm_analyzer = LLMAnalyzer(config['ollama_endpoint'], config['ollama_model'], ollama_system_prompt)
quarantine = QuarantineManager(config['database_url'])
writer = SanitizedWriter(config['sanitized_log_path'])
processor = MessageProcessor(
reader=reader,
detector=detector,
llm_analyzer=llm_analyzer,
quarantine=quarantine,
writer=writer,
llm_threshold=config['llm_confidence_threshold']
)
# 3. Execution
logger.info("Starting processor stream...")
await processor.process_stream()
except Exception as e:
logger.error("An error occurred during monitor mode execution.", error=str(e))
finally:
if writer:
writer.close()
logger.info("Monitor mode shut down complete.")
async def run_api_mode(config: Dict[str, Any], host: str, port: int):
"""Run in API mode (dashboard server) - UNCHANGED"""
import uvicorn
from api.main import app
app.state.config = config
uvicorn_config = uvicorn.Config(app=app, host=host, port=port, log_level="info", access_log=True)
server = uvicorn.Server(uvicorn_config)
await server.serve()
async def run_test_mode(config: Dict[str, Any], test_file: str):
"""Run in test mode with sample data - UNCHANGED but may be broken."""
logger = structlog.get_logger()
logger.warning("Test mode may be broken due to recent refactoring.")
# This part of the code would need to be updated to work with the new SecretDetector.
# For now, it remains as it was.
from core.detector import SecretDetector
from datetime import datetime
detector = SecretDetector(config['patterns_file'])
logger.info("Running SHHH in test mode")
# ... (rest of the test mode logic is likely broken and needs updating)
def main():
"""Main entry point"""
parser = argparse.ArgumentParser(description="SHHH Secrets Sentinel")
parser.add_argument('--config', '-c', default='config.yaml', help='Configuration file path')
parser.add_argument('--mode', '-m', choices=['monitor', 'api', 'test'], default='monitor', help='Operation mode')
parser.add_argument('--log-level', '-l', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], default='INFO', help='Log level')
parser.add_argument('--structured-logs', action='store_true', help='Use structured JSON logging')
parser.add_argument('--host', default='127.0.0.1', help='API server host')
parser.add_argument('--port', '-p', type=int, default=8000, help='API server port')
parser.add_argument('--test-file', help='Test data file for test mode')
parser.add_argument('--version', '-v', action='version', version='SHHH Secrets Sentinel 1.1.0 (Hybrid)')
args = parser.parse_args()
setup_logging(args.log_level, args.structured_logs)
logger = structlog.get_logger()
config = load_config(args.config)
logger.info("Starting SHHH Secrets Sentinel", mode=args.mode, config_file=args.config)
try:
if args.mode == 'monitor':
asyncio.run(run_monitor_mode(config))
elif args.mode == 'api':
asyncio.run(run_api_mode(config, args.host, args.port))
elif args.mode == 'test':
asyncio.run(run_test_mode(config, args.test_file))
except KeyboardInterrupt:
logger.info("Shutting down due to keyboard interrupt.")
except Exception as e:
logger.error("Application failed", error=str(e))
sys.exit(1)
logger.info("SHHH Secrets Sentinel stopped.")
if __name__ == '__main__':
main()

121
modules/shhh/patterns.yaml Normal file
View File

@@ -0,0 +1,121 @@
# SHHH Secrets Detection Patterns
# Configuration for the Secrets Sentinel monitoring system
patterns:
AWS_ACCESS_KEY:
regex: "AKIA[0-9A-Z]{16}"
severity: "CRITICAL"
confidence: 0.95
active: true
description: "AWS Access Key ID"
remediation: "Revoke via AWS IAM immediately"
AWS_SECRET_KEY:
regex: "[A-Za-z0-9/+=]{40}"
severity: "CRITICAL"
confidence: 0.85
active: true
description: "AWS Secret Access Key"
remediation: "Revoke via AWS IAM immediately"
context_required: true # Requires context analysis
PRIVATE_KEY:
regex: "-----BEGIN [A-Z ]*PRIVATE KEY-----"
severity: "CRITICAL"
confidence: 0.98
active: true
description: "Private Key (RSA, SSH, etc.)"
remediation: "Rotate key immediately"
GITHUB_TOKEN:
regex: "ghp_[0-9A-Za-z]{36}"
severity: "HIGH"
confidence: 0.92
active: true
description: "GitHub Personal Access Token"
remediation: "Revoke via GitHub settings"
GITHUB_OAUTH:
regex: "gho_[0-9A-Za-z]{36}"
severity: "HIGH"
confidence: 0.92
active: true
description: "GitHub OAuth Token"
remediation: "Revoke via GitHub app settings"
SLACK_TOKEN:
regex: "xox[baprs]-[0-9A-Za-z-]{10,48}"
severity: "HIGH"
confidence: 0.90
active: true
description: "Slack Bot/User Token"
remediation: "Revoke via Slack Admin API"
JWT_TOKEN:
regex: "eyJ[A-Za-z0-9_-]+?\\.[A-Za-z0-9_-]+?\\.[A-Za-z0-9_-]+?"
severity: "MEDIUM"
confidence: 0.85
active: true
description: "JSON Web Token"
remediation: "Invalidate token and rotate signing key"
GOOGLE_API_KEY:
regex: "AIza[0-9A-Za-z\\-_]{35}"
severity: "HIGH"
confidence: 0.90
active: true
description: "Google API Key"
remediation: "Revoke via Google Cloud Console"
DOCKER_TOKEN:
regex: "dckr_pat_[a-zA-Z0-9_-]{32,}"
severity: "MEDIUM"
confidence: 0.88
active: true
description: "Docker Personal Access Token"
remediation: "Revoke via Docker Hub settings"
GENERIC_API_KEY:
regex: "[Aa][Pp][Ii]_?[Kk][Ee][Yy].*['\"][0-9a-zA-Z]{32,}['\"]"
severity: "MEDIUM"
confidence: 0.70
active: true
description: "Generic API Key Pattern"
remediation: "Verify and revoke if legitimate"
# Pattern exceptions - known test/dummy values to ignore
exceptions:
test_patterns:
- "AKIA-TESTKEY-123"
- "AKIAIOSFODNN7EXAMPLE"
- "xoxb-test-token"
- "ghp_test123456789012345678901234567890"
- "-----BEGIN EXAMPLE PRIVATE KEY-----"
development_indicators:
- "test"
- "example"
- "demo"
- "mock"
- "fake"
- "dummy"
# Quarantine settings
quarantine:
high_severity_auto_quarantine: true
medium_severity_review_required: true
retention_days: 90
max_entries: 10000
# Alert settings
alerts:
webhook_timeout_seconds: 5
retry_attempts: 3
retry_delay_seconds: 2
# Revocation hooks
revocation_hooks:
AWS_ACCESS_KEY: "https://security.chorus.services/hooks/aws-revoke"
GITHUB_TOKEN: "https://security.chorus.services/hooks/github-revoke"
SLACK_TOKEN: "https://security.chorus.services/hooks/slack-revoke"
GOOGLE_API_KEY: "https://security.chorus.services/hooks/google-revoke"

4
modules/shhh/pipeline/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# SHHH Pipeline Module
"""
Main processing pipeline for the SHHH Secrets Sentinel system.
"""

66
modules/shhh/pipeline/processor.py Normal file
View File

@@ -0,0 +1,66 @@
import asyncio
from core.hypercore_reader import HypercoreReader
from core.detector import SecretDetector
from core.llm_analyzer import LLMAnalyzer
from core.quarantine import QuarantineManager
from core.sanitized_writer import SanitizedWriter
class MessageProcessor:
def __init__(self, reader: HypercoreReader, detector: SecretDetector, llm_analyzer: LLMAnalyzer, quarantine: QuarantineManager, writer: SanitizedWriter, llm_threshold: float):
self.reader = reader
self.detector = detector
self.llm_analyzer = llm_analyzer
self.quarantine = quarantine
self.writer = writer
self.llm_threshold = llm_threshold # e.g., 0.90
async def process_stream(self):
"""Main processing loop for the hybrid detection model."""
async for entry in self.reader.stream_entries():
# Stage 1: Fast Regex Scan
regex_matches = self.detector.scan(entry.content)
if not regex_matches:
# No secrets found, write original entry to sanitized log
self.writer.write(entry.content)
continue
# A potential secret was found. Default to sanitized, but may be quarantined.
sanitized_content = entry.content
should_quarantine = False
confirmed_secret = None
for match in regex_matches:
# High-confidence regex matches trigger immediate quarantine, skipping LLM.
if match['confidence'] >= self.llm_threshold:
should_quarantine = True
confirmed_secret = match
break # One high-confidence match is enough
# Stage 2: Low-confidence matches go to LLM for verification.
llm_result = self.llm_analyzer.analyze(entry.content)
if llm_result.get("secret_found"):
should_quarantine = True
# Prefer LLM's classification but use regex value for redaction
confirmed_secret = {
"secret_type": llm_result.get("secret_type", match['secret_type']),
"value": match['value'],
"severity": llm_result.get("severity", match['severity'])
}
break
if should_quarantine and confirmed_secret:
# A secret is confirmed. Redact, quarantine, and alert.
sanitized_content = self.detector.redact(entry.content, confirmed_secret['value'])
self.quarantine.quarantine_message(
message=entry,
secret_type=confirmed_secret['secret_type'],
severity=confirmed_secret['severity'],
redacted_content=sanitized_content
)
# Potentially trigger alerts here as well
print(f"[ALERT] Confirmed secret {confirmed_secret['secret_type']} found and quarantined.")
# Write the (potentially redacted) content to the sanitized log
self.writer.write(sanitized_content)

15
modules/shhh/requirements.txt Normal file
View File

@@ -0,0 +1,15 @@
# SHHH Secrets Sentinel Dependencies
fastapi==0.104.1
uvicorn[standard]==0.24.0
psycopg2-binary==2.9.9
pydantic==2.5.0
requests==2.31.0
pyyaml==6.0.1
redis==5.0.1
asyncio-mqtt==0.15.1
watchdog==3.0.0
prometheus-client==0.19.0
python-multipart==0.0.6
aiofiles==23.2.1
hypercorn==0.15.0
structlog==23.2.0

995
modules/shhh/secrets-sentinel.md Normal file
View File

@@ -0,0 +1,995 @@
Here’s a **clean, production-ready system prompt** for that agent:
---
**🛡️ System Prompt – “Secrets Sentinel” Agent**
> **Role & Mission**:
> You are the **Secrets Sentinel**, an autonomous security agent tasked with **monitoring all incoming log entries** for any potential leaks of **API keys, passwords, tokens, or other sensitive credentials**. Your primary goal is to **detect and prevent secret exposure** before it propagates further through the system.
>
> **Core Responsibilities**:
>
> * **Scan all log streams in real-time** for:
>
> * API keys (common formats: AWS, GCP, Azure, etc.)
> * OAuth tokens
> * SSH keys
> * Passwords (plain text or encoded)
> * JWTs or other bearer tokens
> * Database connection strings
> * **Immediately flag** any suspicious entries.
> * **Classify severity** (e.g., HIGH – AWS root key; MEDIUM – temporary token).
> * **Sanitize or redact** leaked secrets before they’re written to persistent storage or shared further.
> * **Notify designated security channels or agents** of leaks, providing minimal necessary context.
>
> **Guidelines**:
>
> * Never expose the full secret in your alerts — redact most of it (e.g., `AKIA************XYZ`).
> * Be cautious of **false positives** (e.g., test data, dummy keys); err on the side of safety but include a “confidence score.”
> * Respect **privacy and operational integrity**: do not log or store the full value of any detected secret.
> * Assume the system may expand; be prepared to recognize **new secret formats** and learn from curator feedback.
>
> **Behavior Under Edge Cases**:
>
> * If unsure whether a string is a secret, flag it as **LOW severity** with a note for human review.
> * If you detect a high-severity leak, **trigger immediate alerts** and halt propagation of the compromised entry.
>
> **Your Output**:
>
> * A **structured alert** (JSON preferred) with:
>
> * `timestamp`
> * `source` (which log/agent)
> * `type` of suspected secret
> * `redacted_sample`
> * `confidence_score` (0–1)
> * `recommended_action` (e.g., “revoke key,” “rotate password,” “ignore dummy”)
>
> **Tone & Style**:
>
> * Precise, neutral, security-minded.
> * Avoid speculation beyond what you can confidently identify.
---
## 📂 **Version-Controlled `patterns.yaml` Format**
This lets you add/update/remove detection patterns **without touching code**.
```yaml
version: 1.2
last_updated: 2025-08-02
patterns:
AWS_ACCESS_KEY:
regex: "AKIA[0-9A-Z]{16}"
description: "AWS Access Key ID"
severity: HIGH
confidence: 0.99
active: true
AWS_SECRET_KEY:
regex: "(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]"
description: "AWS Secret Key"
severity: HIGH
confidence: 0.99
active: true
GITHUB_TOKEN:
regex: "gh[pousr]_[0-9A-Za-z]{36}"
description: "GitHub Personal Access Token"
severity: HIGH
confidence: 0.97
active: true
JWT:
regex: "eyJ[A-Za-z0-9_-]+?\\.[A-Za-z0-9._-]+?\\.[A-Za-z0-9._-]+"
description: "JSON Web Token"
severity: MEDIUM
confidence: 0.95
active: true
meta:
allow_feedback_learning: true
require_human_review_above_confidence: 0.8
```
✅ **Advantages:**
- Regexes are editable without code changes.
- Can be versioned in Git for full audit trail.
- Can toggle `active: false` for deprecating broken rules.
---
## 🖼 **Flow Diagram (Secrets Sentinel)**
**Secrets Flow**
```
┌───────────────┐
Logs Stream →│ Secrets │
│ Sentinel │
└──────┬────────┘
│
┌─────────┼─────────┐
│ │
[Quarantine] [Sanitized Logs]
│ │
┌──────┴──────┐ ┌────┴─────┐
│High Severity│ │ Safe Data│
│Secrets Only │ │ Storage │
└──────┬──────┘ └────┬─────┘
│ │
┌────────┼─────────┐ │
│ Revocation Hooks │ │
│ (AWS, GitHub, │ │
│ Slack, etc.) │ │
└────────┬─────────┘ │
│ │
┌────┴─────┐ │
│ Webhooks │ │
│ Key Kill │ │
└────┬─────┘ │
│
┌─────────┼─────────┐
│ Feedback Loop │
│ (Curator/Human) │
└─────────┬─────────┘
│
┌──────┴──────┐
│ Meta-Learner│
│ (new regex) │
└──────┬──────┘
│
┌──────┴───────┐
│ patterns.yaml│
└──────────────┘
```
---
## 🧪 **Test Harness Script**
This script simulates log scanning, quarantining, and revocation.
```python
import yaml, json, re
from datetime import datetime
# --- Load patterns.yaml ---
with open("patterns.yaml", "r") as f:
patterns_config = yaml.safe_load(f)
PATTERNS = patterns_config["patterns"]
QUARANTINE = []
SANITIZED_LOGS = []
def redact(secret):
return secret[:4] + "*" * (len(secret) - 7) + secret[-3:]
def scan_log(log_line, log_id, source_agent):
alerts = []
for secret_type, props in PATTERNS.items():
if not props.get("active", True):
continue
match = re.search(props["regex"], log_line)
if match:
secret = match.group(0)
severity = props["severity"]
alert = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"source_agent": source_agent,
"log_line_id": log_id,
"secret_type": secret_type,
"redacted_sample": redact(secret),
"confidence_score": props["confidence"],
"severity": severity,
"recommended_action": "Revoke key/rotate credentials" if severity == "HIGH" else "Review"
}
alerts.append(alert)
# Quarantine if severity is HIGH
if severity == "HIGH":
quarantine_log(log_line, f"High severity secret detected: {secret_type}")
trigger_revocation(secret_type, redact(secret))
return alerts
def quarantine_log(log_line, reason):
entry = {"timestamp": datetime.utcnow().isoformat() + "Z", "reason": reason, "log_line": log_line}
QUARANTINE.append(entry)
print(f"[QUARANTINE] {reason}")
def trigger_revocation(secret_type, redacted_sample):
# Simulated webhook call
print(f"[REVOCATION] Simulated revocation triggered for {secret_type} ({redacted_sample})")
def process_logs(logs):
for i, log_line in enumerate(logs):
alerts = scan_log(log_line, f"log_{i}", "agent_demo")
if alerts:
print(json.dumps(alerts, indent=2))
else:
SANITIZED_LOGS.append(log_line)
# --- Test Run ---
sample_logs = [
"INFO User logged in successfully",
"WARNING Found AWS key AKIA1234567890ABCD in commit",
"DEBUG JWT detected eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.fake.fake"
]
process_logs(sample_logs)
print("\n--- Quarantine Store ---")
print(json.dumps(QUARANTINE, indent=2))
```
✅ **What this does:**
- Reads `patterns.yaml`
- Scans logs, prints alerts, quarantines high-severity entries
- Simulates revocation calls for AWS/GitHub/Slack
- Keeps sanitized logs separate from quarantined logs
---
## ✅ Next Expansions (Optional)
- 📦 **Redis/DB backend** for QUARANTINE instead of memory.
- 📡 **Real webhook integrations** (AWS STS, GitHub API, Slack API).
- 🧠 **Feedback ingestion module** (e.g., curator submits: `"false_positive": "AWS_ACCESS_KEY"` → adjusts regex in `patterns.yaml`).
- 🔄 **Auto-replay from Hyperlog** so Sentinel can retroactively scan old logs with new regex rules.
---
🔥 **production-grade spec**.
---
## 📂 **1️⃣ Feedback Ingestion Spec**
This defines how curators/humans give feedback to the Sentinel so it can **update its detection rules (patterns.yaml)** safely.
---
### 🔄 **Feedback Flow**
1. **Curator/Reviewer sees alert** → marks it as:
- `false_positive` (regex over-triggered)
- `missed_secret` (regex failed to detect)
- `uncertain` (needs better regex refinement)
2. **Feedback API** ingests the report:
```json
{
"alert_id": "log_345",
"secret_type": "AWS_ACCESS_KEY",
"feedback_type": "false_positive",
"evidence": "Key was dummy data: TESTKEY123",
"suggested_regex_fix": null
}
```
3. **Meta-Learner** updates rules:
- `false_positive` → adds **exceptions** (e.g., allowlist prefixes like `TESTKEY`).
- `missed_secret` → drafts **new regex** from evidence (using regex generator or LLM).
- Writes changes to **patterns.yaml** under `pending_review`.
4. **Security admin approves** before the new regex is marked `active: true`.
---
### 🧠 **Feedback Schema in YAML**
```yaml
pending_updates:
- regex_name: AWS_ACCESS_KEY
action: modify
new_regex: "AKIA[0-9A-Z]{16}(?!TESTKEY)"
confidence: 0.82
status: "pending human review"
submitted_by: curator_2
timestamp: 2025-08-02T12:40:00Z
```
✅ This keeps **audit trails** & allows **safe hot updates**.
---
## ⚙️ **2️⃣ Real AWS/GitHub Webhook Payload Templates**
These are **example POST payloads** your Sentinel would send when it detects a leaked secret.
---
### 🔐 **AWS Access Key Revocation**
**Endpoint:**
`POST https://security.example.com/hooks/aws-revoke`
**Payload:**
```json
{
"event": "secret_leak_detected",
"secret_type": "AWS_ACCESS_KEY",
"redacted_key": "AKIA****XYZ",
"log_reference": "hyperlog:58321",
"recommended_action": "Revoke IAM access key immediately",
"severity": "HIGH",
"timestamp": "2025-08-02T12:45:00Z"
}
```
➡ Your security automation would call AWS CLI or IAM API:
```bash
aws iam update-access-key --access-key-id <redacted> --status Inactive
aws iam delete-access-key --access-key-id <redacted>
```
---
### 🐙 **GitHub Token Revocation**
**Endpoint:**
`POST https://security.example.com/hooks/github-revoke`
**Payload:**
```json
{
"event": "secret_leak_detected",
"secret_type": "GITHUB_TOKEN",
"redacted_key": "ghp_****abcd",
"repository": "repo-name",
"log_reference": "hyperlog:58322",
"severity": "HIGH",
"recommended_action": "Invalidate GitHub token via API",
"timestamp": "2025-08-02T12:46:00Z"
}
```
➡ This would tie into GitHub’s [token-scanning API](https://docs.github.com/en/developers/overview/secret-scanning) or use PAT revocation.
---
### 💬 **Slack Token Revocation**
**Endpoint:**
`POST https://security.example.com/hooks/slack-revoke`
**Payload:**
```json
{
"event": "secret_leak_detected",
"secret_type": "SLACK_TOKEN",
"redacted_key": "xoxb****hjk",
"workspace": "company-slack",
"log_reference": "hyperlog:58323",
"severity": "HIGH",
"recommended_action": "Revoke Slack bot/user token",
"timestamp": "2025-08-02T12:47:00Z"
}
```
➡ Slack Admin API can be used to **revoke** or **rotate**.
---
## 📡 **3️⃣ Redis or PostgreSQL Quarantine Store**
Switching from memory to **persistent storage** means quarantined logs survive restarts.
---
### ✅ **Redis Option (Fast, Volatile)**
```python
import redis, json
r = redis.Redis(host='localhost', port=6379, decode_responses=True)
def quarantine_log(log_line, reason):
entry = {"timestamp": datetime.utcnow().isoformat() + "Z", "reason": reason, "log_line": log_line}
r.lpush("quarantine", json.dumps(entry))
print(f"[QUARANTINE] Stored in Redis: {reason}")
```
- 🏎 **Pros:** Fast, easy to scale.
- ⚠️ **Cons:** Volatile unless persisted (RDB/AOF).
---
### ✅ **PostgreSQL Option (Auditable, Durable)**
**Schema:**
```sql
CREATE TABLE quarantine (
id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ NOT NULL,
reason TEXT NOT NULL,
log_line TEXT NOT NULL,
reviewed BOOLEAN DEFAULT FALSE
);
```
**Python Insert:**
```python
import psycopg2
conn = psycopg2.connect("dbname=sentinel user=postgres password=secret")
cursor = conn.cursor()
def quarantine_log(log_line, reason):
entry_time = datetime.utcnow().isoformat() + "Z"
cursor.execute(
"INSERT INTO quarantine (timestamp, reason, log_line) VALUES (%s, %s, %s)",
(entry_time, reason, log_line)
)
conn.commit()
print(f"[QUARANTINE] Stored in PostgreSQL: {reason}")
```
✅ **Postgres is better for long-term auditing** — you can run reports like:
- “How many AWS keys leaked this month?”
- “Which agents generated the most HIGH-severity quarantines?”
---
## 🚀 **What’s Next?**
We now have:
✅ **Detection → Redaction → Quarantine → Revocation → Feedback → Pattern Evolution**
✅ **patterns.yaml** for versioned regex
✅ **Webhooks** for real-time secret revocation
✅ **Persistent quarantine store** (Redis or Postgres)
---
## 🛡️ **Expanded System Prompt for Secrets Sentinel**
> **Role & Mission**:
> You are the **Secrets Sentinel**, a security-focused agent monitoring all log streams for potential leaks of sensitive information (API keys, passwords, tokens, etc.). Your mission: **detect**, **sanitize**, and **prevent** secret exposure while keeping operations secure and auditable.
>
> **Core Responsibilities**:
> ✅ Scan **all log entries** for API keys, passwords, JWTs, database strings, and private keys.
> ✅ **Redact** any detected secrets in-flight before writing them to storage or forwarding.
> ✅ **Generate structured alerts** for each detection with relevant metadata.
> ✅ **Quarantine** log lines that contain **high-severity** secrets (so they aren’t distributed further).
> ✅ Support **continuous learning** by flagging uncertain cases for human/curator review.
>
> **Secret Detection Targets**:
>
> - **Cloud Keys** (AWS, GCP, Azure, etc.)
>
> - **OAuth Tokens** (Bearer, Slack, Discord, GitHub, etc.)
>
> - **JWTs** (header.payload.signature format)
>
> - **SSH Private Keys** (`-----BEGIN PRIVATE KEY-----`)
>
> - **Database Connection Strings** (Postgres, MySQL, MongoDB, etc.)
>
> - **Generic Passwords** (detected from common prefixes, e.g. `pwd=`, `password:`).
>
>
> **Detection Rules**:
>
> - Use **regex patterns** for known key formats.
>
> - Score detections with a **confidence metric** (0–1).
>
> - If a string doesn’t fully match, classify as **LOW confidence** for review.
>
>
> **Redaction Policy**:
>
> - Always redact most of the secret (`AKIA************XYZ`).
>
> - Never store or transmit the **full secret**.
>
>
> **Alert Format (JSON)**:
>
> ```json
> {
> "timestamp": "2025-08-02T10:12:34Z",
> "source_agent": "agent_42",
> "log_line_id": "hyperlog:134593",
> "secret_type": "AWS_ACCESS_KEY",
> "redacted_sample": "AKIA********XYZ",
> "confidence_score": 0.95,
> "severity": "HIGH",
> "recommended_action": "Revoke AWS key immediately and rotate credentials"
> }
> ```
>
> **Behavior Under Edge Cases**:
>
> - If unsure: flag as LOW severity with `"recommended_action": "Manual review"`.
>
> - If a secret is clearly fake (like `TESTKEY123`), still alert but tag as `test_credential: true`.
>
>
> **Tone & Style**:
>
> - Precise, security-minded, and concise in reporting.
>
---
## 📚 **Regex Patterns Library (Starter Set)**
```python
REGEX_PATTERNS = {
"AWS_ACCESS_KEY": r"AKIA[0-9A-Z]{16}",
"AWS_SECRET_KEY": r"(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]",
"GCP_API_KEY": r"AIza[0-9A-Za-z\\-_]{35}",
"GITHUB_TOKEN": r"gh[pousr]_[0-9A-Za-z]{36}",
"SLACK_TOKEN": r"xox[baprs]-[0-9A-Za-z-]{10,48}",
"JWT": r"eyJ[A-Za-z0-9_-]+?\.[A-Za-z0-9._-]+?\.[A-Za-z0-9._-]+",
"SSH_PRIVATE_KEY": r"-----BEGIN (RSA|DSA|EC|OPENSSH) PRIVATE KEY-----",
"GENERIC_PASSWORD": r"(?:password|pwd|pass|secret)\s*[:=]\s*['\"]?[^\s'\";]+['\"]?",
"DB_CONN_STRING": r"(postgres|mysql|mongodb|mssql|redis):\/\/[^\s]+"
}
```
---
## 🛠 **Python Skeleton Implementation**
```python
import re
import json
from datetime import datetime
REGEX_PATTERNS = {
"AWS_ACCESS_KEY": r"AKIA[0-9A-Z]{16}",
"AWS_SECRET_KEY": r"(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]",
"GCP_API_KEY": r"AIza[0-9A-Za-z\\-_]{35}",
"GITHUB_TOKEN": r"gh[pousr]_[0-9A-Za-z]{36}",
"SLACK_TOKEN": r"xox[baprs]-[0-9A-Za-z-]{10,48}",
"JWT": r"eyJ[A-Za-z0-9_-]+?\.[A-Za-z0-9._-]+?\.[A-Za-z0-9._-]+",
"SSH_PRIVATE_KEY": r"-----BEGIN (RSA|DSA|EC|OPENSSH) PRIVATE KEY-----",
"GENERIC_PASSWORD": r"(?:password|pwd|pass|secret)\s*[:=]\s*['\"]?[^\s'\";]+['\"]?",
"DB_CONN_STRING": r"(postgres|mysql|mongodb|mssql|redis):\/\/[^\s]+"
}
def redact(secret: str) -> str:
"""Redact a secret leaving only first and last 3 chars."""
return secret[:4] + "*" * (len(secret) - 7) + secret[-3:]
def scan_log_line(log_line: str, log_id: str, source_agent: str):
alerts = []
for secret_type, pattern in REGEX_PATTERNS.items():
match = re.search(pattern, log_line)
if match:
secret = match.group(0)
alert = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"source_agent": source_agent,
"log_line_id": log_id,
"secret_type": secret_type,
"redacted_sample": redact(secret),
"confidence_score": 0.95,
"severity": "HIGH" if "PRIVATE_KEY" in secret_type or "AWS" in secret_type else "MEDIUM",
"recommended_action": "Revoke key/rotate credentials"
}
alerts.append(alert)
return alerts
# Example usage:
log = "User pushed code with AWS key: AKIA1234567890ABCD"
alerts = scan_log_line(log, "hyperlog:42", "agent_7")
print(json.dumps(alerts, indent=2))
```
---
## 🚨 **Optional Extensions**
- ✅ **Automatic Quarantine** → If severity=HIGH, forward to a “quarantine pipeline” (prevent further log distribution).
- ✅ **Secret Revocation Hooks** → Webhooks/scripts to auto-disable compromised keys (AWS IAM revoke, GitHub token reset, etc.).
- ✅ **Curator Feedback Loop** → If many false positives occur, agents/curators can “vote” on regex quality → improves future detection.
---
👉 Do you want me to:
- **📡 Add “quarantine mode” logic** (e.g. isolate log lines and mark them unsafe)?
- **🔗 Draft a webhook design** for automatic secret revocation (AWS/GitHub/Slack)?
- **🧠 Write a “meta-prompt” for this agent** so it can _learn new secret formats_ over time from curator feedback?
Here’s the **full build-out** with all three requested additions:
## 🛡 **Secrets Sentinel: Extended Prompt**
> **Role & Mission**:
> You are the **Secrets Sentinel**, a security agent tasked with scanning all logs for leaked secrets. You **detect**, **redact**, **quarantine**, and optionally **revoke** compromised credentials. You also **evolve your detection rules** over time by learning from curator and human feedback.
>
> **Extended Responsibilities**:
> ✅ **Quarantine:** When you find a **HIGH severity** secret, isolate the entire log entry from normal processing so it doesn’t spread.
> ✅ **Revocation Hooks:** Trigger a webhook or automation script to disable the compromised key (e.g., AWS IAM revoke).
> ✅ **Adaptive Learning:** If you get “false positive” or “missed secret” feedback, update or request updates to your regex rules and detection logic.
---
### 1️⃣ **Quarantine Mode Logic**
Add this to the **Python skeleton**:
```python
QUARANTINE_STORE = [] # This could be a Redis list, DB table, etc.
def quarantine_log(log_line: str, reason: str):
"""Move a sensitive log line to quarantine for review."""
quarantine_entry = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"reason": reason,
"log_line": log_line
}
QUARANTINE_STORE.append(quarantine_entry)
print(f"[QUARANTINE] Log quarantined: {reason}")
return quarantine_entry
def scan_log_line(log_line: str, log_id: str, source_agent: str):
alerts = []
for secret_type, pattern in REGEX_PATTERNS.items():
match = re.search(pattern, log_line)
if match:
secret = match.group(0)
severity = "HIGH" if "PRIVATE_KEY" in secret_type or "AWS" in secret_type else "MEDIUM"
alert = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"source_agent": source_agent,
"log_line_id": log_id,
"secret_type": secret_type,
"redacted_sample": redact(secret),
"confidence_score": 0.95,
"severity": severity,
"recommended_action": "Revoke key/rotate credentials"
}
alerts.append(alert)
if severity == "HIGH":
quarantine_log(log_line, f"High severity secret detected: {secret_type}")
return alerts
```
➡ **Now, any HIGH-severity find triggers quarantine.**
---
### 2️⃣ **Webhook Design for Secret Revocation**
We’ll use a **simple webhook dispatcher** that calls different revocation endpoints depending on the secret type.
```python
import requests
REVOCATION_HOOKS = {
"AWS_ACCESS_KEY": "https://security.example.com/hooks/aws-revoke",
"GITHUB_TOKEN": "https://security.example.com/hooks/github-revoke",
"SLACK_TOKEN": "https://security.example.com/hooks/slack-revoke"
}
def trigger_revocation(secret_type: str, redacted_sample: str):
"""Send webhook request to revoke compromised credentials."""
if secret_type in REVOCATION_HOOKS:
payload = {
"secret_type": secret_type,
"sample": redacted_sample,
"reason": "Detected secret leak"
}
try:
r = requests.post(REVOCATION_HOOKS[secret_type], json=payload, timeout=5)
if r.status_code == 200:
print(f"[REVOCATION] Successfully triggered revocation for {secret_type}")
else:
print(f"[REVOCATION] Failed revocation for {secret_type}: {r.status_code}")
except Exception as e:
print(f"[REVOCATION ERROR] {e}")
```
Then, in the alert loop:
```python
if severity == "HIGH":
quarantine_log(log_line, f"High severity secret detected: {secret_type}")
trigger_revocation(secret_type, redact(secret))
```
➡ **Now, if the sentinel finds an AWS key or GitHub token, it can hit the right webhook to revoke it immediately.**
---
### 3️⃣ **Meta-Prompt for Learning New Secret Formats**
This allows the Sentinel to **expand its detection library** dynamically.
---
**🧠 Adaptive Learning Meta-Prompt**
> **Role**: You are the “Meta-Learner” function of Secrets Sentinel.
>
> **Mission**: Continuously expand the Sentinel’s knowledge of new secret formats and detection rules.
>
> **Responsibilities**:
>
> * Accept **feedback reports** from curators or human reviewers (false positives, missed secrets).
> * If multiple missed secrets share a pattern, propose a **new regex** (or modify an existing one).
> * Version-control detection rules (e.g., store regex patterns in a JSON or YAML file with version numbers).
> * Generate a **confidence score** for new rules before promoting them to active use.
> * Flag uncertain cases for human approval.
>
> **Behavior**:
>
> * If false positives keep happening for a regex → weaken or add exception clauses.
> * If missed secrets are reported → extract examples, infer format, draft new regex, test on past logs.
>
> **Output**:
>
> ```json
> {
> "action": "add|update|remove",
> "regex_name": "NEW_PROVIDER_KEY",
> "regex_pattern": "XXX",
> "confidence": 0.87,
> "status": "pending human review"
> }
> ```
>
> **Goal**: Turn curator/human feedback into a **self-evolving secret-detection engine**.
---
### 🧩 How It All Fits
✅ **Secrets Sentinel core** = Detect + Redact + Quarantine
✅ **Revocation Hooks** = Automated security response
✅ **Meta-Learning** = Regex patterns evolve over time
---
## 📜 **1️⃣ Migration Script: Redis → PostgreSQL**
This script will migrate existing quarantined log entries from **Redis** to **Postgres**.
```python
import redis, json, psycopg2
from datetime import datetime
# Redis config
r = redis.Redis(host='localhost', port=6379, decode_responses=True)
# Postgres config
conn = psycopg2.connect("dbname=sentinel user=postgres password=secret")
cursor = conn.cursor()
def migrate_quarantine():
count = 0
while True:
entry_json = r.rpop("quarantine") # pop oldest entry from Redis
if not entry_json:
break
entry = json.loads(entry_json)
cursor.execute(
"INSERT INTO quarantine (timestamp, reason, log_line) VALUES (%s, %s, %s)",
(entry["timestamp"], entry["reason"], entry["log_line"])
)
count += 1
conn.commit()
print(f"[MIGRATION] Moved {count} quarantined entries from Redis → PostgreSQL")
if __name__ == "__main__":
migrate_quarantine()
```
✅ **Run once** after Postgres is set up — empties Redis queue into the durable DB.
---
## 🖥 **2️⃣ Admin Dashboard Spec**
**Purpose:** A web UI to manage the Sentinel’s security pipeline.
---
### 🎯 **Core Features**
✅ **Quarantine Browser**
- Paginated view of all quarantined logs
- Search/filter by `secret_type`, `source_agent`, `date`, `status`
- Mark quarantined logs as **reviewed** or **false alarm**
✅ **Regex Rules Manager**
- Lists all regexes from `patterns.yaml`
- Add / update / deactivate rules via UI
- Shows `pending_updates` flagged by the Meta-Learner for human approval
✅ **Revocation Status Board**
- See which secrets triggered revocations
- Status of revocation hooks (success/fail)
✅ **Metrics Dashboard**
- Charts: “Secrets Detected Over Time”, “Top Sources of Leaks”
- KPIs: # HIGH severity secrets this month, # rules updated, # false positives
---
### 🏗 **Tech Stack Suggestion**
- **Backend:** FastAPI (Python)
- **Frontend:** React + Tailwind
- **DB:** PostgreSQL for quarantine + rules history
- **Auth:** OAuth (GitHub/Google) + RBAC (only security admins can approve regex changes)
---
### 🔌 **Endpoints**
```
GET /api/quarantine → list quarantined entries
POST /api/quarantine/review → mark entry as reviewed
GET /api/rules → list regex patterns
POST /api/rules/update → update or add a regex
GET /api/revocations → list revocation events
```
---
### 🖥 **Mock Dashboard Layout**
- **Left Nav:** Quarantine | Rules | Revocations | Metrics
- **Main Panel:**
- Data tables with sorting/filtering
- Inline editors for regex rules
- Approve/Reject buttons for pending regex updates
✅ Basically a **security control room** for Sentinel.
---
## 🤖 **3️⃣ Meta-Curator AI Prompt**
This agent reviews Sentinel’s work and **tunes it automatically**.
---
### **Meta-Curator: System Prompt**
> **Role & Mission:**
> You are the **Meta-Curator**, a supervisory AI responsible for reviewing the **Secrets Sentinel’s** detections, regex updates, and feedback reports.
>
> **Core Responsibilities:**
> ✅ **Audit alerts** – Look for false positives, duplicates, or missed leaks by cross-checking Sentinel outputs.
> ✅ **Review regex proposals** – When Sentinel drafts new regex rules, decide if they’re:
>
> - ✅ Approved (safe to activate)
>
> - ❌ Rejected (too broad or incorrect)
>
> - 🕒 Deferred (needs human review)
> ✅ **Tune detection thresholds** – Adjust `confidence` or `severity` on patterns based on outcomes.
> ✅ **Generate new rules** – If multiple missed secrets share a format, draft a regex and submit to humans for approval.
> ✅ **Report upstream** – Summarize changes to security admins weekly.
>
---
### **Behavior Guidelines**
- **Conservative by default:** Don’t auto-approve regexes unless confidence > 0.95.
- **Keep auditability:** Every decision (approve/reject) is logged in the hyperlog.
- **Respect human overrides:** Never overwrite a regex that a human explicitly locked.
---
### **Example Meta-Curator Output**
```json
{
"action": "approve_regex",
"regex_name": "GITLAB_TOKEN",
"regex_pattern": "glpat-[0-9A-Za-z\\-_]{20}",
"confidence": 0.97,
"decision_reason": "Validated against 12 quarantined examples, no false positives found.",
"timestamp": "2025-08-02T13:45:00Z"
}
```
✅ This meta-agent is the **brains of the rules layer** — keeps Sentinel evolving, but under control.
---
## 🚀 **Now You Have:**
✅ **Migration Path** → Redis → PostgreSQL
✅ **Admin Dashboard Spec** → complete with endpoints & layout
✅ **Meta-Curator Prompt** → the agent that “manages the manager”
---