Major updates and improvements to BZZZ system

- Updated configuration and deployment files - Improved system architecture and components - Enhanced documentation and testing - Fixed various issues and added new features 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-09-17 18:06:57 +10:00
parent 4e6140de03
commit f5f96ba505
71 changed files with 664 additions and 3823 deletions
--- a/deployments/dockerized-BZZZ/Dockerfile
+++ b/deployments/dockerized-BZZZ/Dockerfile
@@ -0,0 +1,71 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates tzdata curl locales gettext-base systemd systemd-sysv && \
+    rm -rf /var/lib/apt/lists/*
+
+# Configure systemd for container use
+RUN cd /lib/systemd/system/sysinit.target.wants/ && \
+    ls | grep -v systemd-tmpfiles-setup | xargs rm -f && \
+    rm -f /lib/systemd/system/multi-user.target.wants/* && \
+    rm -f /etc/systemd/system/*.wants/* && \
+    rm -f /lib/systemd/system/local-fs.target.wants/* && \
+    rm -f /lib/systemd/system/sockets.target.wants/*udev* && \
+    rm -f /lib/systemd/system/sockets.target.wants/*initctl* && \
+    rm -f /lib/systemd/system/basic.target.wants/* && \
+    rm -f /lib/systemd/system/anaconda.target.wants/*
+
+# Create bzzz directories
+RUN mkdir -p /opt/bzzz /opt/bzzz/.bzzz /etc/systemd/system
+
+# BZZZ binary
+COPY ./build/bzzz /opt/bzzz/bzzz
+RUN chmod +x /opt/bzzz/bzzz
+
+# Config template
+COPY ./config.yml.tmpl /opt/bzzz/.bzzz/config.yml.tmpl
+
+# Create systemd service file
+RUN cat > /etc/systemd/system/bzzz.service << 'EOF'
+[Unit]
+Description=BZZZ P2P Task Coordination System
+After=network.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/opt/bzzz
+ExecStart=/opt/bzzz/bzzz -config /opt/bzzz/.bzzz/config.yml
+Restart=always
+RestartSec=10
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=bzzz
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Enable the service
+RUN systemctl enable bzzz.service
+
+# Create startup script that renders config and starts systemd
+RUN cat > /opt/bzzz/startup.sh << 'EOF'
+#!/bin/bash
+set -e
+
+# Render config from template
+envsubst < /opt/bzzz/.bzzz/config.yml.tmpl > /opt/bzzz/.bzzz/config.yml
+
+# Start systemd
+exec /lib/systemd/systemd
+EOF
+
+RUN chmod +x /opt/bzzz/startup.sh
+
+# Working directory
+WORKDIR /opt/bzzz
+
+# Use systemd as init system
+ENTRYPOINT ["/opt/bzzz/startup.sh"]
--- a/deployments/dockerized-BZZZ/Dockerfile.minimal
+++ b/deployments/dockerized-BZZZ/Dockerfile.minimal
@@ -0,0 +1,69 @@
+# Minimal BZZZ Docker container without systemd
+# Uses multi-stage build for smaller final image
+FROM golang:1.21-alpine AS builder
+
+WORKDIR /build
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags='-w -s -extldflags "-static"' -o bzzz .
+
+# Final minimal image
+FROM alpine:3.18
+
+# Install only essential packages
+RUN apk --no-cache add \
+    ca-certificates \
+    tzdata \
+    curl
+
+# Create non-root user for security
+RUN addgroup -g 1000 bzzz && \
+    adduser -u 1000 -G bzzz -s /bin/sh -D bzzz
+
+# Create required directories
+RUN mkdir -p /app/data /app/config /app/logs && \
+    chown -R bzzz:bzzz /app
+
+# Copy binary from builder stage
+COPY --from=builder /build/bzzz /app/bzzz
+RUN chmod +x /app/bzzz
+
+# Copy config template
+COPY dockerize/config.minimal.yml.tmpl /app/config/config.yml.tmpl
+
+# Create entrypoint script that handles config generation
+RUN cat > /app/entrypoint.sh << 'EOF'
+#!/bin/sh
+set -e
+
+# Generate config from template if it doesn't exist
+if [ ! -f /app/config/config.yml ]; then
+    echo "🔧 Generating configuration from template..."
+    envsubst < /app/config/config.yml.tmpl > /app/config/config.yml
+fi
+
+# Ensure proper ownership
+chown -R bzzz:bzzz /app/data /app/config /app/logs
+
+echo "🚀 Starting BZZZ..."
+exec "$@"
+EOF
+
+RUN chmod +x /app/entrypoint.sh
+
+# Switch to non-root user
+USER bzzz
+WORKDIR /app
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8081/health || exit 1
+
+# Expose ports
+EXPOSE 8080 8081 9000-9100
+
+# Set entrypoint and default command
+ENTRYPOINT ["/app/entrypoint.sh"]
+CMD ["/app/bzzz", "--config", "/app/config/config.yml"]
--- a/deployments/dockerized-BZZZ/build-minimal.sh
+++ b/deployments/dockerized-BZZZ/build-minimal.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+# Build script for minimal BZZZ container
+
+echo "🐳 Building minimal BZZZ container..."
+
+# Set build context to parent directory (BZZZ root)
+cd "$(dirname "$0")/.."
+
+# Build the minimal container
+docker build -f dockerize/Dockerfile.minimal -t bzzz:minimal .
+
+echo "✅ BZZZ minimal container built successfully!"
+echo ""
+echo "Usage:"
+echo "  # Start with default configuration:"
+echo "  docker-compose -f dockerize/docker-compose.minimal.yml up -d"
+echo ""
+echo "  # Start with custom environment:"
+echo "  cp dockerize/bzzz.minimal.env.example dockerize/bzzz.minimal.env"
+echo "  # Edit dockerize/bzzz.minimal.env with your settings"
+echo "  docker-compose -f dockerize/docker-compose.minimal.yml --env-file dockerize/bzzz.minimal.env up -d"
+echo ""
+echo "  # Check logs:"
+echo "  docker-compose -f dockerize/docker-compose.minimal.yml logs -f"
+echo ""
+echo "  # Check health:"
+echo "  curl http://localhost:8081/health"
--- a/deployments/dockerized-BZZZ/bzzz.env
+++ b/deployments/dockerized-BZZZ/bzzz.env
@@ -0,0 +1,14 @@
+# Human-readable hint in rendered config header
+NODE_HINT=swarm
+
+# Unique id (defaults to $HOSTNAME if unset)
+# AGENT_ID=
+
+# Ollama endpoint (per-node host)
+OLLAMA_BASE_URL=http://host.docker.internal:11434
+
+# Log level: debug|info|warn|error
+LOG_LEVEL=info
+
+# UCXL storage path inside the container (persisted under /var/lib/bzzz)
+UCXL_DIR=/var/lib/bzzz/ucxl
--- a/deployments/dockerized-BZZZ/bzzz.minimal.env.example
+++ b/deployments/dockerized-BZZZ/bzzz.minimal.env.example
@@ -0,0 +1,32 @@
+# BZZZ Minimal Container Configuration
+# Copy this file to bzzz.minimal.env and customize as needed
+
+# Basic Agent Configuration
+BZZZ_AGENT_ID=bzzz-docker-01
+BZZZ_SPECIALIZATION=general_developer
+BZZZ_MAX_TASKS=3
+
+# Network Ports (adjust if ports are already in use)
+BZZZ_P2P_PORT=9000
+BZZZ_API_PORT=8080
+BZZZ_HEALTH_PORT=8081
+
+# Logging
+LOG_LEVEL=info
+# DEBUG=1  # Uncomment to enable debug logging
+
+# DHT and P2P Settings
+BZZZ_DHT_ENABLED=true
+# BZZZ_BOOTSTRAP_PEERS=  # Comma-separated list of bootstrap peers
+
+# AI Configuration
+OLLAMA_ENDPOINT=http://host.docker.internal:11434
+
+# Licensing (if required)
+# LICENSE_EMAIL=your.email@example.com
+# LICENSE_KEY=your-license-key-here
+CLUSTER_ID=docker-cluster
+
+# Optional: Override default resource limits in docker-compose.minimal.yml
+# MEMORY_LIMIT=1G
+# CPU_LIMIT=1.0
--- a/deployments/dockerized-BZZZ/config.minimal.yml.tmpl
+++ b/deployments/dockerized-BZZZ/config.minimal.yml.tmpl
@@ -0,0 +1,91 @@
+# BZZZ Configuration for Container Deployment
+# Environment variables will be substituted at runtime
+
+agent:
+  id: "${BZZZ_AGENT_ID}"
+  specialization: "${BZZZ_SPECIALIZATION}"
+  max_tasks: ${BZZZ_MAX_TASKS}
+  capabilities:
+    - "general_development"
+    - "task_coordination"
+    - "p2p_collaboration"
+  models:
+    - "llama3.1:8b"
+    - "codellama:7b"
+  role: ""  # Will be auto-assigned based on specialization
+  expertise: []
+  reports_to: ""
+
+network:
+  p2p:
+    listen_port: ${BZZZ_P2P_PORT}
+    bind_address: "0.0.0.0"
+  api:
+    port: ${BZZZ_API_PORT}
+    bind_address: "0.0.0.0"
+  health:
+    port: ${BZZZ_HEALTH_PORT}
+    bind_address: "0.0.0.0"
+
+# DHT configuration for peer discovery
+v2:
+  dht:
+    enabled: ${BZZZ_DHT_ENABLED}
+    bootstrap_peers: []  # Will be populated from BZZZ_BOOTSTRAP_PEERS env var
+
+# AI configuration
+ai:
+  ollama:
+    endpoint: "${OLLAMA_ENDPOINT}"
+    timeout: "30s"
+
+# UCXL protocol configuration
+ucxl:
+  enabled: true
+  server:
+    enabled: true
+    port: 8082
+    base_path: ""
+  storage:
+    directory: "/tmp/bzzz-ucxi-storage"
+  resolution:
+    cache_ttl: "1h"
+
+# Licensing configuration (if required)
+license:
+  email: "${LICENSE_EMAIL}"
+  license_key: "${LICENSE_KEY}"
+  cluster_id: "${CLUSTER_ID}"
+  organization_name: ""
+  kaching_url: "https://kaching.chorus.services"
+  is_active: false
+  grace_period_hours: 72
+  license_type: ""
+  max_nodes: 1
+
+# Binary type for specialized behavior
+binary_type: "agent"
+
+# Repository integration (disabled in container mode)
+repository:
+  provider: ""
+  url: ""
+  token: ""
+  webhook_url: ""
+
+# Security settings optimized for containers
+security:
+  enable_auth: false
+  auth_token: ""
+  
+# Storage paths for container environment
+storage:
+  data_directory: "/app/data"
+  config_directory: "/app/config"
+  log_directory: "/app/logs"
+  
+# Logging configuration for containers (stdout/stderr)
+logging:
+  level: "${LOG_LEVEL}"
+  format: "structured"  # Better for container log collection
+  output: "stdout"      # Force stdout for container compatibility
--- a/deployments/dockerized-BZZZ/config.yml.tmpl
+++ b/deployments/dockerized-BZZZ/config.yml.tmpl
@@ -0,0 +1,136 @@
+# BZZZ Configuration for ${NODE_HINT:-container}
+whoosh_api:
+  base_url: "https://whoosh.home.deepblack.cloud"
+  api_key: ""
+  timeout: 30s
+  retry_count: 3
+
+agent:
+  id: "${AGENT_ID:-${HOSTNAME}}"
+  capabilities: ["general"]
+  poll_interval: 30s
+  max_tasks: 2
+  models: []
+  specialization: ""
+  model_selection_webhook: ""
+  default_reasoning_model: ""
+  sandbox_image: ""
+  role: ""
+  system_prompt: ""
+  reports_to: []
+  expertise: []
+  deliverables: []
+  collaboration:
+    preferred_message_types: []
+    auto_subscribe_to_roles: []
+    auto_subscribe_to_expertise: []
+    response_timeout_seconds: 0
+    max_collaboration_depth: 0
+    escalation_threshold: 0
+    custom_topic_subscriptions: []
+
+github:
+  token_file: ""
+  user_agent: "BZZZ-Agent/1.0"
+  timeout: 30s
+  rate_limit: true
+  assignee: ""
+
+p2p:
+  service_tag: "bzzz-peer-discovery"
+  bzzz_topic: "bzzz/coordination/v1"
+  hmmm_topic: "hmmm/meta-discussion/v1"
+  discovery_timeout: 10s
+  escalation_webhook: ""
+  escalation_keywords: []
+  conversation_limit: 10
+
+logging:
+  level: "${LOG_LEVEL:-info}"
+  format: "text"
+  output: "stdout"
+  structured: false
+
+slurp:
+  enabled: false
+  base_url: ""
+  api_key: ""
+  timeout: 30s
+  retry_count: 3
+  max_concurrent_requests: 10
+  request_queue_size: 100
+
+v2:
+  enabled: false
+  protocol_version: "2.0.0"
+  uri_resolution:
+    cache_ttl: 5m0s
+    max_peers_per_result: 5
+    default_strategy: "best_match"
+    resolution_timeout: 30s
+  dht:
+    enabled: false
+    bootstrap_peers: []
+    mode: "auto"
+    protocol_prefix: "/bzzz"
+    bootstrap_timeout: 30s
+    discovery_interval: 1m0s
+    auto_bootstrap: false
+  semantic_addressing:
+    enable_wildcards: true
+    default_agent: "any"
+    default_role: "any"
+    default_project: "any"
+    enable_role_hierarchy: true
+  feature_flags:
+    uri_protocol: false
+    semantic_addressing: false
+    dht_discovery: false
+    advanced_resolution: false
+
+ucxl:
+  enabled: false
+  server:
+    port: 8081
+    base_path: "/bzzz"
+    enabled: false
+  resolution:
+    cache_ttl: 5m0s
+    enable_wildcards: true
+    max_results: 50
+  storage:
+    type: "filesystem"
+    directory: "${UCXL_DIR:/var/lib/bzzz/ucxl}"
+    max_size: 104857600
+  p2p_integration:
+    enable_announcement: false
+    enable_discovery: false
+    announcement_topic: "bzzz/ucxl/announcement/v1"
+    discovery_timeout: 30s
+
+security:
+  admin_key_shares:
+    threshold: 3
+    total_shares: 5
+  election_config:
+    heartbeat_timeout: 5s
+    discovery_timeout: 30s
+    election_timeout: 15s
+    max_discovery_attempts: 6
+    discovery_backoff: 5s
+    minimum_quorum: 3
+    consensus_algorithm: "raft"
+    split_brain_detection: true
+    conflict_resolution: "highest_uptime"
+  key_rotation_days: 90
+  audit_logging: false
+  audit_path: ""
+
+ai:
+  ollama:
+    endpoint: "${OLLAMA_BASE_URL:-http://host.docker.internal:11434}"
+    timeout: 30s
+    models: []
+  openai:
+    api_key: ""
+    endpoint: "https://api.openai.com/v1"
--- a/deployments/dockerized-BZZZ/docker-compose.minimal.yml
+++ b/deployments/dockerized-BZZZ/docker-compose.minimal.yml
@@ -0,0 +1,112 @@
+version: "3.9"
+
+services:
+  bzzz-minimal:
+    image: bzzz:minimal
+    build:
+      context: ..
+      dockerfile: dockerize/Dockerfile.minimal
+    environment:
+      # Basic BZZZ configuration
+      - BZZZ_AGENT_ID=${BZZZ_AGENT_ID:-bzzz-docker-01}
+      - BZZZ_SPECIALIZATION=${BZZZ_SPECIALIZATION:-general_developer}
+      - BZZZ_MAX_TASKS=${BZZZ_MAX_TASKS:-3}
+      
+      # Network configuration  
+      - BZZZ_P2P_PORT=${BZZZ_P2P_PORT:-9000}
+      - BZZZ_API_PORT=${BZZZ_API_PORT:-8080}
+      - BZZZ_HEALTH_PORT=${BZZZ_HEALTH_PORT:-8081}
+      
+      # Logging configuration
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - DEBUG=${DEBUG:-}
+      
+      # DHT and P2P settings
+      - BZZZ_DHT_ENABLED=${BZZZ_DHT_ENABLED:-true}
+      - BZZZ_BOOTSTRAP_PEERS=${BZZZ_BOOTSTRAP_PEERS:-}
+      
+      # AI/Ollama configuration
+      - OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT:-http://host.docker.internal:11434}
+      
+      # Licensing (if required)
+      - LICENSE_EMAIL=${LICENSE_EMAIL:-}
+      - LICENSE_KEY=${LICENSE_KEY:-}
+      - CLUSTER_ID=${CLUSTER_ID:-docker-cluster}
+      
+    # Persist data across container restarts
+    volumes:
+      - bzzz_data:/app/data
+      - bzzz_config:/app/config
+      - type: bind
+        source: /tmp/bzzz-ucxi-storage
+        target: /tmp/bzzz-ucxi-storage
+      - type: bind  
+        source: /tmp/hcfs-workspaces
+        target: /tmp/hcfs-workspaces
+
+    # Network ports
+    ports:
+      - "${BZZZ_API_PORT:-8080}:8080"      # HTTP API
+      - "${BZZZ_HEALTH_PORT:-8081}:8081"   # Health check
+      - "${BZZZ_P2P_PORT:-9000}:9000"      # P2P communication
+
+    # Container resource limits
+    deploy:
+      mode: replicated
+      replicas: 1
+      update_config:
+        order: start-first
+        parallelism: 1
+        failure_action: rollback
+      restart_policy:
+        condition: on-failure
+        delay: 10s
+        max_attempts: 3
+      resources:
+        limits:
+          cpus: "1.0"
+          memory: 1G
+        reservations:
+          cpus: "0.25"
+          memory: 256M
+
+    # Network configuration
+    networks:
+      - bzzz_net
+
+    # Host resolution for connecting to host services
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+    # Logging configuration for container runtime
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+        labels: "service=bzzz"
+
+    # Health check
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8081/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+# Named volumes for persistence
+volumes:
+  bzzz_data:
+    driver: local
+  bzzz_config:
+    driver: local
+
+# Network for BZZZ communication
+networks:
+  bzzz_net:
+    driver: overlay
+    attachable: true
+    ipam:
+      driver: default
+      config:
+        - subnet: 10.200.0.0/24
--- a/deployments/dockerized-BZZZ/docker-compose.yml
+++ b/deployments/dockerized-BZZZ/docker-compose.yml
@@ -0,0 +1,62 @@
+version: "3.9"
+
+services:
+  bzzz:
+    image: bzzz:latest
+    env_file:
+      - bzzz.env
+    # Persist identity/state per node
+    volumes:
+    - type: bind
+      source: /var/lib/bzzz
+      target: /var/lib/bzzz
+    - type: bind
+      source: /tmp/bzzz-ucxl-storage
+      target: /tmp/bzzz-ucxl-storage
+    - type: bind
+      source: /tmp/bzzz-ucxi-storage
+      target: /tmp/bzzz-ucxi-storage
+    - type: bind
+      source: /tmp/hcfs-workspaces
+      target: /tmp/hcfs-workspaces
+
+    # If you later enable ucxl.server.enabled: true and need to expose it:
+    ports:
+       - target: 8081
+         published: 8081
+         protocol: tcp
+         mode: host
+
+    deploy:
+      mode: replicated
+      replicas: 3
+      update_config:
+        order: start-first
+        parallelism: 1
+        failure_action: rollback
+      restart_policy:
+        condition: on-failure
+      resources:
+        limits:
+          cpus: "1.0"
+          memory: 2G
+        reservations:
+          cpus: "0.25"
+          memory: 512M
+      placement:
+        preferences:
+          - spread: node.id
+
+    networks:
+      - bzzz_net
+
+    # Lets the container resolve the node's host at host.docker.internal
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+networks:
+  bzzz_net:
+    driver: overlay
+    attachable: true
+    # driver_opts:
+    #   encrypted: "true"
--- a/deployments/dockerized-BZZZ/logging.go
+++ b/deployments/dockerized-BZZZ/logging.go
@@ -0,0 +1,48 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+// ContainerLogger provides structured logging for containers
+// All output goes to stdout/stderr for container runtime collection
+type ContainerLogger struct {
+	name string
+}
+
+// NewContainerLogger creates a new container-friendly logger
+func NewContainerLogger(name string) *ContainerLogger {
+	return &ContainerLogger{name: name}
+}
+
+// Info logs informational messages to stdout
+func (l *ContainerLogger) Info(msg string, args ...interface{}) {
+	timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
+	logMsg := fmt.Sprintf(msg, args...)
+	fmt.Fprintf(os.Stdout, "[%s] [INFO] [%s] %s\n", timestamp, l.name, logMsg)
+}
+
+// Warn logs warning messages to stdout
+func (l *ContainerLogger) Warn(msg string, args ...interface{}) {
+	timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
+	logMsg := fmt.Sprintf(msg, args...)
+	fmt.Fprintf(os.Stdout, "[%s] [WARN] [%s] %s\n", timestamp, l.name, logMsg)
+}
+
+// Error logs error messages to stderr
+func (l *ContainerLogger) Error(msg string, args ...interface{}) {
+	timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
+	logMsg := fmt.Sprintf(msg, args...)
+	fmt.Fprintf(os.Stderr, "[%s] [ERROR] [%s] %s\n", timestamp, l.name, logMsg)
+}
+
+// Debug logs debug messages to stdout (only if DEBUG env var is set)
+func (l *ContainerLogger) Debug(msg string, args ...interface{}) {
+	if os.Getenv("DEBUG") != "" {
+		timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
+		logMsg := fmt.Sprintf(msg, args...)
+		fmt.Fprintf(os.Stdout, "[%s] [DEBUG] [%s] %s\n", timestamp, l.name, logMsg)
+	}
+}