Major updates and improvements to BZZZ system

- Updated configuration and deployment files - Improved system architecture and components - Enhanced documentation and testing - Fixed various issues and added new features 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-09-17 18:06:57 +10:00
parent 4e6140de03
commit f5f96ba505
71 changed files with 664 additions and 3823 deletions
--- a/deployments/docker-old/Dockerfile.slurp-coordinator
+++ b/deployments/docker-old/Dockerfile.slurp-coordinator
@@ -0,0 +1,67 @@
+# Multi-stage build for BZZZ SLURP Coordinator
+FROM golang:1.21-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git ca-certificates tzdata make
+
+# Set working directory
+WORKDIR /build
+
+# Copy go mod files
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Build the application with optimizations
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+    -ldflags='-w -s -extldflags "-static"' \
+    -a -installsuffix cgo \
+    -o slurp-coordinator \
+    ./cmd/slurp-coordinator
+
+# Create runtime image with minimal attack surface
+FROM alpine:3.19
+
+# Install runtime dependencies
+RUN apk add --no-cache \
+    ca-certificates \
+    tzdata \
+    curl \
+    && rm -rf /var/cache/apk/*
+
+# Create application user
+RUN addgroup -g 1001 -S slurp && \
+    adduser -u 1001 -S slurp -G slurp -h /home/slurp
+
+# Set working directory
+WORKDIR /app
+
+# Copy the binary
+COPY --from=builder /build/slurp-coordinator .
+COPY --from=builder /build/config ./config
+
+# Create necessary directories
+RUN mkdir -p /app/data /app/logs /app/config && \
+    chown -R slurp:slurp /app
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:8080/health || exit 1
+
+# Switch to non-root user
+USER slurp
+
+# Expose ports
+EXPOSE 8080 9090 9091
+
+# Set entrypoint
+ENTRYPOINT ["./slurp-coordinator"]
+CMD ["--config", "config/coordinator.yaml"]
+
+# Labels
+LABEL maintainer="BZZZ Team"
+LABEL version="1.0.0"
+LABEL component="coordinator"
+LABEL description="BZZZ SLURP Coordination Service"
--- a/deployments/docker-old/Dockerfile.slurp-distributor
+++ b/deployments/docker-old/Dockerfile.slurp-distributor
@@ -0,0 +1,57 @@
+# Multi-stage build for BZZZ SLURP Context Distributor
+FROM golang:1.21-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git ca-certificates tzdata
+
+# Set working directory
+WORKDIR /build
+
+# Copy go mod files
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Build the application with optimizations
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+    -ldflags='-w -s -extldflags "-static"' \
+    -a -installsuffix cgo \
+    -o slurp-distributor \
+    ./cmd/slurp-distributor
+
+# Create minimal runtime image
+FROM scratch
+
+# Copy CA certificates and timezone data from builder
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo
+
+# Copy the binary
+COPY --from=builder /build/slurp-distributor /slurp-distributor
+
+# Create non-root user directories
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /etc/group /etc/group
+
+# Health check endpoint
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
+    CMD ["/slurp-distributor", "health"]
+
+# Expose ports
+EXPOSE 8080 9090 11434
+
+# Set entrypoint
+ENTRYPOINT ["/slurp-distributor"]
+
+# Labels for container metadata
+LABEL maintainer="BZZZ Team"
+LABEL version="1.0.0"
+LABEL description="BZZZ SLURP Distributed Context System"
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.name="slurp-distributor"
+LABEL org.label-schema.description="Enterprise-grade distributed context distribution system"
+LABEL org.label-schema.url="https://github.com/anthonyrawlins/bzzz"
+LABEL org.label-schema.vcs-url="https://github.com/anthonyrawlins/bzzz"
+LABEL org.label-schema.build-date="2024-01-01T00:00:00Z"
--- a/deployments/docker-old/docker-compose.yml
+++ b/deployments/docker-old/docker-compose.yml
@@ -0,0 +1,328 @@
+# BZZZ SLURP Distributed Context Distribution - Development Environment
+version: '3.8'
+
+x-common-variables: &common-env
+  - LOG_LEVEL=info
+  - ENVIRONMENT=development
+  - CLUSTER_NAME=bzzz-slurp-dev
+  - NETWORK_MODE=p2p
+
+x-common-volumes: &common-volumes
+  - ./config:/app/config:ro
+  - ./data:/app/data
+  - ./logs:/app/logs
+
+services:
+  # SLURP Coordinator - Central coordination service
+  slurp-coordinator:
+    build:
+      context: ../..
+      dockerfile: deployments/docker/Dockerfile.slurp-coordinator
+    container_name: slurp-coordinator
+    hostname: coordinator.bzzz.local
+    restart: unless-stopped
+    environment:
+      <<: *common-env
+      - ROLE=coordinator
+      - NODE_ID=coord-01
+      - MONITORING_PORT=9091
+      - DHT_BOOTSTRAP_PEERS=distributor-01:11434,distributor-02:11434
+    volumes: *common-volumes
+    ports:
+      - "8080:8080"   # HTTP API
+      - "9091:9091"   # Metrics
+    networks:
+      - bzzz-slurp
+    depends_on:
+      - prometheus
+      - grafana
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+
+  # SLURP Distributors - Context distribution nodes
+  slurp-distributor-01:
+    build:
+      context: ../..
+      dockerfile: deployments/docker/Dockerfile.slurp-distributor
+    container_name: slurp-distributor-01
+    hostname: distributor-01.bzzz.local
+    restart: unless-stopped
+    environment:
+      <<: *common-env
+      - ROLE=distributor
+      - NODE_ID=dist-01
+      - COORDINATOR_ENDPOINT=http://slurp-coordinator:8080
+      - DHT_PORT=11434
+      - REPLICATION_FACTOR=3
+    volumes: *common-volumes
+    ports:
+      - "8081:8080"   # HTTP API
+      - "11434:11434" # DHT P2P
+      - "9092:9090"   # Metrics
+    networks:
+      - bzzz-slurp
+    depends_on:
+      - slurp-coordinator
+    healthcheck:
+      test: ["CMD", "/slurp-distributor", "health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
+  slurp-distributor-02:
+    build:
+      context: ../..
+      dockerfile: deployments/docker/Dockerfile.slurp-distributor
+    container_name: slurp-distributor-02
+    hostname: distributor-02.bzzz.local
+    restart: unless-stopped
+    environment:
+      <<: *common-env
+      - ROLE=distributor
+      - NODE_ID=dist-02
+      - COORDINATOR_ENDPOINT=http://slurp-coordinator:8080
+      - DHT_PORT=11434
+      - REPLICATION_FACTOR=3
+      - DHT_BOOTSTRAP_PEERS=slurp-distributor-01:11434
+    volumes: *common-volumes
+    ports:
+      - "8082:8080"   # HTTP API
+      - "11435:11434" # DHT P2P
+      - "9093:9090"   # Metrics
+    networks:
+      - bzzz-slurp
+    depends_on:
+      - slurp-coordinator
+      - slurp-distributor-01
+    healthcheck:
+      test: ["CMD", "/slurp-distributor", "health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
+  slurp-distributor-03:
+    build:
+      context: ../..
+      dockerfile: deployments/docker/Dockerfile.slurp-distributor
+    container_name: slurp-distributor-03
+    hostname: distributor-03.bzzz.local
+    restart: unless-stopped
+    environment:
+      <<: *common-env
+      - ROLE=distributor
+      - NODE_ID=dist-03
+      - COORDINATOR_ENDPOINT=http://slurp-coordinator:8080
+      - DHT_PORT=11434
+      - REPLICATION_FACTOR=3
+      - DHT_BOOTSTRAP_PEERS=slurp-distributor-01:11434,slurp-distributor-02:11434
+    volumes: *common-volumes
+    ports:
+      - "8083:8080"   # HTTP API
+      - "11436:11434" # DHT P2P
+      - "9094:9090"   # Metrics
+    networks:
+      - bzzz-slurp
+    depends_on:
+      - slurp-coordinator
+      - slurp-distributor-01
+      - slurp-distributor-02
+    healthcheck:
+      test: ["CMD", "/slurp-distributor", "health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
+  # Prometheus - Metrics collection
+  prometheus:
+    image: prom/prometheus:v2.48.0
+    container_name: slurp-prometheus
+    hostname: prometheus.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    networks:
+      - bzzz-slurp
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=15d'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+      - '--web.enable-admin-api'
+
+  # Grafana - Metrics visualization
+  grafana:
+    image: grafana/grafana:10.2.2
+    container_name: slurp-grafana
+    hostname: grafana.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin123
+      - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_SERVER_ROOT_URL=http://localhost:3000
+      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards:ro
+      - ./grafana/datasources:/etc/grafana/provisioning/datasources:ro
+    networks:
+      - bzzz-slurp
+    depends_on:
+      - prometheus
+
+  # Redis - Shared state and caching
+  redis:
+    image: redis:7.2-alpine
+    container_name: slurp-redis
+    hostname: redis.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
+      - ./redis.conf:/usr/local/etc/redis/redis.conf:ro
+    networks:
+      - bzzz-slurp
+    command: redis-server /usr/local/etc/redis/redis.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # MinIO - Object storage for large contexts
+  minio:
+    image: minio/minio:RELEASE.2023-12-23T07-19-11Z
+    container_name: slurp-minio
+    hostname: minio.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=admin123456
+      - MINIO_REGION_NAME=us-east-1
+    volumes:
+      - minio-data:/data
+    networks:
+      - bzzz-slurp
+    command: server /data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # Jaeger - Distributed tracing
+  jaeger:
+    image: jaegertracing/all-in-one:1.51
+    container_name: slurp-jaeger
+    hostname: jaeger.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "14268:14268"   # HTTP collector
+      - "16686:16686"   # Web UI
+      - "6831:6831/udp" # Agent UDP
+      - "6832:6832/udp" # Agent UDP
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+      - COLLECTOR_ZIPKIN_HOST_PORT=:9411
+    volumes:
+      - jaeger-data:/tmp
+    networks:
+      - bzzz-slurp
+
+  # ElasticSearch - Log storage and search
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.3
+    container_name: slurp-elasticsearch
+    hostname: elasticsearch.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "9200:9200"
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    volumes:
+      - elasticsearch-data:/usr/share/elasticsearch/data
+    networks:
+      - bzzz-slurp
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:9200/_health || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
+  # Kibana - Log visualization
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.11.3
+    container_name: slurp-kibana
+    hostname: kibana.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "5601:5601"
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - SERVER_HOST=0.0.0.0
+    networks:
+      - bzzz-slurp
+    depends_on:
+      - elasticsearch
+
+  # Load Balancer
+  nginx:
+    image: nginx:1.25-alpine
+    container_name: slurp-nginx
+    hostname: nginx.bzzz.local
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./ssl:/etc/nginx/ssl:ro
+    networks:
+      - bzzz-slurp
+    depends_on:
+      - slurp-coordinator
+      - slurp-distributor-01
+      - slurp-distributor-02
+      - slurp-distributor-03
+
+networks:
+  bzzz-slurp:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.20.0.0/16
+    name: bzzz-slurp-network
+
+volumes:
+  prometheus-data:
+    driver: local
+  grafana-data:
+    driver: local
+  redis-data:
+    driver: local
+  minio-data:
+    driver: local
+  jaeger-data:
+    driver: local
+  elasticsearch-data:
+    driver: local