diff --git a/README.md b/README.md index b01b8ce..0dea164 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ # Initialize submodules (first time only) ./chorus.sh init +# Login to Docker registry +./chorus.sh login + +# Build and push images to registry +./chorus.sh build + # Start all services ./chorus.sh start @@ -83,13 +89,22 @@ The `./chorus.sh` script provides unified management: ```bash # Service Management -./chorus.sh start # Start all services +./chorus.sh start # Start all services (registry images) ./chorus.sh stop # Stop all services ./chorus.sh restart # Restart all services ./chorus.sh status # Show service status +./chorus.sh dev # Start in development mode (local builds) + +# Docker Registry Operations +./chorus.sh login # Login to Docker registry +./chorus.sh build # Build and push all images to registry +./chorus.sh pull # Pull latest images from registry + +# Production Deployment +./chorus.sh deploy # Deploy to Docker Swarm (production) +./chorus.sh undeploy # Remove from Docker Swarm # Development & Maintenance -./chorus.sh build # Build all docker images ./chorus.sh update # Update submodules to latest ./chorus.sh logs [service] # View logs ./chorus.sh health # Check service health @@ -176,7 +191,10 @@ cd chorus.services # Or initialize submodules if already cloned ./chorus.sh init -# Build all images +# Login to Docker registry +./chorus.sh login + +# Build and push all images to registry ./chorus.sh build # Start services @@ -185,9 +203,18 @@ cd chorus.services ### Development Workflow ```bash +# Development mode (local builds, live reloading) +./chorus.sh dev + # Update submodules to latest ./chorus.sh update +# Rebuild and push after changes +./chorus.sh build + +# Pull latest images from registry +./chorus.sh pull + # View logs during development ./chorus.sh logs @@ -198,6 +225,17 @@ cd chorus.services ./chorus.sh restart ``` +### Production Deployment +```bash +# Deploy to Docker Swarm (production) +./chorus.sh deploy + +# Access at https://*.home.deepblack.cloud endpoints + +# Remove from swarm +./chorus.sh undeploy +``` + ## 🚀 Git Submodules Guide CHORUS Services uses git submodules to integrate independent components: diff --git a/build-and-push.sh b/build-and-push.sh new file mode 100755 index 0000000..1aa0222 --- /dev/null +++ b/build-and-push.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +# CHORUS Services - Build and Push Script +# Builds all Docker images and pushes them to the local registry + +set -e + +REGISTRY="registry.home.deepblack.cloud" +REGISTRY_USER="tony" +REGISTRY_PASS="silverfrond[1392]" +TAG="latest" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Print colored output +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Docker registry login +docker_login() { + print_info "Logging into Docker registry: $REGISTRY" + echo "$REGISTRY_PASS" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin + print_success "Successfully logged into registry" +} + +# Build and push a single image +build_and_push() { + local name=$1 + local context=$2 + local dockerfile=${3:-"Dockerfile"} + + local image_name="$REGISTRY/tony/chorus-${name}:$TAG" + + print_info "Building $name from $context" + + if [ -f "$context/$dockerfile" ]; then + docker build -t "$image_name" -f "$context/$dockerfile" "$context" + print_success "Built $image_name" + + print_info "Pushing $image_name to registry" + docker push "$image_name" + print_success "Pushed $image_name" + else + print_error "Dockerfile not found: $context/$dockerfile" + return 1 + fi +} + +# Main build process +main() { + print_info "Starting CHORUS Services build and push process" + + # Login to registry + docker_login + + # Build and push all images + print_info "Building all CHORUS Services components..." + + # WHOOSH Backend + if [ -d "modules/whoosh/backend" ]; then + build_and_push "whoosh-backend" "modules/whoosh/backend" + else + print_warning "WHOOSH backend directory not found - skipping" + fi + + # WHOOSH Frontend + if [ -d "modules/whoosh/frontend" ]; then + build_and_push "whoosh-frontend" "modules/whoosh/frontend" + else + print_warning "WHOOSH frontend directory not found - skipping" + fi + + # BZZZ Coordinator + if [ -d "modules/bzzz" ]; then + build_and_push "bzzz-coordinator" "modules/bzzz" + else + print_warning "BZZZ directory not found - skipping" + fi + + # SLURP API + if [ -d "modules/slurp/hcfs-python" ]; then + build_and_push "slurp-api" "modules/slurp/hcfs-python" + else + print_warning "SLURP API directory not found - skipping" + fi + + # SLURP RL Tuner + if [ -d "modules/slurp" ] && [ -f "modules/slurp/Dockerfile.rl-tuner" ]; then + build_and_push "slurp-rl-tuner" "modules/slurp" "Dockerfile.rl-tuner" + else + print_warning "SLURP RL Tuner dockerfile not found - skipping" + fi + + print_success "All CHORUS Services images built and pushed successfully!" + + # List pushed images + print_info "Pushed images:" + echo " - $REGISTRY/tony/chorus-whoosh-backend:$TAG" + echo " - $REGISTRY/tony/chorus-whoosh-frontend:$TAG" + echo " - $REGISTRY/tony/chorus-bzzz-coordinator:$TAG" + echo " - $REGISTRY/tony/chorus-slurp-api:$TAG" + echo " - $REGISTRY/tony/chorus-slurp-rl-tuner:$TAG" +} + +# Handle command line arguments +case "${1:-}" in + "login") + docker_login + ;; + "whoosh-backend") + docker_login + build_and_push "whoosh-backend" "modules/whoosh/backend" + ;; + "whoosh-frontend") + docker_login + build_and_push "whoosh-frontend" "modules/whoosh/frontend" + ;; + "bzzz") + docker_login + build_and_push "bzzz-coordinator" "modules/bzzz" + ;; + "slurp-api") + docker_login + build_and_push "slurp-api" "modules/slurp/hcfs-python" + ;; + "slurp-rl-tuner") + docker_login + build_and_push "slurp-rl-tuner" "modules/slurp" "Dockerfile.rl-tuner" + ;; + "help"|"-h"|"--help") + echo "CHORUS Services Build and Push Script" + echo "" + echo "Usage: $0 [COMMAND]" + echo "" + echo "Commands:" + echo " (no args) Build and push all images" + echo " login Login to Docker registry only" + echo " whoosh-backend Build and push WHOOSH backend only" + echo " whoosh-frontend Build and push WHOOSH frontend only" + echo " bzzz Build and push BZZZ coordinator only" + echo " slurp-api Build and push SLURP API only" + echo " slurp-rl-tuner Build and push SLURP RL Tuner only" + echo " help Show this help message" + ;; + *) + main + ;; +esac \ No newline at end of file diff --git a/chorus.sh b/chorus.sh index 8bf6654..b051a96 100755 --- a/chorus.sh +++ b/chorus.sh @@ -45,11 +45,16 @@ show_usage() { echo " status Show status of all services" echo " logs Show logs from all services" echo " logs [service] Show logs from specific service" - echo " build Build all docker images" + echo " build Build and push all docker images to registry" + echo " pull Pull latest images from registry" echo " update Update all submodules to latest" echo " init Initialize submodules (first time setup)" echo " clean Clean up docker resources" echo " health Check health of all services" + echo " login Login to Docker registry" + echo " deploy Deploy to Docker Swarm (production)" + echo " undeploy Remove from Docker Swarm" + echo " dev Start in development mode with local builds" echo "" echo "Services:" echo " - whoosh-backend : Orchestration API (port 8087)" @@ -78,27 +83,24 @@ update_submodules() { print_success "Submodules updated" } -# Build all docker images +# Build and push all docker images build_images() { - print_info "Building CHORUS Services docker images..." - - # Build each service - print_info "Building WHOOSH backend..." - docker-compose build whoosh-backend - - print_info "Building WHOOSH frontend..." - docker-compose build whoosh-frontend - - print_info "Building BZZZ coordinator..." - docker-compose build bzzz-coordinator - - print_info "Building SLURP API..." - docker-compose build slurp-api - - print_info "Building SLURP RL Tuner..." - docker-compose build slurp-rl-tuner - - print_success "All images built successfully" + print_info "Building and pushing CHORUS Services docker images to registry..." + ./build-and-push.sh + print_success "All images built and pushed successfully" +} + +# Pull latest images from registry +pull_images() { + print_info "Pulling latest CHORUS Services images from registry..." + docker-compose pull + print_success "All images pulled successfully" +} + +# Login to Docker registry +docker_login() { + print_info "Logging into Docker registry..." + ./build-and-push.sh login } # Start all services @@ -120,6 +122,54 @@ start_services() { echo " - Prometheus: http://localhost:9092" } +# Start in development mode +start_dev() { + print_info "Starting CHORUS Services in development mode..." + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build + + print_info "Waiting for services to start..." + sleep 15 + + print_success "CHORUS Services started in development mode" + print_info "Development features enabled:" + echo " - Live code reloading" + echo " - Debug logging enabled" + echo " - Local builds (no registry required)" +} + +# Deploy to Docker Swarm +deploy_swarm() { + print_info "Deploying CHORUS Services to Docker Swarm..." + + # Check if we're in a swarm + if ! docker info --format '{{.Swarm.LocalNodeState}}' | grep -q active; then + print_error "Docker Swarm is not active. Please initialize or join a swarm first." + return 1 + fi + + docker stack deploy -c docker-compose.swarm.yml chorus + + print_info "Waiting for stack to deploy..." + sleep 20 + + print_success "CHORUS Services deployed to swarm" + print_info "Production access points:" + echo " - WHOOSH Dashboard: https://chorus.home.deepblack.cloud" + echo " - WHOOSH API: https://chorus-api.home.deepblack.cloud" + echo " - BZZZ Coordinator: https://chorus-bzzz.home.deepblack.cloud" + echo " - SLURP API: https://chorus-slurp.home.deepblack.cloud" + echo " - COOEE RL Tuner: https://chorus-cooee.home.deepblack.cloud" + echo " - Grafana: https://chorus-grafana.home.deepblack.cloud" + echo " - Prometheus: https://chorus-prometheus.home.deepblack.cloud" +} + +# Remove from Docker Swarm +undeploy_swarm() { + print_info "Removing CHORUS Services from Docker Swarm..." + docker stack rm chorus + print_success "CHORUS Services removed from swarm" +} + # Stop all services stop_services() { print_info "Stopping CHORUS Services..." @@ -212,6 +262,9 @@ case "$1" in build) build_images ;; + pull) + pull_images + ;; update) update_submodules ;; @@ -224,6 +277,18 @@ case "$1" in health) check_health ;; + login) + docker_login + ;; + deploy) + deploy_swarm + ;; + undeploy) + undeploy_swarm + ;; + dev) + start_dev + ;; *) show_usage exit 1 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..eebb241 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,54 @@ +# Docker Compose Override for Development +# This file provides local build configurations for development +# Use: docker-compose -f docker-compose.yml -f docker-compose.dev.yml up + +version: '3.8' + +services: + # Development overrides - builds locally instead of using registry + whoosh-backend: + build: + context: ./modules/whoosh/backend + dockerfile: Dockerfile + volumes: + - ./modules/whoosh/backend:/app + - ./modules/whoosh/config:/app/config + environment: + - ENVIRONMENT=development + - LOG_LEVEL=debug + + whoosh-frontend: + build: + context: ./modules/whoosh/frontend + dockerfile: Dockerfile + volumes: + - ./modules/whoosh/frontend:/app + - /app/node_modules + + bzzz-coordinator: + build: + context: ./modules/bzzz + dockerfile: Dockerfile + volumes: + - ./modules/bzzz/config:/app/config + - ./modules/bzzz/data:/app/data + environment: + - BZZZ_NODE_ENV=development + - BZZZ_LOG_LEVEL=debug + + slurp-api: + build: + context: ./modules/slurp/hcfs-python + dockerfile: Dockerfile + volumes: + - ./modules/slurp/data:/app/data + - ./modules/slurp/config:/app/config + environment: + - HCFS_LOG_LEVEL=debug + + slurp-rl-tuner: + build: + context: ./modules/slurp + dockerfile: Dockerfile.rl-tuner + environment: + - LOG_LEVEL=debug \ No newline at end of file diff --git a/docker-compose.swarm.yml b/docker-compose.swarm.yml new file mode 100644 index 0000000..3f3cf64 --- /dev/null +++ b/docker-compose.swarm.yml @@ -0,0 +1,267 @@ +# Docker Compose for Docker Swarm Deployment +# Optimized for production deployment on deepblack.cloud infrastructure + +version: '3.8' + +services: + # WHOOSH - Orchestration Platform + whoosh-backend: + image: registry.home.deepblack.cloud/tony/chorus-whoosh-backend:latest + deploy: + replicas: 2 + placement: + constraints: + - node.role == worker + resources: + limits: + memory: 1G + reservations: + memory: 512M + labels: + - "traefik.enable=true" + - "traefik.docker.network=tengig" + - "traefik.http.routers.chorus-api.rule=Host(`chorus-api.home.deepblack.cloud`)" + - "traefik.http.routers.chorus-api.entrypoints=web-secured" + - "traefik.http.routers.chorus-api.tls.certresolver=letsencryptresolver" + - "traefik.http.services.chorus-api.loadbalancer.server.port=8000" + - "traefik.http.services.chorus-api.loadbalancer.passhostheader=true" + environment: + - DATABASE_URL=postgresql://chorus:choruspass@postgres:5432/chorus_whoosh + - REDIS_URL=redis://redis:6379 + - CORS_ORIGINS=https://chorus.home.deepblack.cloud + - ENVIRONMENT=production + - LOG_LEVEL=info + networks: + - tengig + - chorus_network + depends_on: + - postgres + - redis + + whoosh-frontend: + image: registry.home.deepblack.cloud/tony/chorus-whoosh-frontend:latest + deploy: + replicas: 2 + placement: + constraints: + - node.role == worker + resources: + limits: + memory: 512M + reservations: + memory: 256M + labels: + - "traefik.enable=true" + - "traefik.docker.network=tengig" + - "traefik.http.routers.chorus.rule=Host(`chorus.home.deepblack.cloud`)" + - "traefik.http.routers.chorus.entrypoints=web-secured" + - "traefik.http.routers.chorus.tls.certresolver=letsencryptresolver" + - "traefik.http.services.chorus.loadbalancer.server.port=3000" + - "traefik.http.services.chorus.loadbalancer.passhostheader=true" + environment: + - REACT_APP_API_URL=https://chorus-api.home.deepblack.cloud + - REACT_APP_WS_URL=wss://chorus-api.home.deepblack.cloud + networks: + - tengig + - chorus_network + depends_on: + - whoosh-backend + + # BZZZ - P2P Agent Coordination + bzzz-coordinator: + image: registry.home.deepblack.cloud/tony/chorus-bzzz-coordinator:latest + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager # P2P networking works better on manager + resources: + limits: + memory: 512M + reservations: + memory: 256M + labels: + - "traefik.enable=true" + - "traefik.docker.network=tengig" + - "traefik.http.routers.chorus-bzzz.rule=Host(`chorus-bzzz.home.deepblack.cloud`)" + - "traefik.http.routers.chorus-bzzz.entrypoints=web-secured" + - "traefik.http.routers.chorus-bzzz.tls.certresolver=letsencryptresolver" + - "traefik.http.services.chorus-bzzz.loadbalancer.server.port=8080" + ports: + - target: 4001 + published: 4001 + protocol: tcp + mode: host # Required for P2P networking + environment: + - BZZZ_NODE_ENV=production + - BZZZ_LOG_LEVEL=info + networks: + - tengig + - chorus_network + volumes: + - bzzz_data:/app/data + + # SLURP - Context Management + slurp-api: + image: registry.home.deepblack.cloud/tony/chorus-slurp-api:latest + deploy: + replicas: 2 + placement: + constraints: + - node.role == worker + resources: + limits: + memory: 1G + reservations: + memory: 512M + labels: + - "traefik.enable=true" + - "traefik.docker.network=tengig" + - "traefik.http.routers.chorus-slurp.rule=Host(`chorus-slurp.home.deepblack.cloud`)" + - "traefik.http.routers.chorus-slurp.entrypoints=web-secured" + - "traefik.http.routers.chorus-slurp.tls.certresolver=letsencryptresolver" + - "traefik.http.services.chorus-slurp.loadbalancer.server.port=8000" + environment: + - HCFS_DATABASE_URL=postgresql://chorus:choruspass@postgres:5432/chorus_slurp + - HCFS_LOG_LEVEL=info + - HCFS_AUTH_ENABLED=true + networks: + - tengig + - chorus_network + volumes: + - slurp_data:/app/data + depends_on: + - postgres + + # COOEE - RL Context Tuner + slurp-rl-tuner: + image: registry.home.deepblack.cloud/tony/chorus-slurp-rl-tuner:latest + deploy: + replicas: 1 + placement: + constraints: + - node.role == worker + resources: + limits: + memory: 512M + reservations: + memory: 256M + labels: + - "traefik.enable=true" + - "traefik.docker.network=tengig" + - "traefik.http.routers.chorus-cooee.rule=Host(`chorus-cooee.home.deepblack.cloud`)" + - "traefik.http.routers.chorus-cooee.entrypoints=web-secured" + - "traefik.http.routers.chorus-cooee.tls.certresolver=letsencryptresolver" + - "traefik.http.services.chorus-cooee.loadbalancer.server.port=8000" + environment: + - RL_TUNER_DATABASE_URL=postgresql://chorus:choruspass@postgres:5432/chorus_rl_tuner + - HCFS_API_URL=http://slurp-api:8000 + - BZZZ_API_URL=http://bzzz-coordinator:8080 + networks: + - tengig + - chorus_network + depends_on: + - postgres + - slurp-api + - bzzz-coordinator + + # Shared Infrastructure + postgres: + image: postgres:15 + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager # Keep database on manager for stability + resources: + limits: + memory: 2G + reservations: + memory: 1G + environment: + - POSTGRES_DB=chorus + - POSTGRES_USER=chorus + - POSTGRES_PASSWORD=choruspass + networks: + - chorus_network + volumes: + - postgres_data:/var/lib/postgresql/data + - ./init-db.sql:/docker-entrypoint-initdb.d/init-db.sql + + redis: + image: redis:7-alpine + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager + resources: + limits: + memory: 256M + reservations: + memory: 128M + networks: + - chorus_network + volumes: + - redis_data:/data + + # Monitoring Stack + prometheus: + image: prom/prometheus:latest + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager + labels: + - "traefik.enable=true" + - "traefik.docker.network=tengig" + - "traefik.http.routers.chorus-prometheus.rule=Host(`chorus-prometheus.home.deepblack.cloud`)" + - "traefik.http.routers.chorus-prometheus.entrypoints=web-secured" + - "traefik.http.routers.chorus-prometheus.tls.certresolver=letsencryptresolver" + - "traefik.http.services.chorus-prometheus.loadbalancer.server.port=9090" + networks: + - tengig + - chorus_network + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + + grafana: + image: grafana/grafana:latest + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager + labels: + - "traefik.enable=true" + - "traefik.docker.network=tengig" + - "traefik.http.routers.chorus-grafana.rule=Host(`chorus-grafana.home.deepblack.cloud`)" + - "traefik.http.routers.chorus-grafana.entrypoints=web-secured" + - "traefik.http.routers.chorus-grafana.tls.certresolver=letsencryptresolver" + - "traefik.http.services.chorus-grafana.loadbalancer.server.port=3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=chorusadmin + networks: + - tengig + - chorus_network + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards + - ./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources + +volumes: + postgres_data: + redis_data: + prometheus_data: + grafana_data: + bzzz_data: + slurp_data: + +networks: + tengig: + external: true + chorus_network: + driver: overlay + attachable: true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b11f57e..220d42f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.8' services: # WHOOSH - Orchestration Platform whoosh-backend: - build: ./modules/whoosh/backend + image: registry.home.deepblack.cloud/tony/chorus-whoosh-backend:latest container_name: chorus_whoosh_backend ports: - "8087:8000" @@ -22,7 +22,7 @@ services: - ./modules/whoosh/config:/app/config whoosh-frontend: - build: ./modules/whoosh/frontend + image: registry.home.deepblack.cloud/tony/chorus-whoosh-frontend:latest container_name: chorus_whoosh_frontend ports: - "3001:3000" @@ -39,9 +39,7 @@ services: # BZZZ - P2P Agent Coordination bzzz-coordinator: - build: - context: ./modules/bzzz - dockerfile: Dockerfile + image: registry.home.deepblack.cloud/tony/chorus-bzzz-coordinator:latest container_name: chorus_bzzz_coordinator ports: - "4001:4001" # libp2p port @@ -58,7 +56,7 @@ services: # SLURP - Context Management (HCFS) slurp-api: - build: ./modules/slurp/hcfs-python + image: registry.home.deepblack.cloud/tony/chorus-slurp-api:latest container_name: chorus_slurp_api ports: - "8088:8000" @@ -74,9 +72,7 @@ services: # RL Context SLURP (COOEE equivalent) slurp-rl-tuner: - build: - context: ./modules/slurp - dockerfile: Dockerfile.rl-tuner + image: registry.home.deepblack.cloud/tony/chorus-slurp-rl-tuner:latest container_name: chorus_slurp_rl_tuner ports: - "8089:8000"