Configure Docker registry integration for CHORUS Services

Registry Configuration:
- Updated docker-compose.yml to use registry.home.deepblack.cloud images
- Created build-and-push.sh script for automated image building/pushing
- Enhanced chorus.sh with registry operations (login, build, pull)
- Added docker-compose.dev.yml for development with local builds
- Added docker-compose.swarm.yml for production deployment

Production Features:
- Docker Swarm deployment with Traefik integration
- All services available at *.home.deepblack.cloud domains
- Production-grade resource limits and placement constraints
- SSL/TLS certificates via Let's Encrypt
- Load balancing and high availability

Development Features:
- ./chorus.sh dev - Local builds with live reloading
- ./chorus.sh build - Build and push to registry
- ./chorus.sh deploy - Production swarm deployment
- Registry authentication with credentials

Images stored at:
- registry.home.deepblack.cloud/tony/chorus-whoosh-backend:latest
- registry.home.deepblack.cloud/tony/chorus-whoosh-frontend:latest
- registry.home.deepblack.cloud/tony/chorus-bzzz-coordinator:latest
- registry.home.deepblack.cloud/tony/chorus-slurp-api:latest
- registry.home.deepblack.cloud/tony/chorus-slurp-rl-tuner:latest

Ready for both development and production deployment.
This commit is contained in:
tony
2025-08-01 09:23:26 +10:00
parent 2090a94f58
commit ecca86b0fe
6 changed files with 619 additions and 33 deletions

View File

@@ -8,6 +8,12 @@
# Initialize submodules (first time only)
./chorus.sh init
# Login to Docker registry
./chorus.sh login
# Build and push images to registry
./chorus.sh build
# Start all services
./chorus.sh start
@@ -83,13 +89,22 @@ The `./chorus.sh` script provides unified management:
```bash
# Service Management
./chorus.sh start # Start all services
./chorus.sh start # Start all services (registry images)
./chorus.sh stop # Stop all services
./chorus.sh restart # Restart all services
./chorus.sh status # Show service status
./chorus.sh dev # Start in development mode (local builds)
# Docker Registry Operations
./chorus.sh login # Login to Docker registry
./chorus.sh build # Build and push all images to registry
./chorus.sh pull # Pull latest images from registry
# Production Deployment
./chorus.sh deploy # Deploy to Docker Swarm (production)
./chorus.sh undeploy # Remove from Docker Swarm
# Development & Maintenance
./chorus.sh build # Build all docker images
./chorus.sh update # Update submodules to latest
./chorus.sh logs [service] # View logs
./chorus.sh health # Check service health
@@ -176,7 +191,10 @@ cd chorus.services
# Or initialize submodules if already cloned
./chorus.sh init
# Build all images
# Login to Docker registry
./chorus.sh login
# Build and push all images to registry
./chorus.sh build
# Start services
@@ -185,9 +203,18 @@ cd chorus.services
### Development Workflow
```bash
# Development mode (local builds, live reloading)
./chorus.sh dev
# Update submodules to latest
./chorus.sh update
# Rebuild and push after changes
./chorus.sh build
# Pull latest images from registry
./chorus.sh pull
# View logs during development
./chorus.sh logs
@@ -198,6 +225,17 @@ cd chorus.services
./chorus.sh restart
```
### Production Deployment
```bash
# Deploy to Docker Swarm (production)
./chorus.sh deploy
# Access at https://*.home.deepblack.cloud endpoints
# Remove from swarm
./chorus.sh undeploy
```
## 🚀 Git Submodules Guide
CHORUS Services uses git submodules to integrate independent components:

166
build-and-push.sh Executable file
View File

@@ -0,0 +1,166 @@
#!/bin/bash
# CHORUS Services - Build and Push Script
# Builds all Docker images and pushes them to the local registry
set -e
REGISTRY="registry.home.deepblack.cloud"
REGISTRY_USER="tony"
REGISTRY_PASS="silverfrond[1392]"
TAG="latest"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Print colored output
print_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
print_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Docker registry login
docker_login() {
print_info "Logging into Docker registry: $REGISTRY"
echo "$REGISTRY_PASS" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin
print_success "Successfully logged into registry"
}
# Build and push a single image
build_and_push() {
local name=$1
local context=$2
local dockerfile=${3:-"Dockerfile"}
local image_name="$REGISTRY/tony/chorus-${name}:$TAG"
print_info "Building $name from $context"
if [ -f "$context/$dockerfile" ]; then
docker build -t "$image_name" -f "$context/$dockerfile" "$context"
print_success "Built $image_name"
print_info "Pushing $image_name to registry"
docker push "$image_name"
print_success "Pushed $image_name"
else
print_error "Dockerfile not found: $context/$dockerfile"
return 1
fi
}
# Main build process
main() {
print_info "Starting CHORUS Services build and push process"
# Login to registry
docker_login
# Build and push all images
print_info "Building all CHORUS Services components..."
# WHOOSH Backend
if [ -d "modules/whoosh/backend" ]; then
build_and_push "whoosh-backend" "modules/whoosh/backend"
else
print_warning "WHOOSH backend directory not found - skipping"
fi
# WHOOSH Frontend
if [ -d "modules/whoosh/frontend" ]; then
build_and_push "whoosh-frontend" "modules/whoosh/frontend"
else
print_warning "WHOOSH frontend directory not found - skipping"
fi
# BZZZ Coordinator
if [ -d "modules/bzzz" ]; then
build_and_push "bzzz-coordinator" "modules/bzzz"
else
print_warning "BZZZ directory not found - skipping"
fi
# SLURP API
if [ -d "modules/slurp/hcfs-python" ]; then
build_and_push "slurp-api" "modules/slurp/hcfs-python"
else
print_warning "SLURP API directory not found - skipping"
fi
# SLURP RL Tuner
if [ -d "modules/slurp" ] && [ -f "modules/slurp/Dockerfile.rl-tuner" ]; then
build_and_push "slurp-rl-tuner" "modules/slurp" "Dockerfile.rl-tuner"
else
print_warning "SLURP RL Tuner dockerfile not found - skipping"
fi
print_success "All CHORUS Services images built and pushed successfully!"
# List pushed images
print_info "Pushed images:"
echo " - $REGISTRY/tony/chorus-whoosh-backend:$TAG"
echo " - $REGISTRY/tony/chorus-whoosh-frontend:$TAG"
echo " - $REGISTRY/tony/chorus-bzzz-coordinator:$TAG"
echo " - $REGISTRY/tony/chorus-slurp-api:$TAG"
echo " - $REGISTRY/tony/chorus-slurp-rl-tuner:$TAG"
}
# Handle command line arguments
case "${1:-}" in
"login")
docker_login
;;
"whoosh-backend")
docker_login
build_and_push "whoosh-backend" "modules/whoosh/backend"
;;
"whoosh-frontend")
docker_login
build_and_push "whoosh-frontend" "modules/whoosh/frontend"
;;
"bzzz")
docker_login
build_and_push "bzzz-coordinator" "modules/bzzz"
;;
"slurp-api")
docker_login
build_and_push "slurp-api" "modules/slurp/hcfs-python"
;;
"slurp-rl-tuner")
docker_login
build_and_push "slurp-rl-tuner" "modules/slurp" "Dockerfile.rl-tuner"
;;
"help"|"-h"|"--help")
echo "CHORUS Services Build and Push Script"
echo ""
echo "Usage: $0 [COMMAND]"
echo ""
echo "Commands:"
echo " (no args) Build and push all images"
echo " login Login to Docker registry only"
echo " whoosh-backend Build and push WHOOSH backend only"
echo " whoosh-frontend Build and push WHOOSH frontend only"
echo " bzzz Build and push BZZZ coordinator only"
echo " slurp-api Build and push SLURP API only"
echo " slurp-rl-tuner Build and push SLURP RL Tuner only"
echo " help Show this help message"
;;
*)
main
;;
esac

107
chorus.sh
View File

@@ -45,11 +45,16 @@ show_usage() {
echo " status Show status of all services"
echo " logs Show logs from all services"
echo " logs [service] Show logs from specific service"
echo " build Build all docker images"
echo " build Build and push all docker images to registry"
echo " pull Pull latest images from registry"
echo " update Update all submodules to latest"
echo " init Initialize submodules (first time setup)"
echo " clean Clean up docker resources"
echo " health Check health of all services"
echo " login Login to Docker registry"
echo " deploy Deploy to Docker Swarm (production)"
echo " undeploy Remove from Docker Swarm"
echo " dev Start in development mode with local builds"
echo ""
echo "Services:"
echo " - whoosh-backend : Orchestration API (port 8087)"
@@ -78,27 +83,24 @@ update_submodules() {
print_success "Submodules updated"
}
# Build all docker images
# Build and push all docker images
build_images() {
print_info "Building CHORUS Services docker images..."
# Build each service
print_info "Building WHOOSH backend..."
docker-compose build whoosh-backend
print_info "Building WHOOSH frontend..."
docker-compose build whoosh-frontend
print_info "Building BZZZ coordinator..."
docker-compose build bzzz-coordinator
print_info "Building SLURP API..."
docker-compose build slurp-api
print_info "Building SLURP RL Tuner..."
docker-compose build slurp-rl-tuner
print_success "All images built successfully"
print_info "Building and pushing CHORUS Services docker images to registry..."
./build-and-push.sh
print_success "All images built and pushed successfully"
}
# Pull latest images from registry
pull_images() {
print_info "Pulling latest CHORUS Services images from registry..."
docker-compose pull
print_success "All images pulled successfully"
}
# Login to Docker registry
docker_login() {
print_info "Logging into Docker registry..."
./build-and-push.sh login
}
# Start all services
@@ -120,6 +122,54 @@ start_services() {
echo " - Prometheus: http://localhost:9092"
}
# Start in development mode
start_dev() {
print_info "Starting CHORUS Services in development mode..."
docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
print_info "Waiting for services to start..."
sleep 15
print_success "CHORUS Services started in development mode"
print_info "Development features enabled:"
echo " - Live code reloading"
echo " - Debug logging enabled"
echo " - Local builds (no registry required)"
}
# Deploy to Docker Swarm
deploy_swarm() {
print_info "Deploying CHORUS Services to Docker Swarm..."
# Check if we're in a swarm
if ! docker info --format '{{.Swarm.LocalNodeState}}' | grep -q active; then
print_error "Docker Swarm is not active. Please initialize or join a swarm first."
return 1
fi
docker stack deploy -c docker-compose.swarm.yml chorus
print_info "Waiting for stack to deploy..."
sleep 20
print_success "CHORUS Services deployed to swarm"
print_info "Production access points:"
echo " - WHOOSH Dashboard: https://chorus.home.deepblack.cloud"
echo " - WHOOSH API: https://chorus-api.home.deepblack.cloud"
echo " - BZZZ Coordinator: https://chorus-bzzz.home.deepblack.cloud"
echo " - SLURP API: https://chorus-slurp.home.deepblack.cloud"
echo " - COOEE RL Tuner: https://chorus-cooee.home.deepblack.cloud"
echo " - Grafana: https://chorus-grafana.home.deepblack.cloud"
echo " - Prometheus: https://chorus-prometheus.home.deepblack.cloud"
}
# Remove from Docker Swarm
undeploy_swarm() {
print_info "Removing CHORUS Services from Docker Swarm..."
docker stack rm chorus
print_success "CHORUS Services removed from swarm"
}
# Stop all services
stop_services() {
print_info "Stopping CHORUS Services..."
@@ -212,6 +262,9 @@ case "$1" in
build)
build_images
;;
pull)
pull_images
;;
update)
update_submodules
;;
@@ -224,6 +277,18 @@ case "$1" in
health)
check_health
;;
login)
docker_login
;;
deploy)
deploy_swarm
;;
undeploy)
undeploy_swarm
;;
dev)
start_dev
;;
*)
show_usage
exit 1

54
docker-compose.dev.yml Normal file
View File

@@ -0,0 +1,54 @@
# Docker Compose Override for Development
# This file provides local build configurations for development
# Use: docker-compose -f docker-compose.yml -f docker-compose.dev.yml up
version: '3.8'
services:
# Development overrides - builds locally instead of using registry
whoosh-backend:
build:
context: ./modules/whoosh/backend
dockerfile: Dockerfile
volumes:
- ./modules/whoosh/backend:/app
- ./modules/whoosh/config:/app/config
environment:
- ENVIRONMENT=development
- LOG_LEVEL=debug
whoosh-frontend:
build:
context: ./modules/whoosh/frontend
dockerfile: Dockerfile
volumes:
- ./modules/whoosh/frontend:/app
- /app/node_modules
bzzz-coordinator:
build:
context: ./modules/bzzz
dockerfile: Dockerfile
volumes:
- ./modules/bzzz/config:/app/config
- ./modules/bzzz/data:/app/data
environment:
- BZZZ_NODE_ENV=development
- BZZZ_LOG_LEVEL=debug
slurp-api:
build:
context: ./modules/slurp/hcfs-python
dockerfile: Dockerfile
volumes:
- ./modules/slurp/data:/app/data
- ./modules/slurp/config:/app/config
environment:
- HCFS_LOG_LEVEL=debug
slurp-rl-tuner:
build:
context: ./modules/slurp
dockerfile: Dockerfile.rl-tuner
environment:
- LOG_LEVEL=debug

267
docker-compose.swarm.yml Normal file
View File

@@ -0,0 +1,267 @@
# Docker Compose for Docker Swarm Deployment
# Optimized for production deployment on deepblack.cloud infrastructure
version: '3.8'
services:
# WHOOSH - Orchestration Platform
whoosh-backend:
image: registry.home.deepblack.cloud/tony/chorus-whoosh-backend:latest
deploy:
replicas: 2
placement:
constraints:
- node.role == worker
resources:
limits:
memory: 1G
reservations:
memory: 512M
labels:
- "traefik.enable=true"
- "traefik.docker.network=tengig"
- "traefik.http.routers.chorus-api.rule=Host(`chorus-api.home.deepblack.cloud`)"
- "traefik.http.routers.chorus-api.entrypoints=web-secured"
- "traefik.http.routers.chorus-api.tls.certresolver=letsencryptresolver"
- "traefik.http.services.chorus-api.loadbalancer.server.port=8000"
- "traefik.http.services.chorus-api.loadbalancer.passhostheader=true"
environment:
- DATABASE_URL=postgresql://chorus:choruspass@postgres:5432/chorus_whoosh
- REDIS_URL=redis://redis:6379
- CORS_ORIGINS=https://chorus.home.deepblack.cloud
- ENVIRONMENT=production
- LOG_LEVEL=info
networks:
- tengig
- chorus_network
depends_on:
- postgres
- redis
whoosh-frontend:
image: registry.home.deepblack.cloud/tony/chorus-whoosh-frontend:latest
deploy:
replicas: 2
placement:
constraints:
- node.role == worker
resources:
limits:
memory: 512M
reservations:
memory: 256M
labels:
- "traefik.enable=true"
- "traefik.docker.network=tengig"
- "traefik.http.routers.chorus.rule=Host(`chorus.home.deepblack.cloud`)"
- "traefik.http.routers.chorus.entrypoints=web-secured"
- "traefik.http.routers.chorus.tls.certresolver=letsencryptresolver"
- "traefik.http.services.chorus.loadbalancer.server.port=3000"
- "traefik.http.services.chorus.loadbalancer.passhostheader=true"
environment:
- REACT_APP_API_URL=https://chorus-api.home.deepblack.cloud
- REACT_APP_WS_URL=wss://chorus-api.home.deepblack.cloud
networks:
- tengig
- chorus_network
depends_on:
- whoosh-backend
# BZZZ - P2P Agent Coordination
bzzz-coordinator:
image: registry.home.deepblack.cloud/tony/chorus-bzzz-coordinator:latest
deploy:
replicas: 1
placement:
constraints:
- node.role == manager # P2P networking works better on manager
resources:
limits:
memory: 512M
reservations:
memory: 256M
labels:
- "traefik.enable=true"
- "traefik.docker.network=tengig"
- "traefik.http.routers.chorus-bzzz.rule=Host(`chorus-bzzz.home.deepblack.cloud`)"
- "traefik.http.routers.chorus-bzzz.entrypoints=web-secured"
- "traefik.http.routers.chorus-bzzz.tls.certresolver=letsencryptresolver"
- "traefik.http.services.chorus-bzzz.loadbalancer.server.port=8080"
ports:
- target: 4001
published: 4001
protocol: tcp
mode: host # Required for P2P networking
environment:
- BZZZ_NODE_ENV=production
- BZZZ_LOG_LEVEL=info
networks:
- tengig
- chorus_network
volumes:
- bzzz_data:/app/data
# SLURP - Context Management
slurp-api:
image: registry.home.deepblack.cloud/tony/chorus-slurp-api:latest
deploy:
replicas: 2
placement:
constraints:
- node.role == worker
resources:
limits:
memory: 1G
reservations:
memory: 512M
labels:
- "traefik.enable=true"
- "traefik.docker.network=tengig"
- "traefik.http.routers.chorus-slurp.rule=Host(`chorus-slurp.home.deepblack.cloud`)"
- "traefik.http.routers.chorus-slurp.entrypoints=web-secured"
- "traefik.http.routers.chorus-slurp.tls.certresolver=letsencryptresolver"
- "traefik.http.services.chorus-slurp.loadbalancer.server.port=8000"
environment:
- HCFS_DATABASE_URL=postgresql://chorus:choruspass@postgres:5432/chorus_slurp
- HCFS_LOG_LEVEL=info
- HCFS_AUTH_ENABLED=true
networks:
- tengig
- chorus_network
volumes:
- slurp_data:/app/data
depends_on:
- postgres
# COOEE - RL Context Tuner
slurp-rl-tuner:
image: registry.home.deepblack.cloud/tony/chorus-slurp-rl-tuner:latest
deploy:
replicas: 1
placement:
constraints:
- node.role == worker
resources:
limits:
memory: 512M
reservations:
memory: 256M
labels:
- "traefik.enable=true"
- "traefik.docker.network=tengig"
- "traefik.http.routers.chorus-cooee.rule=Host(`chorus-cooee.home.deepblack.cloud`)"
- "traefik.http.routers.chorus-cooee.entrypoints=web-secured"
- "traefik.http.routers.chorus-cooee.tls.certresolver=letsencryptresolver"
- "traefik.http.services.chorus-cooee.loadbalancer.server.port=8000"
environment:
- RL_TUNER_DATABASE_URL=postgresql://chorus:choruspass@postgres:5432/chorus_rl_tuner
- HCFS_API_URL=http://slurp-api:8000
- BZZZ_API_URL=http://bzzz-coordinator:8080
networks:
- tengig
- chorus_network
depends_on:
- postgres
- slurp-api
- bzzz-coordinator
# Shared Infrastructure
postgres:
image: postgres:15
deploy:
replicas: 1
placement:
constraints:
- node.role == manager # Keep database on manager for stability
resources:
limits:
memory: 2G
reservations:
memory: 1G
environment:
- POSTGRES_DB=chorus
- POSTGRES_USER=chorus
- POSTGRES_PASSWORD=choruspass
networks:
- chorus_network
volumes:
- postgres_data:/var/lib/postgresql/data
- ./init-db.sql:/docker-entrypoint-initdb.d/init-db.sql
redis:
image: redis:7-alpine
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
resources:
limits:
memory: 256M
reservations:
memory: 128M
networks:
- chorus_network
volumes:
- redis_data:/data
# Monitoring Stack
prometheus:
image: prom/prometheus:latest
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
labels:
- "traefik.enable=true"
- "traefik.docker.network=tengig"
- "traefik.http.routers.chorus-prometheus.rule=Host(`chorus-prometheus.home.deepblack.cloud`)"
- "traefik.http.routers.chorus-prometheus.entrypoints=web-secured"
- "traefik.http.routers.chorus-prometheus.tls.certresolver=letsencryptresolver"
- "traefik.http.services.chorus-prometheus.loadbalancer.server.port=9090"
networks:
- tengig
- chorus_network
volumes:
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus_data:/prometheus
grafana:
image: grafana/grafana:latest
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
labels:
- "traefik.enable=true"
- "traefik.docker.network=tengig"
- "traefik.http.routers.chorus-grafana.rule=Host(`chorus-grafana.home.deepblack.cloud`)"
- "traefik.http.routers.chorus-grafana.entrypoints=web-secured"
- "traefik.http.routers.chorus-grafana.tls.certresolver=letsencryptresolver"
- "traefik.http.services.chorus-grafana.loadbalancer.server.port=3000"
environment:
- GF_SECURITY_ADMIN_PASSWORD=chorusadmin
networks:
- tengig
- chorus_network
volumes:
- grafana_data:/var/lib/grafana
- ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
- ./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
volumes:
postgres_data:
redis_data:
prometheus_data:
grafana_data:
bzzz_data:
slurp_data:
networks:
tengig:
external: true
chorus_network:
driver: overlay
attachable: true

View File

@@ -3,7 +3,7 @@ version: '3.8'
services:
# WHOOSH - Orchestration Platform
whoosh-backend:
build: ./modules/whoosh/backend
image: registry.home.deepblack.cloud/tony/chorus-whoosh-backend:latest
container_name: chorus_whoosh_backend
ports:
- "8087:8000"
@@ -22,7 +22,7 @@ services:
- ./modules/whoosh/config:/app/config
whoosh-frontend:
build: ./modules/whoosh/frontend
image: registry.home.deepblack.cloud/tony/chorus-whoosh-frontend:latest
container_name: chorus_whoosh_frontend
ports:
- "3001:3000"
@@ -39,9 +39,7 @@ services:
# BZZZ - P2P Agent Coordination
bzzz-coordinator:
build:
context: ./modules/bzzz
dockerfile: Dockerfile
image: registry.home.deepblack.cloud/tony/chorus-bzzz-coordinator:latest
container_name: chorus_bzzz_coordinator
ports:
- "4001:4001" # libp2p port
@@ -58,7 +56,7 @@ services:
# SLURP - Context Management (HCFS)
slurp-api:
build: ./modules/slurp/hcfs-python
image: registry.home.deepblack.cloud/tony/chorus-slurp-api:latest
container_name: chorus_slurp_api
ports:
- "8088:8000"
@@ -74,9 +72,7 @@ services:
# RL Context SLURP (COOEE equivalent)
slurp-rl-tuner:
build:
context: ./modules/slurp
dockerfile: Dockerfile.rl-tuner
image: registry.home.deepblack.cloud/tony/chorus-slurp-rl-tuner:latest
container_name: chorus_slurp_rl_tuner
ports:
- "8089:8000"