bzzz/infrastructure/ci-cd/.gitlab-ci.yml

# BZZZ v2 GitLab CI/CD Pipeline
# Comprehensive build, test, and deployment pipeline for BZZZ v2

variables:
  REGISTRY: registry.home.deepblack.cloud
  REGISTRY_NAMESPACE: bzzz
  GO_VERSION: "1.21"
  DOCKER_BUILDKIT: "1"
  COMPOSE_DOCKER_CLI_BUILD: "1"
  POSTGRES_VERSION: "15"
  REDIS_VERSION: "7"
  
  # Semantic versioning
  VERSION_PREFIX: "v2"
  
stages:
  - lint
  - test
  - build
  - security-scan
  - integration-test
  - deploy-staging
  - performance-test
  - deploy-production
  - post-deploy-validation

# Cache configuration
cache:
  key: "${CI_COMMIT_REF_SLUG}"
  paths:
    - .cache/go-mod/
    - .cache/docker/
    - vendor/

before_script:
  - export GOPATH=$CI_PROJECT_DIR/.cache/go-mod
  - export GOCACHE=$CI_PROJECT_DIR/.cache/go-build
  - mkdir -p .cache/{go-mod,go-build,docker}

# ================================
# LINT STAGE
# ================================

golang-lint:
  stage: lint
  image: golangci/golangci-lint:v1.55-alpine
  script:
    - golangci-lint run ./... --timeout 10m
    - go mod tidy
    - git diff --exit-code go.mod go.sum
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_BRANCH == "develop"'

dockerfile-lint:
  stage: lint
  image: hadolint/hadolint:latest-debian
  script:
    - hadolint infrastructure/dockerfiles/Dockerfile.*
    - hadolint Dockerfile
  rules:
    - changes:
        - "infrastructure/dockerfiles/*"
        - "Dockerfile*"

yaml-lint:
  stage: lint
  image: cytopia/yamllint:latest
  script:
    - yamllint infrastructure/
    - yamllint .gitlab-ci.yml
  rules:
    - changes:
        - "infrastructure/**/*.yml"
        - "infrastructure/**/*.yaml" 
        - ".gitlab-ci.yml"

# ================================
# TEST STAGE
# ================================

unit-tests:
  stage: test
  image: golang:$GO_VERSION-alpine
  services:
    - name: postgres:$POSTGRES_VERSION-alpine
      alias: postgres
    - name: redis:$REDIS_VERSION-alpine
      alias: redis
  variables:
    POSTGRES_DB: bzzz_test
    POSTGRES_USER: test
    POSTGRES_PASSWORD: testpass
    POSTGRES_HOST: postgres
    REDIS_HOST: redis
    CGO_ENABLED: 0
  before_script:
    - apk add --no-cache git make gcc musl-dev
    - export GOPATH=$CI_PROJECT_DIR/.cache/go-mod
    - export GOCACHE=$CI_PROJECT_DIR/.cache/go-build
  script:
    - go mod download
    - go test -v -race -coverprofile=coverage.out ./...
    - go tool cover -html=coverage.out -o coverage.html
    - go tool cover -func=coverage.out | grep total | awk '{print "Coverage: " $3}'
  coverage: '/Coverage: \d+\.\d+/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
    paths:
      - coverage.html
      - coverage.out
    expire_in: 1 week

p2p-protocol-tests:
  stage: test
  image: golang:$GO_VERSION-alpine
  script:
    - apk add --no-cache git make gcc musl-dev
    - go test -v -tags=p2p ./p2p/... ./dht/... 
    - go test -v -tags=integration ./test/p2p/...
  rules:
    - changes:
        - "p2p/**/*"
        - "dht/**/*"
        - "test/p2p/**/*"

content-store-tests:
  stage: test
  image: golang:$GO_VERSION-alpine
  script:
    - apk add --no-cache git make gcc musl-dev
    - go test -v -tags=storage ./storage/... ./blake3/...
    - go test -v -benchmem -bench=. ./storage/...
  artifacts:
    paths:
      - benchmark.out
    expire_in: 1 week
  rules:
    - changes:
        - "storage/**/*"
        - "blake3/**/*"

conversation-tests:
  stage: test
  image: golang:$GO_VERSION-alpine
  services:
    - name: postgres:$POSTGRES_VERSION-alpine
      alias: postgres
  variables:
    POSTGRES_DB: bzzz_conversation_test
    POSTGRES_USER: test
    POSTGRES_PASSWORD: testpass
    POSTGRES_HOST: postgres
  script:
    - apk add --no-cache git make gcc musl-dev postgresql-client
    - until pg_isready -h postgres -p 5432 -U test; do sleep 1; done
    - go test -v -tags=conversation ./conversation/... ./threading/...
  rules:
    - changes:
        - "conversation/**/*"
        - "threading/**/*"

# ================================
# BUILD STAGE 
# ================================

build-binaries:
  stage: build
  image: golang:$GO_VERSION-alpine
  before_script:
    - apk add --no-cache git make gcc musl-dev upx
    - export GOPATH=$CI_PROJECT_DIR/.cache/go-mod
    - export GOCACHE=$CI_PROJECT_DIR/.cache/go-build
  script:
    - make build-all
    - upx --best --lzma dist/bzzz-*
    - ls -la dist/
  artifacts:
    paths:
      - dist/
    expire_in: 1 week

build-docker-images:
  stage: build
  image: docker:24-dind
  services:
    - docker:24-dind
  variables:
    IMAGE_TAG: ${CI_COMMIT_SHORT_SHA}
    DOCKER_DRIVER: overlay2
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $REGISTRY
    - docker buildx create --use --driver docker-container
  script:
    # Build all images in parallel
    - |
      docker buildx build \
        --platform linux/amd64,linux/arm64 \
        --build-arg VERSION=${VERSION_PREFIX}.${CI_PIPELINE_ID} \
        --build-arg COMMIT=${CI_COMMIT_SHORT_SHA} \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-agent:$IMAGE_TAG \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-agent:latest \
        --file infrastructure/dockerfiles/Dockerfile.agent \
        --push .
    
    - |
      docker buildx build \
        --platform linux/amd64,linux/arm64 \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-mcp:$IMAGE_TAG \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-mcp:latest \
        --file infrastructure/dockerfiles/Dockerfile.mcp \
        --push .
    
    - |
      docker buildx build \
        --platform linux/amd64,linux/arm64 \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-openai-proxy:$IMAGE_TAG \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-openai-proxy:latest \
        --file infrastructure/dockerfiles/Dockerfile.proxy \
        --push .
    
    - |
      docker buildx build \
        --platform linux/amd64,linux/arm64 \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-resolver:$IMAGE_TAG \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-resolver:latest \
        --file infrastructure/dockerfiles/Dockerfile.resolver \
        --push .
        
    - |
      docker buildx build \
        --platform linux/amd64,linux/arm64 \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-dht:$IMAGE_TAG \
        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-dht:latest \
        --file infrastructure/dockerfiles/Dockerfile.dht \
        --push .
  
  dependencies:
    - build-binaries

# ================================
# SECURITY SCAN STAGE
# ================================

container-security-scan:
  stage: security-scan
  image: aquasec/trivy:latest
  script:
    - |
      for component in agent mcp openai-proxy resolver dht; do
        echo "Scanning bzzz-${component}..."
        trivy image --exit-code 1 --severity HIGH,CRITICAL \
          --format json --output trivy-${component}.json \
          $REGISTRY/$REGISTRY_NAMESPACE/bzzz-${component}:${CI_COMMIT_SHORT_SHA}
      done
  artifacts:
    reports:
      container_scanning: trivy-*.json
    expire_in: 1 week
  dependencies:
    - build-docker-images
  allow_failure: true

dependency-security-scan:
  stage: security-scan
  image: golang:$GO_VERSION-alpine
  script:
    - go install golang.org/x/vuln/cmd/govulncheck@latest
    - govulncheck ./...
  allow_failure: true

secrets-scan:
  stage: security-scan
  image: trufflesecurity/trufflehog:latest
  script:
    - trufflehog filesystem --directory=. --fail --json
  allow_failure: true
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

# ================================
# INTEGRATION TEST STAGE
# ================================

p2p-integration-test:
  stage: integration-test
  image: docker:24-dind
  services:
    - docker:24-dind
  variables:
    COMPOSE_PROJECT_NAME: bzzz-integration-${CI_PIPELINE_ID}
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $REGISTRY
    - apk add --no-cache docker-compose curl jq
  script:
    - cd infrastructure/testing
    - docker-compose -f docker-compose.integration.yml up -d
    - sleep 60  # Wait for services to start
    - ./scripts/test-p2p-mesh.sh
    - ./scripts/test-dht-discovery.sh  
    - ./scripts/test-content-addressing.sh
    - docker-compose -f docker-compose.integration.yml logs
  after_script:
    - cd infrastructure/testing
    - docker-compose -f docker-compose.integration.yml down -v
  artifacts:
    paths:
      - infrastructure/testing/test-results/
    expire_in: 1 week
    when: always
  dependencies:
    - build-docker-images

mcp-integration-test:
  stage: integration-test
  image: node:18-alpine
  services:
    - name: $REGISTRY/$REGISTRY_NAMESPACE/bzzz-mcp:${CI_COMMIT_SHORT_SHA}
      alias: mcp-server
    - name: $REGISTRY/$REGISTRY_NAMESPACE/bzzz-agent:${CI_COMMIT_SHORT_SHA} 
      alias: bzzz-agent
  script:
    - cd test/mcp
    - npm install
    - npm test
  artifacts:
    reports:
      junit: test/mcp/junit.xml
  dependencies:
    - build-docker-images

openai-proxy-test:
  stage: integration-test
  image: python:3.11-alpine
  services:
    - name: $REGISTRY/$REGISTRY_NAMESPACE/bzzz-openai-proxy:${CI_COMMIT_SHORT_SHA}
      alias: openai-proxy
    - name: redis:$REDIS_VERSION-alpine
      alias: redis
  variables:
    OPENAI_API_KEY: "test-key-mock"
    REDIS_HOST: redis
  script:
    - cd test/openai-proxy
    - pip install -r requirements.txt
    - python -m pytest -v --junitxml=junit.xml
  artifacts:
    reports:
      junit: test/openai-proxy/junit.xml
  dependencies:
    - build-docker-images

# ================================
# STAGING DEPLOYMENT
# ================================

deploy-staging:
  stage: deploy-staging
  image: docker:24-dind
  services:
    - docker:24-dind
  variables:
    DEPLOY_ENV: staging
    STACK_NAME: bzzz-v2-staging
  environment:
    name: staging
    url: https://bzzz-staging.deepblack.cloud
  before_script:
    - apk add --no-cache openssh-client
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - ssh-keyscan -H 192.168.1.27 >> ~/.ssh/known_hosts
  script:
    # Copy deployment files to staging environment
    - scp infrastructure/docker-compose.staging.yml tony@192.168.1.27:/rust/bzzz-v2/
    - scp infrastructure/configs/staging/* tony@192.168.1.27:/rust/bzzz-v2/config/
    
    # Deploy to staging swarm
    - |
      ssh tony@192.168.1.27 << 'EOF'
        cd /rust/bzzz-v2
        export IMAGE_TAG=${CI_COMMIT_SHORT_SHA}
        docker stack deploy -c docker-compose.staging.yml ${STACK_NAME}
        
        # Wait for deployment
        timeout 300 bash -c 'until docker service ls --filter label=com.docker.stack.namespace=${STACK_NAME} --format "{{.Replicas}}" | grep -v "0/"; do sleep 10; done'
      EOF
    
    # Health check staging deployment
    - sleep 60
    - curl -f https://bzzz-staging.deepblack.cloud/health
  dependencies:
    - build-docker-images
    - p2p-integration-test
  rules:
    - if: '$CI_COMMIT_BRANCH == "develop"'
    - if: '$CI_COMMIT_BRANCH == "main"'

# ================================
# PERFORMANCE TESTING
# ================================

performance-test:
  stage: performance-test
  image: loadimpact/k6:latest
  script:
    - cd test/performance
    - k6 run --out json=performance-results.json performance-test.js
    - k6 run --out json=dht-performance.json dht-performance-test.js
  artifacts:
    paths:
      - test/performance/performance-results.json
      - test/performance/dht-performance.json
    reports:
      performance: test/performance/performance-results.json
    expire_in: 1 week
  environment:
    name: staging
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
    - when: manual
      if: '$CI_COMMIT_BRANCH == "develop"'

load-test:
  stage: performance-test
  image: python:3.11-alpine
  script:
    - cd test/load
    - pip install locust requests
    - locust --headless --users 100 --spawn-rate 10 --run-time 5m --host https://bzzz-staging.deepblack.cloud
  artifacts:
    paths:
      - test/load/locust_stats.html
    expire_in: 1 week
  environment:
    name: staging
  rules:
    - when: manual

# ================================
# PRODUCTION DEPLOYMENT
# ================================

deploy-production:
  stage: deploy-production
  image: docker:24-dind
  services:
    - docker:24-dind
  variables:
    DEPLOY_ENV: production
    STACK_NAME: bzzz-v2
  environment:
    name: production
    url: https://bzzz.deepblack.cloud
  before_script:
    - apk add --no-cache openssh-client
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - ssh-keyscan -H 192.168.1.27 >> ~/.ssh/known_hosts
  script:
    # Backup current production state
    - |
      ssh tony@192.168.1.27 << 'EOF'
        mkdir -p /rust/bzzz-v2/backup/$(date +%Y%m%d-%H%M%S)
        docker service ls --filter label=com.docker.stack.namespace=bzzz-v2 --format "table {{.Name}}\t{{.Image}}" > /rust/bzzz-v2/backup/$(date +%Y%m%d-%H%M%S)/pre-deployment-services.txt
      EOF
    
    # Copy production deployment files
    - scp infrastructure/docker-compose.swarm.yml tony@192.168.1.27:/rust/bzzz-v2/
    - scp infrastructure/configs/production/* tony@192.168.1.27:/rust/bzzz-v2/config/
    
    # Deploy to production with blue-green strategy
    - |
      ssh tony@192.168.1.27 << 'EOF'
        cd /rust/bzzz-v2
        export IMAGE_TAG=${CI_COMMIT_SHORT_SHA}
        
        # Deploy new version
        docker stack deploy -c docker-compose.swarm.yml ${STACK_NAME}
        
        # Wait for healthy deployment
        timeout 600 bash -c 'until docker service ls --filter label=com.docker.stack.namespace=${STACK_NAME} --format "{{.Replicas}}" | grep -v "0/" | wc -l | grep -q 8; do sleep 15; done'
        
        echo "Production deployment completed successfully"
      EOF
    
    # Verify production health
    - sleep 120
    - curl -f https://bzzz.deepblack.cloud/health
    - curl -f https://mcp.deepblack.cloud/health
  dependencies:
    - deploy-staging
    - performance-test
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: manual
  
rollback-production:
  stage: deploy-production
  image: docker:24-dind
  variables:
    STACK_NAME: bzzz-v2
  environment:
    name: production
    action: rollback
  before_script:
    - apk add --no-cache openssh-client
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - ssh-keyscan -H 192.168.1.27 >> ~/.ssh/known_hosts
  script:
    - |
      ssh tony@192.168.1.27 << 'EOF'
        cd /rust/bzzz-v2
        
        # Get previous stable image tags
        PREVIOUS_TAG=$(docker service inspect bzzz-v2_bzzz-agent --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' | cut -d: -f2)
        
        # Rollback by redeploying previous version
        export IMAGE_TAG=$PREVIOUS_TAG
        docker stack deploy -c docker-compose.swarm.yml ${STACK_NAME}
        
        echo "Production rollback completed"
      EOF
  rules:
    - when: manual
      if: '$CI_COMMIT_BRANCH == "main"'

# ================================
# POST-DEPLOYMENT VALIDATION
# ================================

post-deploy-validation:
  stage: post-deploy-validation
  image: curlimages/curl:latest
  script:
    - curl -f https://bzzz.deepblack.cloud/health
    - curl -f https://mcp.deepblack.cloud/health  
    - curl -f https://resolve.deepblack.cloud/health
    - curl -f https://openai.deepblack.cloud/health
    
    # Test basic functionality
    - |
      # Test bzzz:// address resolution
      CONTENT_HASH=$(curl -s https://bzzz.deepblack.cloud/api/v2/test-content | jq -r '.hash')
      curl -f "https://resolve.deepblack.cloud/bzzz://${CONTENT_HASH}"
      
      # Test MCP endpoint
      curl -X POST https://mcp.deepblack.cloud/api/tools/list \
        -H "Content-Type: application/json" \
        -d '{"method": "tools/list"}'
  environment:
    name: production
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
  needs:
    - deploy-production

smoke-tests:
  stage: post-deploy-validation
  image: golang:$GO_VERSION-alpine
  script:
    - cd test/smoke
    - go test -v ./... -base-url=https://bzzz.deepblack.cloud
  environment:
    name: production
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
  needs:
    - deploy-production

# ================================
# NOTIFICATION STAGE (implicit)
# ================================

notify-success:
  stage: .post
  image: curlimages/curl:latest
  script:
    - |
      curl -X POST $SLACK_WEBHOOK_URL \
        -H 'Content-type: application/json' \
        -d '{
          "text": "🚀 BZZZ v2 Pipeline Success",
          "attachments": [{
            "color": "good",
            "fields": [{
              "title": "Branch",
              "value": "'$CI_COMMIT_BRANCH'",
              "short": true
            }, {
              "title": "Commit",
              "value": "'$CI_COMMIT_SHORT_SHA'",
              "short": true
            }, {
              "title": "Pipeline",
              "value": "'$CI_PIPELINE_URL'",
              "short": false
            }]
          }]
        }'
  rules:
    - if: '$CI_PIPELINE_STATUS == "success" && $CI_COMMIT_BRANCH == "main"'
  when: on_success

notify-failure:
  stage: .post
  image: curlimages/curl:latest
  script:
    - |
      curl -X POST $SLACK_WEBHOOK_URL \
        -H 'Content-type: application/json' \
        -d '{
          "text": "❌ BZZZ v2 Pipeline Failed",
          "attachments": [{
            "color": "danger",
            "fields": [{
              "title": "Branch", 
              "value": "'$CI_COMMIT_BRANCH'",
              "short": true
            }, {
              "title": "Commit",
              "value": "'$CI_COMMIT_SHORT_SHA'",
              "short": true
            }, {
              "title": "Pipeline",
              "value": "'$CI_PIPELINE_URL'",
              "short": false
            }]
          }]
        }'
  rules:
    - when: on_failure