Prepare for v2 development: Add MCP integration and future development planning

- Add FUTURE_DEVELOPMENT.md with comprehensive v2 protocol specification
- Add MCP integration design and implementation foundation
- Add infrastructure and deployment configurations
- Update system architecture for v2 evolution

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

infrastructure/ci-cd/.gitlab-ci.yml

@@ -0,0 +1,643 @@
+# BZZZ v2 GitLab CI/CD Pipeline
+# Comprehensive build, test, and deployment pipeline for BZZZ v2
+
+variables:
+  REGISTRY: registry.home.deepblack.cloud
+  REGISTRY_NAMESPACE: bzzz
+  GO_VERSION: "1.21"
+  DOCKER_BUILDKIT: "1"
+  COMPOSE_DOCKER_CLI_BUILD: "1"
+  POSTGRES_VERSION: "15"
+  REDIS_VERSION: "7"
+  
+  # Semantic versioning
+  VERSION_PREFIX: "v2"
+  
+stages:
+  - lint
+  - test
+  - build
+  - security-scan
+  - integration-test
+  - deploy-staging
+  - performance-test
+  - deploy-production
+  - post-deploy-validation
+
+# Cache configuration
+cache:
+  key: "${CI_COMMIT_REF_SLUG}"
+  paths:
+    - .cache/go-mod/
+    - .cache/docker/
+    - vendor/
+
+before_script:
+  - export GOPATH=$CI_PROJECT_DIR/.cache/go-mod
+  - export GOCACHE=$CI_PROJECT_DIR/.cache/go-build
+  - mkdir -p .cache/{go-mod,go-build,docker}
+
+# ================================
+# LINT STAGE
+# ================================
+
+golang-lint:
+  stage: lint
+  image: golangci/golangci-lint:v1.55-alpine
+  script:
+    - golangci-lint run ./... --timeout 10m
+    - go mod tidy
+    - git diff --exit-code go.mod go.sum
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_COMMIT_BRANCH == "main"'
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+
+dockerfile-lint:
+  stage: lint
+  image: hadolint/hadolint:latest-debian
+  script:
+    - hadolint infrastructure/dockerfiles/Dockerfile.*
+    - hadolint Dockerfile
+  rules:
+    - changes:
+        - "infrastructure/dockerfiles/*"
+        - "Dockerfile*"
+
+yaml-lint:
+  stage: lint
+  image: cytopia/yamllint:latest
+  script:
+    - yamllint infrastructure/
+    - yamllint .gitlab-ci.yml
+  rules:
+    - changes:
+        - "infrastructure/**/*.yml"
+        - "infrastructure/**/*.yaml" 
+        - ".gitlab-ci.yml"
+
+# ================================
+# TEST STAGE
+# ================================
+
+unit-tests:
+  stage: test
+  image: golang:$GO_VERSION-alpine
+  services:
+    - name: postgres:$POSTGRES_VERSION-alpine
+      alias: postgres
+    - name: redis:$REDIS_VERSION-alpine
+      alias: redis
+  variables:
+    POSTGRES_DB: bzzz_test
+    POSTGRES_USER: test
+    POSTGRES_PASSWORD: testpass
+    POSTGRES_HOST: postgres
+    REDIS_HOST: redis
+    CGO_ENABLED: 0
+  before_script:
+    - apk add --no-cache git make gcc musl-dev
+    - export GOPATH=$CI_PROJECT_DIR/.cache/go-mod
+    - export GOCACHE=$CI_PROJECT_DIR/.cache/go-build
+  script:
+    - go mod download
+    - go test -v -race -coverprofile=coverage.out ./...
+    - go tool cover -html=coverage.out -o coverage.html
+    - go tool cover -func=coverage.out | grep total | awk '{print "Coverage: " $3}'
+  coverage: '/Coverage: \d+\.\d+/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+    paths:
+      - coverage.html
+      - coverage.out
+    expire_in: 1 week
+
+p2p-protocol-tests:
+  stage: test
+  image: golang:$GO_VERSION-alpine
+  script:
+    - apk add --no-cache git make gcc musl-dev
+    - go test -v -tags=p2p ./p2p/... ./dht/... 
+    - go test -v -tags=integration ./test/p2p/...
+  rules:
+    - changes:
+        - "p2p/**/*"
+        - "dht/**/*"
+        - "test/p2p/**/*"
+
+content-store-tests:
+  stage: test
+  image: golang:$GO_VERSION-alpine
+  script:
+    - apk add --no-cache git make gcc musl-dev
+    - go test -v -tags=storage ./storage/... ./blake3/...
+    - go test -v -benchmem -bench=. ./storage/...
+  artifacts:
+    paths:
+      - benchmark.out
+    expire_in: 1 week
+  rules:
+    - changes:
+        - "storage/**/*"
+        - "blake3/**/*"
+
+conversation-tests:
+  stage: test
+  image: golang:$GO_VERSION-alpine
+  services:
+    - name: postgres:$POSTGRES_VERSION-alpine
+      alias: postgres
+  variables:
+    POSTGRES_DB: bzzz_conversation_test
+    POSTGRES_USER: test
+    POSTGRES_PASSWORD: testpass
+    POSTGRES_HOST: postgres
+  script:
+    - apk add --no-cache git make gcc musl-dev postgresql-client
+    - until pg_isready -h postgres -p 5432 -U test; do sleep 1; done
+    - go test -v -tags=conversation ./conversation/... ./threading/...
+  rules:
+    - changes:
+        - "conversation/**/*"
+        - "threading/**/*"
+
+# ================================
+# BUILD STAGE 
+# ================================
+
+build-binaries:
+  stage: build
+  image: golang:$GO_VERSION-alpine
+  before_script:
+    - apk add --no-cache git make gcc musl-dev upx
+    - export GOPATH=$CI_PROJECT_DIR/.cache/go-mod
+    - export GOCACHE=$CI_PROJECT_DIR/.cache/go-build
+  script:
+    - make build-all
+    - upx --best --lzma dist/bzzz-*
+    - ls -la dist/
+  artifacts:
+    paths:
+      - dist/
+    expire_in: 1 week
+
+build-docker-images:
+  stage: build
+  image: docker:24-dind
+  services:
+    - docker:24-dind
+  variables:
+    IMAGE_TAG: ${CI_COMMIT_SHORT_SHA}
+    DOCKER_DRIVER: overlay2
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $REGISTRY
+    - docker buildx create --use --driver docker-container
+  script:
+    # Build all images in parallel
+    - |
+      docker buildx build \
+        --platform linux/amd64,linux/arm64 \
+        --build-arg VERSION=${VERSION_PREFIX}.${CI_PIPELINE_ID} \
+        --build-arg COMMIT=${CI_COMMIT_SHORT_SHA} \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-agent:$IMAGE_TAG \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-agent:latest \
+        --file infrastructure/dockerfiles/Dockerfile.agent \
+        --push .
+    
+    - |
+      docker buildx build \
+        --platform linux/amd64,linux/arm64 \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-mcp:$IMAGE_TAG \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-mcp:latest \
+        --file infrastructure/dockerfiles/Dockerfile.mcp \
+        --push .
+    
+    - |
+      docker buildx build \
+        --platform linux/amd64,linux/arm64 \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-openai-proxy:$IMAGE_TAG \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-openai-proxy:latest \
+        --file infrastructure/dockerfiles/Dockerfile.proxy \
+        --push .
+    
+    - |
+      docker buildx build \
+        --platform linux/amd64,linux/arm64 \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-resolver:$IMAGE_TAG \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-resolver:latest \
+        --file infrastructure/dockerfiles/Dockerfile.resolver \
+        --push .
+        
+    - |
+      docker buildx build \
+        --platform linux/amd64,linux/arm64 \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-dht:$IMAGE_TAG \
+        --tag $REGISTRY/$REGISTRY_NAMESPACE/bzzz-dht:latest \
+        --file infrastructure/dockerfiles/Dockerfile.dht \
+        --push .
+  
+  dependencies:
+    - build-binaries
+
+# ================================
+# SECURITY SCAN STAGE
+# ================================
+
+container-security-scan:
+  stage: security-scan
+  image: aquasec/trivy:latest
+  script:
+    - |
+      for component in agent mcp openai-proxy resolver dht; do
+        echo "Scanning bzzz-${component}..."
+        trivy image --exit-code 1 --severity HIGH,CRITICAL \
+          --format json --output trivy-${component}.json \
+          $REGISTRY/$REGISTRY_NAMESPACE/bzzz-${component}:${CI_COMMIT_SHORT_SHA}
+      done
+  artifacts:
+    reports:
+      container_scanning: trivy-*.json
+    expire_in: 1 week
+  dependencies:
+    - build-docker-images
+  allow_failure: true
+
+dependency-security-scan:
+  stage: security-scan
+  image: golang:$GO_VERSION-alpine
+  script:
+    - go install golang.org/x/vuln/cmd/govulncheck@latest
+    - govulncheck ./...
+  allow_failure: true
+
+secrets-scan:
+  stage: security-scan
+  image: trufflesecurity/trufflehog:latest
+  script:
+    - trufflehog filesystem --directory=. --fail --json
+  allow_failure: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+
+# ================================
+# INTEGRATION TEST STAGE
+# ================================
+
+p2p-integration-test:
+  stage: integration-test
+  image: docker:24-dind
+  services:
+    - docker:24-dind
+  variables:
+    COMPOSE_PROJECT_NAME: bzzz-integration-${CI_PIPELINE_ID}
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $REGISTRY
+    - apk add --no-cache docker-compose curl jq
+  script:
+    - cd infrastructure/testing
+    - docker-compose -f docker-compose.integration.yml up -d
+    - sleep 60  # Wait for services to start
+    - ./scripts/test-p2p-mesh.sh
+    - ./scripts/test-dht-discovery.sh  
+    - ./scripts/test-content-addressing.sh
+    - docker-compose -f docker-compose.integration.yml logs
+  after_script:
+    - cd infrastructure/testing
+    - docker-compose -f docker-compose.integration.yml down -v
+  artifacts:
+    paths:
+      - infrastructure/testing/test-results/
+    expire_in: 1 week
+    when: always
+  dependencies:
+    - build-docker-images
+
+mcp-integration-test:
+  stage: integration-test
+  image: node:18-alpine
+  services:
+    - name: $REGISTRY/$REGISTRY_NAMESPACE/bzzz-mcp:${CI_COMMIT_SHORT_SHA}
+      alias: mcp-server
+    - name: $REGISTRY/$REGISTRY_NAMESPACE/bzzz-agent:${CI_COMMIT_SHORT_SHA} 
+      alias: bzzz-agent
+  script:
+    - cd test/mcp
+    - npm install
+    - npm test
+  artifacts:
+    reports:
+      junit: test/mcp/junit.xml
+  dependencies:
+    - build-docker-images
+
+openai-proxy-test:
+  stage: integration-test
+  image: python:3.11-alpine
+  services:
+    - name: $REGISTRY/$REGISTRY_NAMESPACE/bzzz-openai-proxy:${CI_COMMIT_SHORT_SHA}
+      alias: openai-proxy
+    - name: redis:$REDIS_VERSION-alpine
+      alias: redis
+  variables:
+    OPENAI_API_KEY: "test-key-mock"
+    REDIS_HOST: redis
+  script:
+    - cd test/openai-proxy
+    - pip install -r requirements.txt
+    - python -m pytest -v --junitxml=junit.xml
+  artifacts:
+    reports:
+      junit: test/openai-proxy/junit.xml
+  dependencies:
+    - build-docker-images
+
+# ================================
+# STAGING DEPLOYMENT
+# ================================
+
+deploy-staging:
+  stage: deploy-staging
+  image: docker:24-dind
+  services:
+    - docker:24-dind
+  variables:
+    DEPLOY_ENV: staging
+    STACK_NAME: bzzz-v2-staging
+  environment:
+    name: staging
+    url: https://bzzz-staging.deepblack.cloud
+  before_script:
+    - apk add --no-cache openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan -H 192.168.1.27 >> ~/.ssh/known_hosts
+  script:
+    # Copy deployment files to staging environment
+    - scp infrastructure/docker-compose.staging.yml tony@192.168.1.27:/rust/bzzz-v2/
+    - scp infrastructure/configs/staging/* tony@192.168.1.27:/rust/bzzz-v2/config/
+    
+    # Deploy to staging swarm
+    - |
+      ssh tony@192.168.1.27 << 'EOF'
+        cd /rust/bzzz-v2
+        export IMAGE_TAG=${CI_COMMIT_SHORT_SHA}
+        docker stack deploy -c docker-compose.staging.yml ${STACK_NAME}
+        
+        # Wait for deployment
+        timeout 300 bash -c 'until docker service ls --filter label=com.docker.stack.namespace=${STACK_NAME} --format "{{.Replicas}}" | grep -v "0/"; do sleep 10; done'
+      EOF
+    
+    # Health check staging deployment
+    - sleep 60
+    - curl -f https://bzzz-staging.deepblack.cloud/health
+  dependencies:
+    - build-docker-images
+    - p2p-integration-test
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+    - if: '$CI_COMMIT_BRANCH == "main"'
+
+# ================================
+# PERFORMANCE TESTING
+# ================================
+
+performance-test:
+  stage: performance-test
+  image: loadimpact/k6:latest
+  script:
+    - cd test/performance
+    - k6 run --out json=performance-results.json performance-test.js
+    - k6 run --out json=dht-performance.json dht-performance-test.js
+  artifacts:
+    paths:
+      - test/performance/performance-results.json
+      - test/performance/dht-performance.json
+    reports:
+      performance: test/performance/performance-results.json
+    expire_in: 1 week
+  environment:
+    name: staging
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+    - when: manual
+      if: '$CI_COMMIT_BRANCH == "develop"'
+
+load-test:
+  stage: performance-test
+  image: python:3.11-alpine
+  script:
+    - cd test/load
+    - pip install locust requests
+    - locust --headless --users 100 --spawn-rate 10 --run-time 5m --host https://bzzz-staging.deepblack.cloud
+  artifacts:
+    paths:
+      - test/load/locust_stats.html
+    expire_in: 1 week
+  environment:
+    name: staging
+  rules:
+    - when: manual
+
+# ================================
+# PRODUCTION DEPLOYMENT
+# ================================
+
+deploy-production:
+  stage: deploy-production
+  image: docker:24-dind
+  services:
+    - docker:24-dind
+  variables:
+    DEPLOY_ENV: production
+    STACK_NAME: bzzz-v2
+  environment:
+    name: production
+    url: https://bzzz.deepblack.cloud
+  before_script:
+    - apk add --no-cache openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan -H 192.168.1.27 >> ~/.ssh/known_hosts
+  script:
+    # Backup current production state
+    - |
+      ssh tony@192.168.1.27 << 'EOF'
+        mkdir -p /rust/bzzz-v2/backup/$(date +%Y%m%d-%H%M%S)
+        docker service ls --filter label=com.docker.stack.namespace=bzzz-v2 --format "table {{.Name}}\t{{.Image}}" > /rust/bzzz-v2/backup/$(date +%Y%m%d-%H%M%S)/pre-deployment-services.txt
+      EOF
+    
+    # Copy production deployment files
+    - scp infrastructure/docker-compose.swarm.yml tony@192.168.1.27:/rust/bzzz-v2/
+    - scp infrastructure/configs/production/* tony@192.168.1.27:/rust/bzzz-v2/config/
+    
+    # Deploy to production with blue-green strategy
+    - |
+      ssh tony@192.168.1.27 << 'EOF'
+        cd /rust/bzzz-v2
+        export IMAGE_TAG=${CI_COMMIT_SHORT_SHA}
+        
+        # Deploy new version
+        docker stack deploy -c docker-compose.swarm.yml ${STACK_NAME}
+        
+        # Wait for healthy deployment
+        timeout 600 bash -c 'until docker service ls --filter label=com.docker.stack.namespace=${STACK_NAME} --format "{{.Replicas}}" | grep -v "0/" | wc -l | grep -q 8; do sleep 15; done'
+        
+        echo "Production deployment completed successfully"
+      EOF
+    
+    # Verify production health
+    - sleep 120
+    - curl -f https://bzzz.deepblack.cloud/health
+    - curl -f https://mcp.deepblack.cloud/health
+  dependencies:
+    - deploy-staging
+    - performance-test
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      when: manual
+  
+rollback-production:
+  stage: deploy-production
+  image: docker:24-dind
+  variables:
+    STACK_NAME: bzzz-v2
+  environment:
+    name: production
+    action: rollback
+  before_script:
+    - apk add --no-cache openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan -H 192.168.1.27 >> ~/.ssh/known_hosts
+  script:
+    - |
+      ssh tony@192.168.1.27 << 'EOF'
+        cd /rust/bzzz-v2
+        
+        # Get previous stable image tags
+        PREVIOUS_TAG=$(docker service inspect bzzz-v2_bzzz-agent --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' | cut -d: -f2)
+        
+        # Rollback by redeploying previous version
+        export IMAGE_TAG=$PREVIOUS_TAG
+        docker stack deploy -c docker-compose.swarm.yml ${STACK_NAME}
+        
+        echo "Production rollback completed"
+      EOF
+  rules:
+    - when: manual
+      if: '$CI_COMMIT_BRANCH == "main"'
+
+# ================================
+# POST-DEPLOYMENT VALIDATION
+# ================================
+
+post-deploy-validation:
+  stage: post-deploy-validation
+  image: curlimages/curl:latest
+  script:
+    - curl -f https://bzzz.deepblack.cloud/health
+    - curl -f https://mcp.deepblack.cloud/health  
+    - curl -f https://resolve.deepblack.cloud/health
+    - curl -f https://openai.deepblack.cloud/health
+    
+    # Test basic functionality
+    - |
+      # Test bzzz:// address resolution
+      CONTENT_HASH=$(curl -s https://bzzz.deepblack.cloud/api/v2/test-content | jq -r '.hash')
+      curl -f "https://resolve.deepblack.cloud/bzzz://${CONTENT_HASH}"
+      
+      # Test MCP endpoint
+      curl -X POST https://mcp.deepblack.cloud/api/tools/list \
+        -H "Content-Type: application/json" \
+        -d '{"method": "tools/list"}'
+  environment:
+    name: production
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+  needs:
+    - deploy-production
+
+smoke-tests:
+  stage: post-deploy-validation
+  image: golang:$GO_VERSION-alpine
+  script:
+    - cd test/smoke
+    - go test -v ./... -base-url=https://bzzz.deepblack.cloud
+  environment:
+    name: production
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+  needs:
+    - deploy-production
+
+# ================================
+# NOTIFICATION STAGE (implicit)
+# ================================
+
+notify-success:
+  stage: .post
+  image: curlimages/curl:latest
+  script:
+    - |
+      curl -X POST $SLACK_WEBHOOK_URL \
+        -H 'Content-type: application/json' \
+        -d '{
+          "text": "🚀 BZZZ v2 Pipeline Success",
+          "attachments": [{
+            "color": "good",
+            "fields": [{
+              "title": "Branch",
+              "value": "'$CI_COMMIT_BRANCH'",
+              "short": true
+            }, {
+              "title": "Commit",
+              "value": "'$CI_COMMIT_SHORT_SHA'",
+              "short": true
+            }, {
+              "title": "Pipeline",
+              "value": "'$CI_PIPELINE_URL'",
+              "short": false
+            }]
+          }]
+        }'
+  rules:
+    - if: '$CI_PIPELINE_STATUS == "success" && $CI_COMMIT_BRANCH == "main"'
+  when: on_success
+
+notify-failure:
+  stage: .post
+  image: curlimages/curl:latest
+  script:
+    - |
+      curl -X POST $SLACK_WEBHOOK_URL \
+        -H 'Content-type: application/json' \
+        -d '{
+          "text": "❌ BZZZ v2 Pipeline Failed",
+          "attachments": [{
+            "color": "danger",
+            "fields": [{
+              "title": "Branch", 
+              "value": "'$CI_COMMIT_BRANCH'",
+              "short": true
+            }, {
+              "title": "Commit",
+              "value": "'$CI_COMMIT_SHORT_SHA'",
+              "short": true
+            }, {
+              "title": "Pipeline",
+              "value": "'$CI_PIPELINE_URL'",
+              "short": false
+            }]
+          }]
+        }'
+  rules:
+    - when: on_failure