package api import ( "context" "encoding/json" "fmt" "net" "net/http" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" "sync" "time" "golang.org/x/crypto/ssh" "chorus.services/bzzz/pkg/config" "chorus.services/bzzz/pkg/security" "chorus.services/bzzz/repository" ) // SetupManager handles the initial configuration setup for BZZZ type SetupManager struct { configPath string factory repository.ProviderFactory validator *security.SecurityValidator } // NewSetupManager creates a new setup manager func NewSetupManager(configPath string) *SetupManager { return &SetupManager{ configPath: configPath, factory: &repository.DefaultProviderFactory{}, validator: security.NewSecurityValidator(), } } // IsSetupRequired checks if initial setup is needed func (s *SetupManager) IsSetupRequired() bool { // Check if config file exists and is valid if _, err := os.Stat(s.configPath); os.IsNotExist(err) { return true } // Try to load and validate existing config cfg, err := config.LoadConfig(s.configPath) if err != nil { return true } // Check if essential configuration is present return cfg.Agent.ID == "" || cfg.WHOOSHAPI.BaseURL == "" } // SystemInfo holds system detection information type SystemInfo struct { OS string `json:"os"` Architecture string `json:"architecture"` CPUCores int `json:"cpu_cores"` Memory int64 `json:"memory_mb"` GPUs []GPUInfo `json:"gpus"` Network NetworkInfo `json:"network"` Storage StorageInfo `json:"storage"` Docker DockerInfo `json:"docker"` } // GPUInfo holds GPU detection information type GPUInfo struct { Name string `json:"name"` Memory string `json:"memory"` Driver string `json:"driver"` Type string `json:"type"` // nvidia, amd, intel } // NetworkInfo holds network configuration type NetworkInfo struct { Hostname string `json:"hostname"` Interfaces []string `json:"interfaces"` PublicIP string `json:"public_ip,omitempty"` PrivateIPs []string `json:"private_ips"` DockerBridge string `json:"docker_bridge,omitempty"` } // StorageInfo holds storage information type StorageInfo struct { TotalSpace int64 `json:"total_space_gb"` FreeSpace int64 `json:"free_space_gb"` MountPath string `json:"mount_path"` } // DockerInfo holds Docker environment information type DockerInfo struct { Available bool `json:"available"` Version string `json:"version,omitempty"` ComposeAvailable bool `json:"compose_available"` SwarmMode bool `json:"swarm_mode"` } // DetectSystemInfo performs comprehensive system detection func (s *SetupManager) DetectSystemInfo() (*SystemInfo, error) { info := &SystemInfo{ OS: runtime.GOOS, Architecture: runtime.GOARCH, CPUCores: runtime.NumCPU(), } // Detect memory if memory, err := s.detectMemory(); err == nil { info.Memory = memory } // Detect GPUs if gpus, err := s.detectGPUs(); err == nil { info.GPUs = gpus } // Detect network configuration if network, err := s.detectNetwork(); err == nil { info.Network = network } // Detect storage if storage, err := s.detectStorage(); err == nil { info.Storage = storage } // Detect Docker if docker, err := s.detectDocker(); err == nil { info.Docker = docker } return info, nil } // detectMemory detects system memory func (s *SetupManager) detectMemory() (int64, error) { switch runtime.GOOS { case "linux": // Read from /proc/meminfo content, err := os.ReadFile("/proc/meminfo") if err != nil { return 0, err } lines := strings.Split(string(content), "\n") for _, line := range lines { if strings.HasPrefix(line, "MemTotal:") { parts := strings.Fields(line) if len(parts) >= 2 { kb, err := strconv.ParseInt(parts[1], 10, 64) if err == nil { return kb / 1024, nil // Convert KB to MB } } } } case "darwin": // Use sysctl on macOS cmd := exec.Command("sysctl", "-n", "hw.memsize") output, err := cmd.Output() if err == nil { bytes, err := strconv.ParseInt(strings.TrimSpace(string(output)), 10, 64) if err == nil { return bytes / (1024 * 1024), nil // Convert bytes to MB } } } return 0, fmt.Errorf("memory detection not supported on %s", runtime.GOOS) } // detectGPUs detects available GPUs func (s *SetupManager) detectGPUs() ([]GPUInfo, error) { var gpus []GPUInfo // Try NVIDIA GPUs first if nvidiaGPUs, err := s.detectNVIDIAGPUs(); err == nil { gpus = append(gpus, nvidiaGPUs...) } // Try AMD GPUs if amdGPUs, err := s.detectAMDGPUs(); err == nil { gpus = append(gpus, amdGPUs...) } // Try Intel GPUs (basic detection) if intelGPUs, err := s.detectIntelGPUs(); err == nil { gpus = append(gpus, intelGPUs...) } return gpus, nil } // detectNVIDIAGPUs detects NVIDIA GPUs using nvidia-smi func (s *SetupManager) detectNVIDIAGPUs() ([]GPUInfo, error) { var gpus []GPUInfo // Check if nvidia-smi is available cmd := exec.Command("nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader,nounits") output, err := cmd.Output() if err != nil { return nil, err } lines := strings.Split(strings.TrimSpace(string(output)), "\n") for _, line := range lines { if line == "" { continue } parts := strings.Split(line, ", ") if len(parts) >= 3 { gpu := GPUInfo{ Name: strings.TrimSpace(parts[0]), Memory: strings.TrimSpace(parts[1]) + " MB", Driver: strings.TrimSpace(parts[2]), Type: "nvidia", } gpus = append(gpus, gpu) } } return gpus, nil } // detectAMDGPUs detects AMD GPUs func (s *SetupManager) detectAMDGPUs() ([]GPUInfo, error) { var gpus []GPUInfo // Try rocm-smi for AMD GPUs cmd := exec.Command("rocm-smi", "--showproductname", "--showmeminfo", "vram") output, err := cmd.Output() if err != nil { return nil, err } // Parse rocm-smi output (simplified) if strings.Contains(string(output), "GPU") { gpu := GPUInfo{ Name: "AMD GPU (detected)", Memory: "Unknown", Driver: "ROCm", Type: "amd", } gpus = append(gpus, gpu) } return gpus, nil } // detectIntelGPUs detects Intel integrated GPUs func (s *SetupManager) detectIntelGPUs() ([]GPUInfo, error) { var gpus []GPUInfo switch runtime.GOOS { case "linux": // Check for Intel GPU in /sys/class/drm if _, err := os.Stat("/sys/class/drm/card0"); err == nil { // Basic Intel GPU detection gpu := GPUInfo{ Name: "Intel Integrated Graphics", Memory: "Shared", Driver: "i915", Type: "intel", } gpus = append(gpus, gpu) } } return gpus, nil } // detectNetwork detects network configuration func (s *SetupManager) detectNetwork() (NetworkInfo, error) { info := NetworkInfo{ PrivateIPs: []string{}, } // Get hostname if hostname, err := os.Hostname(); err == nil { info.Hostname = hostname } // Detect network interfaces (simplified) switch runtime.GOOS { case "linux": cmd := exec.Command("ip", "addr", "show") output, err := cmd.Output() if err == nil { s.parseLinuxNetworkInfo(string(output), &info) } case "darwin": cmd := exec.Command("ifconfig") output, err := cmd.Output() if err == nil { s.parseDarwinNetworkInfo(string(output), &info) } } return info, nil } // parseLinuxNetworkInfo parses Linux network info from ip command func (s *SetupManager) parseLinuxNetworkInfo(output string, info *NetworkInfo) { lines := strings.Split(output, "\n") var currentInterface string for _, line := range lines { line = strings.TrimSpace(line) // Interface line if strings.Contains(line, ": ") && !strings.HasPrefix(line, "inet") { parts := strings.Split(line, ":") if len(parts) >= 2 { currentInterface = strings.TrimSpace(parts[1]) if currentInterface != "lo" { // Skip loopback info.Interfaces = append(info.Interfaces, currentInterface) } } } // IP address line if strings.HasPrefix(line, "inet ") && currentInterface != "lo" { parts := strings.Fields(line) if len(parts) >= 2 { ip := strings.Split(parts[1], "/")[0] if s.isPrivateIP(ip) { info.PrivateIPs = append(info.PrivateIPs, ip) } } } } } // parseDarwinNetworkInfo parses macOS network info from ifconfig func (s *SetupManager) parseDarwinNetworkInfo(output string, info *NetworkInfo) { lines := strings.Split(output, "\n") var currentInterface string for _, line := range lines { // Interface line if !strings.HasPrefix(line, "\t") && strings.Contains(line, ":") { parts := strings.Split(line, ":") if len(parts) >= 1 { currentInterface = strings.TrimSpace(parts[0]) if currentInterface != "lo0" { // Skip loopback info.Interfaces = append(info.Interfaces, currentInterface) } } } // IP address line if strings.Contains(line, "inet ") && currentInterface != "lo0" { parts := strings.Fields(strings.TrimSpace(line)) if len(parts) >= 2 { ip := parts[1] if s.isPrivateIP(ip) { info.PrivateIPs = append(info.PrivateIPs, ip) } } } } } // isPrivateIP checks if an IP address is private func (s *SetupManager) isPrivateIP(ip string) bool { return strings.HasPrefix(ip, "192.168.") || strings.HasPrefix(ip, "10.") || strings.HasPrefix(ip, "172.16.") || strings.HasPrefix(ip, "172.17.") || strings.HasPrefix(ip, "172.18.") || strings.HasPrefix(ip, "172.19.") || strings.HasPrefix(ip, "172.20.") || strings.HasPrefix(ip, "172.21.") || strings.HasPrefix(ip, "172.22.") || strings.HasPrefix(ip, "172.23.") || strings.HasPrefix(ip, "172.24.") || strings.HasPrefix(ip, "172.25.") || strings.HasPrefix(ip, "172.26.") || strings.HasPrefix(ip, "172.27.") || strings.HasPrefix(ip, "172.28.") || strings.HasPrefix(ip, "172.29.") || strings.HasPrefix(ip, "172.30.") || strings.HasPrefix(ip, "172.31.") } // detectStorage detects storage information func (s *SetupManager) detectStorage() (StorageInfo, error) { info := StorageInfo{ MountPath: "/", } // Get current working directory for storage detection wd, err := os.Getwd() if err != nil { wd = "/" } info.MountPath = wd switch runtime.GOOS { case "linux", "darwin": cmd := exec.Command("df", "-BG", wd) output, err := cmd.Output() if err == nil { lines := strings.Split(string(output), "\n") if len(lines) >= 2 { fields := strings.Fields(lines[1]) if len(fields) >= 4 { // Parse total and available space if total, err := strconv.ParseInt(strings.TrimSuffix(fields[1], "G"), 10, 64); err == nil { info.TotalSpace = total } if free, err := strconv.ParseInt(strings.TrimSuffix(fields[3], "G"), 10, 64); err == nil { info.FreeSpace = free } } } } } return info, nil } // detectDocker detects Docker environment func (s *SetupManager) detectDocker() (DockerInfo, error) { info := DockerInfo{} // Check if docker command is available cmd := exec.Command("docker", "--version") output, err := cmd.Output() if err == nil { info.Available = true info.Version = strings.TrimSpace(string(output)) } // Check if docker compose is available (modern Docker includes compose as subcommand) cmd = exec.Command("docker", "compose", "version") if err := cmd.Run(); err == nil { info.ComposeAvailable = true } else { // Fallback to legacy docker-compose for older systems cmd = exec.Command("docker-compose", "--version") if err := cmd.Run(); err == nil { info.ComposeAvailable = true } } // Check if Docker is in swarm mode if info.Available { cmd = exec.Command("docker", "info", "--format", "{{.Swarm.LocalNodeState}}") output, err := cmd.Output() if err == nil && strings.TrimSpace(string(output)) == "active" { info.SwarmMode = true } } return info, nil } // RepositoryConfig holds repository configuration for setup type RepositoryConfig struct { Provider string `json:"provider"` BaseURL string `json:"baseURL,omitempty"` AccessToken string `json:"accessToken"` Owner string `json:"owner"` Repository string `json:"repository"` } // ValidateRepositoryConfig validates repository configuration func (s *SetupManager) ValidateRepositoryConfig(repoConfig *RepositoryConfig) error { if repoConfig.Provider == "" { return fmt.Errorf("provider is required") } if repoConfig.AccessToken == "" { return fmt.Errorf("access token is required") } if repoConfig.Owner == "" { return fmt.Errorf("owner is required") } if repoConfig.Repository == "" { return fmt.Errorf("repository is required") } // Validate provider-specific requirements switch strings.ToLower(repoConfig.Provider) { case "gitea": if repoConfig.BaseURL == "" { return fmt.Errorf("base_url is required for Gitea") } case "github": // GitHub uses default URL if repoConfig.BaseURL == "" { repoConfig.BaseURL = "https://api.github.com" } default: return fmt.Errorf("unsupported provider: %s", repoConfig.Provider) } // Test connection to repository return s.testRepositoryConnection(repoConfig) } // testRepositoryConnection tests connection to the repository func (s *SetupManager) testRepositoryConnection(repoConfig *RepositoryConfig) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() config := &repository.Config{ Provider: repoConfig.Provider, BaseURL: repoConfig.BaseURL, AccessToken: repoConfig.AccessToken, Owner: repoConfig.Owner, Repository: repoConfig.Repository, TaskLabel: "bzzz-task", InProgressLabel: "in-progress", CompletedLabel: "completed", BaseBranch: "main", BranchPrefix: "bzzz/", } provider, err := s.factory.CreateProvider(ctx, config) if err != nil { return fmt.Errorf("failed to create provider: %w", err) } // Try to list tasks to test connection _, err = provider.ListAvailableTasks() if err != nil { return fmt.Errorf("failed to connect to repository: %w", err) } return nil } // SetupConfig holds the complete setup configuration type SetupConfig struct { AgentID string `json:"agent_id"` Capabilities []string `json:"capabilities"` Models []string `json:"models"` Repository *RepositoryConfig `json:"repository"` Network map[string]interface{} `json:"network"` Storage map[string]interface{} `json:"storage"` Security map[string]interface{} `json:"security"` } // SaveConfiguration saves the setup configuration to file func (s *SetupManager) SaveConfiguration(setupConfig *SetupConfig) error { // Create configuration directory if it doesn't exist configDir := filepath.Dir(s.configPath) if err := os.MkdirAll(configDir, 0755); err != nil { return fmt.Errorf("failed to create config directory: %w", err) } // Load default configuration cfg, err := config.LoadConfig("") if err != nil { // If loading fails, we'll create a minimal config cfg = &config.Config{} } // Apply setup configuration if setupConfig.AgentID != "" { cfg.Agent.ID = setupConfig.AgentID } if len(setupConfig.Capabilities) > 0 { cfg.Agent.Capabilities = setupConfig.Capabilities } if len(setupConfig.Models) > 0 { cfg.Agent.Models = setupConfig.Models } // Configure repository if provided if setupConfig.Repository != nil { // This would integrate with the existing repository configuration // For now, we'll store it in a way that can be used by the main application } // Save configuration to file if err := config.SaveConfig(cfg, s.configPath); err != nil { return fmt.Errorf("failed to save configuration: %w", err) } return nil } // GetSupportedProviders returns list of supported repository providers func (s *SetupManager) GetSupportedProviders() []string { return s.factory.SupportedProviders() } // Machine represents a discovered network machine type Machine struct { IP string `json:"ip"` Hostname string `json:"hostname"` OS string `json:"os,omitempty"` OSVersion string `json:"os_version,omitempty"` SystemInfo map[string]interface{} `json:"system_info,omitempty"` } // DiscoverNetworkMachines scans the network subnet for available machines func (s *SetupManager) DiscoverNetworkMachines(subnet string, sshKey string) ([]Machine, error) { var machines []Machine // Parse CIDR subnet _, ipNet, err := net.ParseCIDR(subnet) if err != nil { return nil, fmt.Errorf("invalid subnet: %w", err) } // Create a semaphore to limit concurrent goroutines (max 50 for faster scanning) sem := make(chan struct{}, 50) var wg sync.WaitGroup var mu sync.Mutex // Context for early termination when we have enough results ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Generate list of IPs to scan var ips []string // Start from the network address ip := make(net.IP, len(ipNet.IP)) copy(ip, ipNet.IP.Mask(ipNet.Mask)) for ipNet.Contains(ip) { // Skip network and broadcast addresses ipStr := ip.String() if !ip.Equal(ipNet.IP) && !isBroadcast(ip, ipNet) { ips = append(ips, ipStr) } // Increment IP inc(ip) // Limit total IPs to scan (avoid scanning entire /16) if len(ips) >= 254 { break } } // Scan IPs with limited concurrency for _, targetIP := range ips { // Check if context is cancelled (early termination) select { case <-ctx.Done(): break default: } wg.Add(1) go func(ip string) { defer wg.Done() // Check context again select { case <-ctx.Done(): return default: } // Acquire semaphore sem <- struct{}{} defer func() { <-sem }() // Quick ping test with shorter timeout pingCtx, pingCancel := context.WithTimeout(ctx, 500*time.Millisecond) defer pingCancel() cmd := exec.CommandContext(pingCtx, "ping", "-c", "1", "-W", "300", ip) if err := cmd.Run(); err == nil { // Machine is pingable, try to get more info machine := Machine{ IP: ip, Hostname: s.getHostname(ip), } mu.Lock() machines = append(machines, machine) // Stop early if we have enough machines if len(machines) >= 20 { mu.Unlock() cancel() // Signal other goroutines to stop return } mu.Unlock() } }(targetIP) } wg.Wait() return machines, nil } // inc increments IP address func inc(ip net.IP) { for j := len(ip) - 1; j >= 0; j-- { ip[j]++ if ip[j] > 0 { break } } } // isBroadcast checks if IP is the broadcast address for the network func isBroadcast(ip net.IP, ipNet *net.IPNet) bool { if ip == nil || ipNet == nil { return false } // Calculate broadcast address broadcast := make(net.IP, len(ip)) copy(broadcast, ipNet.IP.Mask(ipNet.Mask)) // Set all host bits to 1 for i := range broadcast { broadcast[i] |= ^ipNet.Mask[i] } return ip.Equal(broadcast) } // getHostname attempts to resolve hostname for IP func (s *SetupManager) getHostname(ip string) string { names, err := net.LookupAddr(ip) if err != nil || len(names) == 0 { return "Unknown" } return strings.TrimSuffix(names[0], ".") } // SSHTestResult represents the result of SSH connection test type SSHTestResult struct { Success bool `json:"success"` Error string `json:"error,omitempty"` OS string `json:"os,omitempty"` OSVersion string `json:"os_version,omitempty"` SystemInfo map[string]interface{} `json:"system_info,omitempty"` } // TestSSHConnection tests SSH connectivity and gathers system info func (s *SetupManager) TestSSHConnection(ip string, privateKey string, username string, password string, port int) (*SSHTestResult, error) { result := &SSHTestResult{} // SECURITY: Validate all input parameters with zero-trust approach if err := s.validator.ValidateSSHConnectionRequest(ip, username, password, privateKey, port); err != nil { result.Success = false result.Error = fmt.Sprintf("Security validation failed: %s", err.Error()) return result, nil } // Set default port if not provided if port == 0 { port = 22 } // SSH client config with flexible authentication var authMethods []ssh.AuthMethod var authErrors []string if privateKey != "" { // Try private key authentication first if signer, err := ssh.ParsePrivateKey([]byte(privateKey)); err == nil { authMethods = append(authMethods, ssh.PublicKeys(signer)) } else { authErrors = append(authErrors, fmt.Sprintf("Invalid SSH private key: %v", err)) } } if password != "" { // Add password authentication authMethods = append(authMethods, ssh.Password(password)) } if len(authMethods) == 0 { result.Success = false result.Error = fmt.Sprintf("No valid authentication methods available. Errors: %v", strings.Join(authErrors, "; ")) return result, nil } config := &ssh.ClientConfig{ User: username, Auth: authMethods, HostKeyCallback: ssh.InsecureIgnoreHostKey(), // For setup phase Timeout: 10 * time.Second, } // Connect to SSH with detailed error reporting address := fmt.Sprintf("%s:%d", ip, port) client, err := ssh.Dial("tcp", address, config) if err != nil { result.Success = false // Provide specific error messages based on error type if strings.Contains(err.Error(), "connection refused") { result.Error = fmt.Sprintf("SSH connection refused to %s:%d - SSH service may not be running or port blocked", ip, port) } else if strings.Contains(err.Error(), "permission denied") { result.Error = fmt.Sprintf("SSH authentication failed for user '%s' on %s:%d - check username/password/key", username, ip, port) } else if strings.Contains(err.Error(), "no route to host") { result.Error = fmt.Sprintf("No network route to host %s - check IP address and network connectivity", ip) } else if strings.Contains(err.Error(), "timeout") { result.Error = fmt.Sprintf("SSH connection timeout to %s:%d - host may be unreachable or SSH service slow", ip, port) } else { result.Error = fmt.Sprintf("SSH connection failed to %s@%s:%d - %v", username, ip, port, err) } return result, nil } defer client.Close() result.Success = true // Gather system information session, err := client.NewSession() if err == nil { defer session.Close() // Get OS info if output, err := session.Output("uname -s"); err == nil { result.OS = strings.TrimSpace(string(output)) } // Get OS version session, _ = client.NewSession() if output, err := session.Output("lsb_release -d 2>/dev/null || cat /etc/os-release | head -1"); err == nil { result.OSVersion = strings.TrimSpace(string(output)) } session.Close() // Get basic system info session, _ = client.NewSession() if output, err := session.Output("nproc && free -m | grep Mem | awk '{print $2}' && df -h / | tail -1 | awk '{print $4}'"); err == nil { lines := strings.Split(strings.TrimSpace(string(output)), "\n") systemInfo := make(map[string]interface{}) if len(lines) >= 3 { if cpu, err := strconv.Atoi(lines[0]); err == nil { systemInfo["cpu"] = cpu } if mem, err := strconv.Atoi(lines[1]); err == nil { systemInfo["memory"] = mem / 1024 // Convert MB to GB } systemInfo["disk"] = lines[2] } result.SystemInfo = systemInfo } session.Close() } return result, nil } // DeploymentResult represents the result of service deployment type DeploymentResult struct { Success bool `json:"success"` Error string `json:"error,omitempty"` Steps []DeploymentStep `json:"steps,omitempty"` RollbackLog []string `json:"rollback_log,omitempty"` SystemInfo *DeploymentSystemInfo `json:"system_info,omitempty"` } // DeploymentStep represents a single deployment step with detailed status type DeploymentStep struct { Name string `json:"name"` Status string `json:"status"` // "pending", "running", "success", "failed" Command string `json:"command,omitempty"` Output string `json:"output,omitempty"` Error string `json:"error,omitempty"` Duration string `json:"duration,omitempty"` Verified bool `json:"verified"` } // DeployServiceToMachine deploys BZZZ service to a remote machine with full verification func (s *SetupManager) DeployServiceToMachine(ip string, privateKey string, username string, password string, port int, config interface{}) (*DeploymentResult, error) { result := &DeploymentResult{ Steps: []DeploymentStep{}, RollbackLog: []string{}, } // SECURITY: Validate all input parameters with zero-trust approach if err := s.validator.ValidateSSHConnectionRequest(ip, username, password, privateKey, port); err != nil { result.Success = false result.Error = fmt.Sprintf("Security validation failed: %s", err.Error()) return result, nil } // Set default port if not provided if port == 0 { port = 22 } // SSH client config with flexible authentication var authMethods []ssh.AuthMethod var authErrors []string if privateKey != "" { // Try private key authentication first if signer, err := ssh.ParsePrivateKey([]byte(privateKey)); err == nil { authMethods = append(authMethods, ssh.PublicKeys(signer)) } else { authErrors = append(authErrors, fmt.Sprintf("Invalid SSH private key: %v", err)) } } if password != "" { // Add password authentication authMethods = append(authMethods, ssh.Password(password)) } if len(authMethods) == 0 { result.Success = false result.Error = fmt.Sprintf("No valid authentication methods available. Errors: %v", strings.Join(authErrors, "; ")) return result, nil } sshConfig := &ssh.ClientConfig{ User: username, Auth: authMethods, HostKeyCallback: ssh.InsecureIgnoreHostKey(), Timeout: 30 * time.Second, } // Connect to SSH with detailed error reporting address := fmt.Sprintf("%s:%d", ip, port) client, err := ssh.Dial("tcp", address, sshConfig) if err != nil { result.Success = false // Provide specific error messages based on error type if strings.Contains(err.Error(), "connection refused") { result.Error = fmt.Sprintf("SSH connection refused to %s:%d - SSH service may not be running or port blocked", ip, port) } else if strings.Contains(err.Error(), "permission denied") { result.Error = fmt.Sprintf("SSH authentication failed for user '%s' on %s:%d - check username/password/key", username, ip, port) } else if strings.Contains(err.Error(), "no route to host") { result.Error = fmt.Sprintf("No network route to host %s - check IP address and network connectivity", ip) } else if strings.Contains(err.Error(), "timeout") { result.Error = fmt.Sprintf("SSH connection timeout to %s:%d - host may be unreachable or SSH service slow", ip, port) } else { result.Error = fmt.Sprintf("SSH connection failed to %s@%s:%d - %v", username, ip, port, err) } return result, nil } defer client.Close() s.addStep(result, "SSH Connection", "success", "", "SSH connection established successfully", "", true) // Execute deployment steps with verification steps := []func(*ssh.Client, interface{}, string, *DeploymentResult) error{ s.verifiedPreDeploymentCheck, s.verifiedStopExistingServices, s.verifiedCopyBinary, s.verifiedDeployConfiguration, s.verifiedConfigureFirewall, s.verifiedCreateSystemdService, s.verifiedStartService, s.verifiedPostDeploymentTest, } for _, step := range steps { if err := step(client, config, password, result); err != nil { result.Success = false result.Error = err.Error() s.performRollbackWithPassword(client, password, result) return result, nil } } result.Success = true return result, nil } // addStep adds a deployment step to the result with timing information func (s *SetupManager) addStep(result *DeploymentResult, name, status, command, output, error string, verified bool) { step := DeploymentStep{ Name: name, Status: status, Command: command, Output: output, Error: error, Verified: verified, Duration: "", // Will be filled by the calling function if needed } result.Steps = append(result.Steps, step) } // executeSSHCommand executes a command via SSH and returns output, error func (s *SetupManager) executeSSHCommand(client *ssh.Client, command string) (string, error) { session, err := client.NewSession() if err != nil { return "", fmt.Errorf("failed to create SSH session: %w", err) } defer session.Close() var stdout, stderr strings.Builder session.Stdout = &stdout session.Stderr = &stderr err = session.Run(command) output := stdout.String() if stderr.Len() > 0 { output += "\n[STDERR]: " + stderr.String() } return output, err } // executeSudoCommand executes a command with sudo using the provided password, or tries passwordless sudo if no password func (s *SetupManager) executeSudoCommand(client *ssh.Client, password string, command string) (string, error) { // SECURITY: Sanitize command to prevent injection safeCommand := s.validator.SanitizeForCommand(command) if safeCommand != command { return "", fmt.Errorf("command contained unsafe characters and was sanitized: original='%s', safe='%s'", command, safeCommand) } if password != "" { // SECURITY: Use here-document to avoid password exposure in process list // This keeps the password out of command line arguments and process lists escapedPassword := strings.ReplaceAll(password, "'", "'\"'\"'") secureCommand := fmt.Sprintf(`sudo -S %s <<'BZZZ_EOF' %s BZZZ_EOF`, safeCommand, escapedPassword) return s.executeSSHCommand(client, secureCommand) } else { // Try passwordless sudo sudoCommand := fmt.Sprintf("sudo -n %s", safeCommand) return s.executeSSHCommand(client, sudoCommand) } } // DeploymentSystemInfo holds information about the target system for deployment type DeploymentSystemInfo struct { OS string `json:"os"` // linux, darwin, freebsd, etc. Distro string `json:"distro"` // ubuntu, centos, debian, etc. ServiceMgr string `json:"service_mgr"` // systemd, sysv, openrc, launchd Architecture string `json:"architecture"` // x86_64, arm64, etc. BinaryPath string `json:"binary_path"` // Where to install binary ServicePath string `json:"service_path"` // Where to install service file } // detectSystemInfo detects target system information func (s *SetupManager) detectSystemInfo(client *ssh.Client) (*DeploymentSystemInfo, error) { info := &DeploymentSystemInfo{} // Detect OS osOutput, err := s.executeSSHCommand(client, "uname -s") if err != nil { return nil, fmt.Errorf("failed to detect OS: %v", err) } info.OS = strings.ToLower(strings.TrimSpace(osOutput)) // Detect architecture archOutput, err := s.executeSSHCommand(client, "uname -m") if err != nil { return nil, fmt.Errorf("failed to detect architecture: %v", err) } info.Architecture = strings.TrimSpace(archOutput) // Detect distribution (Linux only) if info.OS == "linux" { if distroOutput, err := s.executeSSHCommand(client, "cat /etc/os-release 2>/dev/null | grep '^ID=' | cut -d= -f2 | tr -d '\"' || echo 'unknown'"); err == nil { info.Distro = strings.TrimSpace(distroOutput) } } // Detect service manager and set paths if err := s.detectServiceManager(client, info); err != nil { return nil, fmt.Errorf("failed to detect service manager: %v", err) } return info, nil } // detectServiceManager detects the service manager and sets appropriate paths func (s *SetupManager) detectServiceManager(client *ssh.Client, info *DeploymentSystemInfo) error { switch info.OS { case "linux": // Check for systemd if _, err := s.executeSSHCommand(client, "which systemctl"); err == nil { if pidOutput, err := s.executeSSHCommand(client, "ps -p 1 -o comm="); err == nil && strings.Contains(pidOutput, "systemd") { info.ServiceMgr = "systemd" info.ServicePath = "/etc/systemd/system" info.BinaryPath = "/usr/local/bin" return nil } } // Check for OpenRC if _, err := s.executeSSHCommand(client, "which rc-service"); err == nil { info.ServiceMgr = "openrc" info.ServicePath = "/etc/init.d" info.BinaryPath = "/usr/local/bin" return nil } // Check for SysV init if _, err := s.executeSSHCommand(client, "ls /etc/init.d/ 2>/dev/null"); err == nil { info.ServiceMgr = "sysv" info.ServicePath = "/etc/init.d" info.BinaryPath = "/usr/local/bin" return nil } return fmt.Errorf("unsupported service manager on Linux") case "darwin": info.ServiceMgr = "launchd" info.ServicePath = "/Library/LaunchDaemons" info.BinaryPath = "/usr/local/bin" return nil case "freebsd": info.ServiceMgr = "rc" info.ServicePath = "/usr/local/etc/rc.d" info.BinaryPath = "/usr/local/bin" return nil default: return fmt.Errorf("unsupported operating system: %s", info.OS) } } // verifiedPreDeploymentCheck checks system requirements and existing installations func (s *SetupManager) verifiedPreDeploymentCheck(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Pre-deployment Check" s.addStep(result, stepName, "running", "", "", "", false) // Detect system information sysInfo, err := s.detectSystemInfo(client) if err != nil { s.updateLastStep(result, "failed", "system detection", "", fmt.Sprintf("System detection failed: %v", err), false) return fmt.Errorf("system detection failed: %v", err) } // Store system info for other steps to use result.SystemInfo = sysInfo // Check for existing BZZZ processes (informational only - cleanup step will handle) output, err := s.executeSSHCommand(client, "ps aux | grep bzzz | grep -v grep || echo 'No BZZZ processes found'") if err != nil { s.updateLastStep(result, "failed", "process check", output, fmt.Sprintf("Failed to check processes: %v", err), false) return fmt.Errorf("pre-deployment check failed: %v", err) } // Log existing processes but don't fail - cleanup step will handle this var processStatus string if !strings.Contains(output, "No BZZZ processes found") { processStatus = "Existing BZZZ processes detected (will be stopped in cleanup step)" } else { processStatus = "No existing BZZZ processes detected" } // Check for existing systemd services output2, _ := s.executeSSHCommand(client, "systemctl status bzzz 2>/dev/null || echo 'No BZZZ service'") // Check system requirements output3, _ := s.executeSSHCommand(client, "uname -a && free -m && df -h /tmp") combinedOutput := fmt.Sprintf("Process status: %s\n\nProcess details:\n%s\n\nService check:\n%s\n\nSystem info:\n%s", processStatus, output, output2, output3) s.updateLastStep(result, "success", "", combinedOutput, "", true) return nil } // verifiedStopExistingServices stops any existing BZZZ services func (s *SetupManager) verifiedStopExistingServices(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Stop & Remove Existing Services" s.addStep(result, stepName, "running", "", "", "", false) // Stop systemd service if exists cmd1 := "systemctl stop bzzz 2>/dev/null || echo 'No systemd service to stop'" output1, _ := s.executeSudoCommand(client, password, cmd1) // Disable systemd service if exists - separate command for better error tracking cmd2a := "systemctl disable bzzz 2>/dev/null || echo 'No systemd service to disable'" output2a, _ := s.executeSudoCommand(client, password, cmd2a) // Remove service files cmd2b := "rm -f /etc/systemd/system/bzzz.service ~/.config/systemd/user/bzzz.service 2>/dev/null || echo 'No service file to remove'" output2b, _ := s.executeSudoCommand(client, password, cmd2b) // Kill any remaining processes cmd3 := "pkill -f bzzz || echo 'No processes to kill'" output3, _ := s.executeSSHCommand(client, cmd3) // Remove old binaries from standard locations cmd4 := "rm -f /usr/local/bin/bzzz ~/bin/bzzz ~/bzzz 2>/dev/null || echo 'No old binaries to remove'" output4, _ := s.executeSudoCommand(client, password, cmd4) // Reload systemd after changes cmd5 := "systemctl daemon-reload 2>/dev/null || echo 'Systemd reload completed'" output5, _ := s.executeSudoCommand(client, password, cmd5) // Verify no processes remain output6, err := s.executeSSHCommand(client, "ps aux | grep bzzz | grep -v grep || echo 'All BZZZ processes stopped'") if err != nil { combinedOutput := fmt.Sprintf("Stop service:\n%s\n\nDisable service:\n%s\n\nRemove service files:\n%s\n\nKill processes:\n%s\n\nRemove binaries:\n%s\n\nReload systemd:\n%s\n\nVerification:\n%s", output1, output2a, output2b, output3, output4, output5, output6) s.updateLastStep(result, "failed", "cleanup verification", combinedOutput, fmt.Sprintf("Failed verification: %v", err), false) return fmt.Errorf("failed to verify process cleanup: %v", err) } if !strings.Contains(output6, "All BZZZ processes stopped") { combinedOutput := fmt.Sprintf("Stop service:\n%s\n\nDisable service:\n%s\n\nRemove service files:\n%s\n\nKill processes:\n%s\n\nRemove binaries:\n%s\n\nReload systemd:\n%s\n\nVerification:\n%s", output1, output2a, output2b, output3, output4, output5, output6) s.updateLastStep(result, "failed", "process verification", combinedOutput, "BZZZ processes still running after cleanup", false) return fmt.Errorf("failed to stop all BZZZ processes") } combinedOutput := fmt.Sprintf("Stop service:\n%s\n\nDisable service:\n%s\n\nRemove service files:\n%s\n\nKill processes:\n%s\n\nRemove binaries:\n%s\n\nReload systemd:\n%s\n\nVerification:\n%s", output1, output2a, output2b, output3, output4, output5, output6) s.updateLastStep(result, "success", "stop + cleanup + verify", combinedOutput, "", true) return nil } // updateLastStep updates the last step in the result func (s *SetupManager) updateLastStep(result *DeploymentResult, status, command, output, error string, verified bool) { if len(result.Steps) > 0 { lastStep := &result.Steps[len(result.Steps)-1] lastStep.Status = status if command != "" { lastStep.Command = command } if output != "" { lastStep.Output = output } if error != "" { lastStep.Error = error } lastStep.Verified = verified } } // performRollbackWithPassword attempts to undo changes made during failed deployment using password func (s *SetupManager) performRollbackWithPassword(client *ssh.Client, password string, result *DeploymentResult) { result.RollbackLog = append(result.RollbackLog, "Starting rollback procedure...") // Stop any services we might have started if output, err := s.executeSudoCommand(client, password, "systemctl stop bzzz 2>/dev/null || echo 'No service to stop'"); err == nil { result.RollbackLog = append(result.RollbackLog, "Stopped service: "+output) } // Remove systemd service if output, err := s.executeSudoCommand(client, password, "systemctl disable bzzz 2>/dev/null; rm -f /etc/systemd/system/bzzz.service 2>/dev/null || echo 'No service file to remove'"); err == nil { result.RollbackLog = append(result.RollbackLog, "Removed service: "+output) } // Remove binary if output, err := s.executeSudoCommand(client, password, "rm -f /usr/local/bin/bzzz 2>/dev/null || echo 'No binary to remove'"); err == nil { result.RollbackLog = append(result.RollbackLog, "Removed binary: "+output) } // Reload systemd if output, err := s.executeSudoCommand(client, password, "systemctl daemon-reload"); err == nil { result.RollbackLog = append(result.RollbackLog, "Reloaded systemd: "+output) } } // performRollback attempts to rollback any changes made during failed deployment func (s *SetupManager) performRollback(client *ssh.Client, result *DeploymentResult) { result.RollbackLog = append(result.RollbackLog, "Starting rollback procedure...") // Stop any services we might have started if output, err := s.executeSSHCommand(client, "sudo -n systemctl stop bzzz 2>/dev/null || echo 'No service to stop'"); err == nil { result.RollbackLog = append(result.RollbackLog, "Stopped service: "+output) } // Remove binaries we might have copied if output, err := s.executeSSHCommand(client, "rm -f ~/bzzz /usr/local/bin/bzzz 2>/dev/null || echo 'No binaries to remove'"); err == nil { result.RollbackLog = append(result.RollbackLog, "Removed binaries: "+output) } result.RollbackLog = append(result.RollbackLog, "Rollback completed") } // verifiedCopyBinary copies BZZZ binary and verifies installation func (s *SetupManager) verifiedCopyBinary(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Copy Binary" s.addStep(result, stepName, "running", "", "", "", false) // Copy binary using existing function but with verification if err := s.copyBinaryToMachineWithPassword(client, password); err != nil { s.updateLastStep(result, "failed", "scp binary", "", err.Error(), false) return fmt.Errorf("binary copy failed: %v", err) } // Verify binary was copied and is executable checkCmd := "ls -la /usr/local/bin/bzzz ~/bin/bzzz 2>/dev/null || echo 'Binary not found in expected locations'" output, err := s.executeSSHCommand(client, checkCmd) if err != nil { s.updateLastStep(result, "failed", checkCmd, output, fmt.Sprintf("Verification failed: %v", err), false) return fmt.Errorf("binary verification failed: %v", err) } // Verify binary can execute (note: BZZZ doesn't have --version flag, use --help) versionCmd := "timeout 3s /usr/local/bin/bzzz --help 2>&1 | head -n1 || timeout 3s ~/bin/bzzz --help 2>&1 | head -n1 || echo 'Binary not executable'" versionOutput, _ := s.executeSSHCommand(client, versionCmd) combinedOutput := fmt.Sprintf("File check:\n%s\n\nBinary test:\n%s", output, versionOutput) if strings.Contains(output, "Binary not found") { s.updateLastStep(result, "failed", checkCmd, combinedOutput, "Binary not found in expected locations", false) return fmt.Errorf("binary installation verification failed") } s.updateLastStep(result, "success", "scp + verify", combinedOutput, "", true) return nil } // verifiedDeployConfiguration deploys configuration and verifies correctness func (s *SetupManager) verifiedDeployConfiguration(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Deploy Configuration" s.addStep(result, stepName, "running", "", "", "", false) // Generate and deploy configuration using existing function if err := s.generateAndDeployConfig(client, "remote-host", config); err != nil { s.updateLastStep(result, "failed", "deploy config", "", err.Error(), false) return fmt.Errorf("configuration deployment failed: %v", err) } // Verify configuration file was created and is valid YAML verifyCmd := "ls -la ~/.bzzz/config.yaml && echo '--- Config Preview ---' && head -20 ~/.bzzz/config.yaml" output, err := s.executeSSHCommand(client, verifyCmd) if err != nil { s.updateLastStep(result, "failed", verifyCmd, output, fmt.Sprintf("Config verification failed: %v", err), false) return fmt.Errorf("configuration verification failed: %v", err) } // Check if config contains expected sections for complex config structure if !strings.Contains(output, "agent:") || !strings.Contains(output, "whoosh_api:") || !strings.Contains(output, "ai:") { s.updateLastStep(result, "failed", verifyCmd, output, "Configuration missing required sections", false) return fmt.Errorf("configuration incomplete - missing required sections") } s.updateLastStep(result, "success", "deploy + verify config", output, "", true) return nil } // verifiedConfigureFirewall configures firewall and verifies rules func (s *SetupManager) verifiedConfigureFirewall(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Configure Firewall" s.addStep(result, stepName, "running", "", "", "", false) // Configure firewall using existing function if err := s.configureFirewall(client, config); err != nil { s.updateLastStep(result, "failed", "configure firewall", "", err.Error(), false) return fmt.Errorf("firewall configuration failed: %v", err) } // Verify firewall rules (this is informational, not critical) verifyCmd := "ufw status 2>/dev/null || firewall-cmd --list-ports 2>/dev/null || echo 'Firewall status unavailable'" output, _ := s.executeSudoCommand(client, password, verifyCmd) s.updateLastStep(result, "success", "configure + verify firewall", output, "", true) return nil } // verifiedCreateSystemdService creates systemd service and verifies configuration func (s *SetupManager) verifiedCreateSystemdService(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Create SystemD Service" s.addStep(result, stepName, "running", "", "", "", false) // Create systemd service using password-based sudo if err := s.createSystemdServiceWithPassword(client, config, password); err != nil { s.updateLastStep(result, "failed", "create service", "", err.Error(), false) return fmt.Errorf("systemd service creation failed: %v", err) } // Verify service file was created and contains correct paths verifyCmd := "systemctl cat bzzz" output, err := s.executeSudoCommand(client, password, verifyCmd) if err != nil { // Try to check if the service file exists another way checkCmd := "ls -la /etc/systemd/system/bzzz.service" checkOutput, checkErr := s.executeSudoCommand(client, password, checkCmd) if checkErr != nil { s.updateLastStep(result, "failed", verifyCmd, output, fmt.Sprintf("Service verification failed: %v. Service file check also failed: %v", err, checkErr), false) return fmt.Errorf("systemd service verification failed: %v", err) } s.updateLastStep(result, "warning", verifyCmd, checkOutput, "Service file exists but systemctl cat failed, continuing", false) } // Verify service can be enabled enableCmd := "systemctl enable bzzz" enableOutput, enableErr := s.executeSudoCommand(client, password, enableCmd) if enableErr != nil { combinedOutput := fmt.Sprintf("Service file:\n%s\n\nEnable attempt:\n%s", output, enableOutput) s.updateLastStep(result, "failed", enableCmd, combinedOutput, fmt.Sprintf("Failed to enable service: %v", enableErr), false) return fmt.Errorf("failed to enable systemd service: %v", enableErr) } combinedOutput := fmt.Sprintf("Service file:\n%s\n\nService enabled:\n%s", output, enableOutput) s.updateLastStep(result, "success", "create + enable service", combinedOutput, "", true) return nil } // verifiedStartService starts the service and verifies it's running properly func (s *SetupManager) verifiedStartService(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Start Service" s.addStep(result, stepName, "running", "", "", "", false) // Check if auto-start is enabled configMap, ok := config.(map[string]interface{}) if !ok || configMap["autoStart"] != true { s.updateLastStep(result, "success", "", "Auto-start disabled, skipping service start", "", true) return nil } // Pre-flight checks before starting service s.addStep(result, "Pre-Start Checks", "running", "", "", "", false) // Check if config file exists and is readable by the service user configCheck := "ls -la /home/*/bzzz/config.yaml 2>/dev/null || echo 'Config file not found'" configOutput, _ := s.executeSSHCommand(client, configCheck) // Check if binary is executable binCheck := "ls -la /usr/local/bin/bzzz" binOutput, _ := s.executeSudoCommand(client, password, binCheck) preflightInfo := fmt.Sprintf("Binary check:\n%s\n\nConfig check:\n%s", binOutput, configOutput) s.updateLastStep(result, "success", "pre-flight", preflightInfo, "Pre-start checks completed", false) // Start the service startCmd := "systemctl start bzzz" startOutput, err := s.executeSudoCommand(client, password, startCmd) if err != nil { // Get detailed error information statusCmd := "systemctl status bzzz" statusOutput, _ := s.executeSudoCommand(client, password, statusCmd) logsCmd := "journalctl -u bzzz --no-pager -n 20" logsOutput, _ := s.executeSudoCommand(client, password, logsCmd) // Combine all error information detailedError := fmt.Sprintf("Start command output:\n%s\n\nService status:\n%s\n\nRecent logs:\n%s", startOutput, statusOutput, logsOutput) s.updateLastStep(result, "failed", startCmd, detailedError, fmt.Sprintf("Failed to start service: %v", err), false) return fmt.Errorf("failed to start systemd service: %v", err) } // Wait for service to fully initialize (BZZZ needs time to start all subsystems) time.Sleep(8 * time.Second) // Verify service is running statusCmd := "systemctl status bzzz" statusOutput, _ := s.executeSSHCommand(client, statusCmd) // Check if service is active if !strings.Contains(statusOutput, "active (running)") { // Get detailed logs to understand why service failed logsCmd := "journalctl -u bzzz --no-pager -n 20" logsOutput, _ := s.executeSudoCommand(client, password, logsCmd) // Check if config file exists and is readable configCheckCmd := "ls -la ~/.bzzz/config.yaml && head -5 ~/.bzzz/config.yaml" configCheckOutput, _ := s.executeSSHCommand(client, configCheckCmd) combinedOutput := fmt.Sprintf("Start attempt:\n%s\n\nStatus check:\n%s\n\nRecent logs:\n%s\n\nConfig check:\n%s", startOutput, statusOutput, logsOutput, configCheckOutput) s.updateLastStep(result, "failed", startCmd, combinedOutput, "Service failed to reach running state", false) return fmt.Errorf("service is not running after start attempt") } combinedOutput := fmt.Sprintf("Service started:\n%s\n\nStatus verification:\n%s", startOutput, statusOutput) s.updateLastStep(result, "success", startCmd+" + verify", combinedOutput, "", true) return nil } // verifiedPostDeploymentTest performs final verification that deployment is functional func (s *SetupManager) verifiedPostDeploymentTest(client *ssh.Client, config interface{}, password string, result *DeploymentResult) error { stepName := "Post-deployment Test" s.addStep(result, stepName, "running", "", "", "", false) // Test 1: Verify binary is executable // Note: BZZZ binary doesn't have --version flag, so just check if it's executable and can start help versionCmd := "if pgrep -f bzzz >/dev/null; then echo 'BZZZ process running'; else timeout 3s /usr/local/bin/bzzz --help 2>&1 | head -n1 || timeout 3s ~/bin/bzzz --help 2>&1 | head -n1 || echo 'Binary not executable'; fi" versionOutput, _ := s.executeSSHCommand(client, versionCmd) // Test 2: Verify service status serviceCmd := "systemctl status bzzz --no-pager" serviceOutput, _ := s.executeSSHCommand(client, serviceCmd) // Test 3: Wait for API to be ready, then check if setup API is responding // Poll for API readiness with timeout (up to 15 seconds) var apiOutput string apiReady := false for i := 0; i < 15; i++ { apiCmd := "curl -s -m 2 http://localhost:8090/api/setup/required 2>/dev/null" output, err := s.executeSSHCommand(client, apiCmd) if err == nil && !strings.Contains(output, "Connection refused") && !strings.Contains(output, "timeout") { apiOutput = fmt.Sprintf("API ready (after %ds): %s", i+1, output) apiReady = true break } if i < 14 { // Don't sleep on the last iteration time.Sleep(1 * time.Second) } } if !apiReady { apiOutput = "API not responding after 15s timeout" } // Test 4: Verify configuration is readable configCmd := "test -r ~/.bzzz/config.yaml && echo 'Config readable' || echo 'Config not readable'" configOutput, _ := s.executeSSHCommand(client, configCmd) combinedOutput := fmt.Sprintf("Binary test:\n%s\n\nService test:\n%s\n\nAPI test:\n%s\n\nConfig test:\n%s", versionOutput, serviceOutput, apiOutput, configOutput) // Determine if tests passed and provide detailed failure information // Binary test passes if BZZZ is running OR if help command succeeded binaryFailed := strings.Contains(versionOutput, "Binary not executable") && !strings.Contains(versionOutput, "BZZZ process running") configFailed := strings.Contains(configOutput, "Config not readable") if binaryFailed || configFailed { var failures []string if binaryFailed { failures = append(failures, "Binary not executable or accessible") } if configFailed { failures = append(failures, "Config file not readable") } failureMsg := fmt.Sprintf("Tests failed: %s", strings.Join(failures, ", ")) s.updateLastStep(result, "failed", "post-deployment tests", combinedOutput, failureMsg, false) return fmt.Errorf("post-deployment verification failed: %s", failureMsg) } s.updateLastStep(result, "success", "comprehensive verification", combinedOutput, "", true) return nil } // copyBinaryToMachineWithPassword copies the BZZZ binary to remote machine using SCP protocol with sudo password func (s *SetupManager) copyBinaryToMachineWithPassword(client *ssh.Client, password string) error { // Read current binary binaryPath, err := os.Executable() if err != nil { return err } binaryData, err := os.ReadFile(binaryPath) if err != nil { return err } // SCP protocol implementation // Create SCP session session, err := client.NewSession() if err != nil { return err } defer session.Close() // Set up pipes stdin, err := session.StdinPipe() if err != nil { return err } defer stdin.Close() stdout, err := session.StdoutPipe() if err != nil { return err } // Start SCP receive command on remote host remotePath := "~/bzzz" go func() { defer stdin.Close() // Send SCP header: C \n header := fmt.Sprintf("C0755 %d bzzz\n", len(binaryData)) stdin.Write([]byte(header)) // Wait for acknowledgment response := make([]byte, 1) stdout.Read(response) if response[0] != 0 { return } // Send file content stdin.Write(binaryData) // Send final null byte stdin.Write([]byte{0}) }() // Execute SCP receive command cmd := fmt.Sprintf("scp -t %s", remotePath) if err := session.Run(cmd); err != nil { return fmt.Errorf("failed to copy binary via SCP: %w", err) } // Make the binary executable session, err = client.NewSession() if err != nil { return err } defer session.Close() if err := session.Run("chmod +x ~/bzzz"); err != nil { return fmt.Errorf("failed to make binary executable: %w", err) } // Try to move to /usr/local/bin with sudo, fall back to user bin if needed session, err = client.NewSession() if err != nil { return err } defer session.Close() // Try to move to /usr/local/bin with sudo (with or without password), fall back to user bin if needed var sudoCmd string if password == "" { // Try passwordless sudo first sudoCmd = "sudo -n mv ~/bzzz /usr/local/bin/bzzz && sudo -n chmod +x /usr/local/bin/bzzz" } else { // Use password sudo escapedPassword := strings.ReplaceAll(password, "'", "'\"'\"'") sudoCmd = fmt.Sprintf("echo '%s' | sudo -S mv ~/bzzz /usr/local/bin/bzzz && echo '%s' | sudo -S chmod +x /usr/local/bin/bzzz", escapedPassword, escapedPassword) } if err := session.Run(sudoCmd); err != nil { // If sudo fails, create user bin directory and install there session, err = client.NewSession() if err != nil { return err } defer session.Close() // Create ~/bin directory and add to PATH if it doesn't exist if err := session.Run("mkdir -p ~/bin && mv ~/bzzz ~/bin/bzzz && chmod +x ~/bin/bzzz"); err != nil { return fmt.Errorf("failed to install binary to ~/bin: %w", err) } // Add ~/bin to PATH in .bashrc if not already there session, err = client.NewSession() if err != nil { return err } defer session.Close() session.Run("grep -q 'export PATH=\"$HOME/bin:$PATH\"' ~/.bashrc || echo 'export PATH=\"$HOME/bin:$PATH\"' >> ~/.bashrc") } return nil } // copyBinaryToMachine copies the BZZZ binary to remote machine using SCP protocol (passwordless sudo) func (s *SetupManager) copyBinaryToMachine(client *ssh.Client) error { return s.copyBinaryToMachineWithPassword(client, "") } // createSystemdServiceWithPassword creates systemd service file using password sudo func (s *SetupManager) createSystemdServiceWithPassword(client *ssh.Client, config interface{}, password string) error { // Determine the correct binary path session, err := client.NewSession() if err != nil { return err } defer session.Close() var stdout strings.Builder session.Stdout = &stdout // Check where the binary was installed binaryPath := "/usr/local/bin/bzzz" if err := session.Run("test -f /usr/local/bin/bzzz"); err != nil { // If not in /usr/local/bin, it should be in ~/bin session, err = client.NewSession() if err != nil { return err } defer session.Close() session.Stdout = &stdout if err := session.Run("echo $HOME/bin/bzzz"); err == nil { binaryPath = strings.TrimSpace(stdout.String()) } } // Get the actual username for the service session, err = client.NewSession() if err != nil { return err } defer session.Close() var userBuilder strings.Builder session.Stdout = &userBuilder if err := session.Run("whoami"); err != nil { return fmt.Errorf("failed to get username: %w", err) } username := strings.TrimSpace(userBuilder.String()) // Create service file with actual username serviceFile := fmt.Sprintf(`[Unit] Description=BZZZ P2P Task Coordination System Documentation=https://chorus.services/docs/bzzz After=network.target [Service] Type=simple ExecStart=%s --config /home/%s/.bzzz/config.yaml Restart=always RestartSec=10 User=%s Group=%s [Install] WantedBy=multi-user.target `, binaryPath, username, username, username) // Create service file in temp location first, then move with sudo createCmd := fmt.Sprintf("cat > /tmp/bzzz.service << 'EOF'\n%sEOF", serviceFile) if _, err := s.executeSSHCommand(client, createCmd); err != nil { return fmt.Errorf("failed to create temp service file: %w", err) } // Move to systemd directory using password sudo moveCmd := "mv /tmp/bzzz.service /etc/systemd/system/bzzz.service" if _, err := s.executeSudoCommand(client, password, moveCmd); err != nil { return fmt.Errorf("failed to install system service file: %w", err) } // Reload systemd to recognize new service reloadCmd := "systemctl daemon-reload" if _, err := s.executeSudoCommand(client, password, reloadCmd); err != nil { return fmt.Errorf("failed to reload systemd: %w", err) } return nil } // createSystemdService creates systemd service file func (s *SetupManager) createSystemdService(client *ssh.Client, config interface{}) error { // Determine the correct binary path session, err := client.NewSession() if err != nil { return err } defer session.Close() var stdout strings.Builder session.Stdout = &stdout // Check where the binary was installed binaryPath := "/usr/local/bin/bzzz" if err := session.Run("test -f /usr/local/bin/bzzz"); err != nil { // If not in /usr/local/bin, it should be in ~/bin session, err = client.NewSession() if err != nil { return err } defer session.Close() session.Stdout = &stdout if err := session.Run("echo $HOME/bin/bzzz"); err == nil { binaryPath = strings.TrimSpace(stdout.String()) } } // Create service file that works for both system and user services serviceFile := fmt.Sprintf(`[Unit] Description=BZZZ P2P Task Coordination System Documentation=https://chorus.services/docs/bzzz After=network.target [Service] Type=simple ExecStart=%s --config %%h/.bzzz/config.yaml Restart=always RestartSec=10 Environment=HOME=%%h [Install] WantedBy=default.target `, binaryPath) // Create service file using a more robust approach session, err = client.NewSession() if err != nil { return err } defer session.Close() // Create service file in temp location first, then move with sudo cmd := fmt.Sprintf("cat > /tmp/bzzz.service << 'EOF'\n%sEOF", serviceFile) if err := session.Run(cmd); err != nil { return fmt.Errorf("failed to create temp service file: %w", err) } // Try to install as system service first, fall back to user service session, err = client.NewSession() if err != nil { return err } defer session.Close() // Try passwordless sudo for system service if err := session.Run("sudo -n mv /tmp/bzzz.service /etc/systemd/system/bzzz.service"); err != nil { // Sudo failed, create user-level service instead session, err = client.NewSession() if err != nil { return err } defer session.Close() // Create user systemd directory and install service there if err := session.Run("mkdir -p ~/.config/systemd/user && mv /tmp/bzzz.service ~/.config/systemd/user/bzzz.service"); err != nil { return fmt.Errorf("failed to install user service file: %w", err) } // Reload user systemd and enable service session, err = client.NewSession() if err != nil { return err } defer session.Close() if err := session.Run("systemctl --user daemon-reload && systemctl --user enable bzzz"); err != nil { return fmt.Errorf("failed to enable user bzzz service: %w", err) } // Enable lingering so user services start at boot session, err = client.NewSession() if err != nil { return err } defer session.Close() session.Run("sudo -n loginctl enable-linger $(whoami) 2>/dev/null || true") } else { // System service installation succeeded, continue with system setup session, err = client.NewSession() if err != nil { return err } defer session.Close() if err := session.Run("sudo -n useradd -r -s /bin/false bzzz 2>/dev/null || true"); err != nil { return fmt.Errorf("failed to create bzzz user: %w", err) } session, err = client.NewSession() if err != nil { return err } defer session.Close() if err := session.Run("sudo -n mkdir -p /opt/bzzz && sudo -n chown bzzz:bzzz /opt/bzzz"); err != nil { return fmt.Errorf("failed to create bzzz directory: %w", err) } // Reload systemd and enable service session, err = client.NewSession() if err != nil { return err } defer session.Close() if err := session.Run("sudo -n systemctl daemon-reload && sudo -n systemctl enable bzzz"); err != nil { return fmt.Errorf("failed to enable bzzz service: %w", err) } } return nil } // startService starts the BZZZ service (system or user level) func (s *SetupManager) startService(client *ssh.Client) error { session, err := client.NewSession() if err != nil { return err } defer session.Close() // Try system service first, fall back to user service if err := session.Run("sudo -n systemctl start bzzz"); err != nil { // Try user service instead session, err = client.NewSession() if err != nil { return err } defer session.Close() return session.Run("systemctl --user start bzzz") } return nil } // GenerateConfigForMachine generates the YAML configuration for a specific machine (for download/inspection) func (s *SetupManager) GenerateConfigForMachine(machineIP string, config interface{}) (string, error) { // Extract configuration from the setup data configMap, ok := config.(map[string]interface{}) if !ok { return "", fmt.Errorf("invalid configuration format: expected map[string]interface{}, got %T: %+v", config, config) } // Use machine IP to determine hostname (simplified) hostname := strings.ReplaceAll(machineIP, ".", "-") // Extract ports from configuration ports := map[string]interface{}{ "api": 8080, "mcp": 3000, "webui": 8080, "p2p": 7000, } // Override with configured ports if available if portsConfig, exists := configMap["ports"]; exists { if portsMap, ok := portsConfig.(map[string]interface{}); ok { for key, value := range portsMap { ports[key] = value } } } // Extract security configuration securityConfig := map[string]interface{}{ "cluster_secret": "default-secret", } if security, exists := configMap["security"]; exists { if securityMap, ok := security.(map[string]interface{}); ok { if secret, exists := securityMap["clusterSecret"]; exists { securityConfig["cluster_secret"] = secret } } } // Generate YAML configuration that matches the Go struct layout configYAML := fmt.Sprintf(`# BZZZ Configuration for %s whoosh_api: base_url: "https://whoosh.home.deepblack.cloud" timeout: 30s retry_count: 3 agent: id: "%s-agent" capabilities: ["general", "reasoning", "task-coordination"] poll_interval: 30s max_tasks: 3 models: ["phi3", "llama3.1"] specialization: "general_developer" model_selection_webhook: "https://n8n.home.deepblack.cloud/webhook/model-selection" default_reasoning_model: "phi3" sandbox_image: "registry.home.deepblack.cloud/bzzz-sandbox:latest" role: "" system_prompt: "" reports_to: [] expertise: [] deliverables: [] collaboration: preferred_message_types: [] auto_subscribe_to_roles: [] auto_subscribe_to_expertise: [] response_timeout_seconds: 0 max_collaboration_depth: 0 escalation_threshold: 0 custom_topic_subscriptions: [] github: token_file: "" user_agent: "Bzzz-P2P-Agent/1.0" timeout: 30s rate_limit: true assignee: "" p2p: service_tag: "bzzz-peer-discovery" bzzz_topic: "bzzz/coordination/v1" hmmm_topic: "hmmm/meta-discussion/v1" discovery_timeout: 10s escalation_webhook: "https://n8n.home.deepblack.cloud/webhook-test/human-escalation" escalation_keywords: ["stuck", "help", "human", "escalate", "clarification needed", "manual intervention"] conversation_limit: 10 logging: level: "info" format: "text" output: "stdout" structured: false slurp: enabled: true base_url: "" api_key: "" timeout: 30s retry_count: 3 max_concurrent_requests: 10 request_queue_size: 100 v2: enabled: false protocol_version: "2.0.0" uri_resolution: cache_ttl: 5m0s max_peers_per_result: 5 default_strategy: "best_match" resolution_timeout: 30s dht: enabled: false bootstrap_peers: [] mode: "auto" protocol_prefix: "/bzzz" bootstrap_timeout: 30s discovery_interval: 1m0s auto_bootstrap: false semantic_addressing: enable_wildcards: true default_agent: "any" default_role: "any" default_project: "any" enable_role_hierarchy: true feature_flags: uri_protocol: false semantic_addressing: false dht_discovery: false advanced_resolution: false ucxl: enabled: false server: port: 8081 base_path: "/bzzz" enabled: true resolution: cache_ttl: 5m0s enable_wildcards: true max_results: 50 storage: type: "filesystem" directory: "/tmp/bzzz-ucxl-storage" max_size: 104857600 p2p_integration: enable_announcement: true enable_discovery: true announcement_topic: "bzzz/ucxl/announcement/v1" discovery_timeout: 30s security: admin_key_shares: threshold: 3 total_shares: 5 election_config: heartbeat_timeout: 5s discovery_timeout: 30s election_timeout: 15s max_discovery_attempts: 6 discovery_backoff: 5s minimum_quorum: 3 consensus_algorithm: "raft" split_brain_detection: true conflict_resolution: "highest_uptime" key_rotation_days: 90 audit_logging: true audit_path: ".bzzz/security-audit.log" ai: ollama: endpoint: "http://192.168.1.27:11434" timeout: 30s models: ["phi3", "llama3.1"] openai: api_key: "" endpoint: "https://api.openai.com/v1" timeout: 30s `, hostname, hostname) return configYAML, nil } // GenerateConfigForMachineSimple generates a simple BZZZ configuration that matches the working config structure func (s *SetupManager) GenerateConfigForMachineSimple(machineIP string, config interface{}) (string, error) { // Note: Configuration extraction not needed for minimal template _ = config // Avoid unused parameter warning // Use machine IP to determine hostname (simplified) hostname := strings.ReplaceAll(machineIP, ".", "-") // Note: Using minimal config template - ports and security can be configured later // Generate YAML configuration that matches the Go struct requirements (minimal valid config) configYAML := fmt.Sprintf(`# BZZZ Configuration for %s whoosh_api: base_url: "https://whoosh.home.deepblack.cloud" api_key: "" timeout: 30s retry_count: 3 agent: id: "%s-agent" capabilities: ["general"] poll_interval: 30s max_tasks: 2 models: [] specialization: "" model_selection_webhook: "" default_reasoning_model: "" sandbox_image: "" role: "" system_prompt: "" reports_to: [] expertise: [] deliverables: [] collaboration: preferred_message_types: [] auto_subscribe_to_roles: [] auto_subscribe_to_expertise: [] response_timeout_seconds: 0 max_collaboration_depth: 0 escalation_threshold: 0 custom_topic_subscriptions: [] github: token_file: "" user_agent: "BZZZ-Agent/1.0" timeout: 30s rate_limit: true assignee: "" p2p: service_tag: "bzzz-peer-discovery" bzzz_topic: "bzzz/coordination/v1" hmmm_topic: "hmmm/meta-discussion/v1" discovery_timeout: 10s escalation_webhook: "" escalation_keywords: [] conversation_limit: 10 logging: level: "info" format: "text" output: "stdout" structured: false slurp: enabled: false base_url: "" api_key: "" timeout: 30s retry_count: 3 max_concurrent_requests: 10 request_queue_size: 100 v2: enabled: false protocol_version: "2.0.0" uri_resolution: cache_ttl: 5m0s max_peers_per_result: 5 default_strategy: "best_match" resolution_timeout: 30s dht: enabled: false bootstrap_peers: [] mode: "auto" protocol_prefix: "/bzzz" bootstrap_timeout: 30s discovery_interval: 1m0s auto_bootstrap: false semantic_addressing: enable_wildcards: true default_agent: "any" default_role: "any" default_project: "any" enable_role_hierarchy: true feature_flags: uri_protocol: false semantic_addressing: false dht_discovery: false advanced_resolution: false ucxl: enabled: false server: port: 8081 base_path: "/bzzz" enabled: false resolution: cache_ttl: 5m0s enable_wildcards: true max_results: 50 storage: type: "filesystem" directory: "/tmp/bzzz-ucxl-storage" max_size: 104857600 p2p_integration: enable_announcement: false enable_discovery: false announcement_topic: "bzzz/ucxl/announcement/v1" discovery_timeout: 30s security: admin_key_shares: threshold: 3 total_shares: 5 election_config: heartbeat_timeout: 5s discovery_timeout: 30s election_timeout: 15s max_discovery_attempts: 6 discovery_backoff: 5s minimum_quorum: 3 consensus_algorithm: "raft" split_brain_detection: true conflict_resolution: "highest_uptime" key_rotation_days: 90 audit_logging: false audit_path: "" ai: ollama: endpoint: "" timeout: 30s models: [] openai: api_key: "" endpoint: "https://api.openai.com/v1" timeout: 30s `, hostname, hostname) return configYAML, nil } // generateAndDeployConfig generates node-specific config.yaml and deploys it func (s *SetupManager) generateAndDeployConfig(client *ssh.Client, nodeIP string, config interface{}) error { // Get hostname for unique agent ID session, err := client.NewSession() if err != nil { return err } defer session.Close() var stdout strings.Builder session.Stdout = &stdout if err := session.Run("hostname"); err != nil { return fmt.Errorf("failed to get hostname: %w", err) } hostname := strings.TrimSpace(stdout.String()) // Generate YAML configuration using the shared method configYAML, err := s.GenerateConfigForMachineSimple(hostname, config) if err != nil { return fmt.Errorf("failed to generate config: %w", err) } // Create configuration directory session, err = client.NewSession() if err != nil { return err } defer session.Close() if err := session.Run("mkdir -p ~/.bzzz ~/.bzzz/data ~/.bzzz/logs"); err != nil { return fmt.Errorf("failed to create config directories: %w", err) } // Deploy configuration file session, err = client.NewSession() if err != nil { return err } defer session.Close() stdin, err := session.StdinPipe() if err != nil { return err } go func() { defer stdin.Close() stdin.Write([]byte(configYAML)) }() if err := session.Run("cat > ~/.bzzz/config.yaml"); err != nil { return fmt.Errorf("failed to deploy config file: %w", err) } return nil } // configureFirewall configures firewall rules for BZZZ ports func (s *SetupManager) configureFirewall(client *ssh.Client, config interface{}) error { // Extract ports from configuration configMap, ok := config.(map[string]interface{}) if !ok { return fmt.Errorf("invalid configuration format in firewall: expected map[string]interface{}, got %T: %+v", config, config) } ports := []string{"22"} // Always include SSH // Add BZZZ ports if portsConfig, exists := configMap["ports"]; exists { if portsMap, ok := portsConfig.(map[string]interface{}); ok { for _, value := range portsMap { if portStr := fmt.Sprintf("%v", value); portStr != "" { ports = append(ports, portStr) } } } } // Detect firewall system and configure rules session, err := client.NewSession() if err != nil { return err } defer session.Close() // Try ufw first (Ubuntu/Debian) if err := session.Run("which ufw > /dev/null 2>&1"); err == nil { return s.configureUFW(client, ports) } // Try firewalld (RHEL/CentOS/Fedora) session, err = client.NewSession() if err != nil { return err } defer session.Close() if err := session.Run("which firewall-cmd > /dev/null 2>&1"); err == nil { return s.configureFirewalld(client, ports) } // If no firewall detected, that's okay - just log it return nil } // configureUFW configures UFW firewall rules func (s *SetupManager) configureUFW(client *ssh.Client, ports []string) error { for _, port := range ports { session, err := client.NewSession() if err != nil { return err } defer session.Close() // Try with sudo, ignore failures for non-sudo users cmd := fmt.Sprintf("sudo -n ufw allow %s 2>/dev/null || true", port) session.Run(cmd) } return nil } // configureFirewalld configures firewalld rules func (s *SetupManager) configureFirewalld(client *ssh.Client, ports []string) error { for _, port := range ports { session, err := client.NewSession() if err != nil { return err } defer session.Close() // Try with sudo, ignore failures for non-sudo users cmd := fmt.Sprintf("sudo -n firewall-cmd --permanent --add-port=%s/tcp 2>/dev/null || true", port) session.Run(cmd) } // Reload firewall rules session, err := client.NewSession() if err != nil { return err } defer session.Close() session.Run("sudo -n firewall-cmd --reload 2>/dev/null || true") return nil } // ValidateOllamaEndpoint tests if an Ollama endpoint is accessible and returns available models func (s *SetupManager) ValidateOllamaEndpoint(endpoint string) (bool, []string, error) { if endpoint == "" { return false, nil, fmt.Errorf("endpoint cannot be empty") } // Ensure endpoint has proper format if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { endpoint = "http://" + endpoint } // Create HTTP client with timeout client := &http.Client{ Timeout: 10 * time.Second, } // Test connection to /api/tags endpoint apiURL := strings.TrimRight(endpoint, "/") + "/api/tags" resp, err := client.Get(apiURL) if err != nil { return false, nil, fmt.Errorf("failed to connect to Ollama API: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return false, nil, fmt.Errorf("Ollama API returned status %d", resp.StatusCode) } // Parse the response to get available models var tagsResponse struct { Models []struct { Name string `json:"name"` } `json:"models"` } if err := json.NewDecoder(resp.Body).Decode(&tagsResponse); err != nil { return false, nil, fmt.Errorf("failed to decode Ollama response: %w", err) } // Extract model names var models []string for _, model := range tagsResponse.Models { models = append(models, model.Name) } return true, models, nil }