Implement initial scan logic and council formation for WHOOSH project kickoffs

- Replace incremental sync with full scan for new repositories
- Add initial_scan status to bypass Since parameter filtering
- Implement council formation detection for Design Brief issues
- Add version display to WHOOSH UI header for debugging
- Fix Docker token authentication with trailing newline removal
- Add comprehensive council orchestration with Docker Swarm integration
- Include BACKBEAT prototype integration for distributed timing
- Support council-specific agent roles and deployment strategies
- Transition repositories to active status after content discovery

Key architectural improvements:
- Full scan approach for new project detection vs incremental sync
- Council formation triggered by chorus-entrypoint labeled Design Briefs
- Proper token handling and authentication for Gitea API calls
- Support for both initial discovery and ongoing task monitoring

This enables autonomous project kickoff workflows where Design Brief issues
automatically trigger formation of specialized agent councils for new projects.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude Code
2025-09-12 09:49:36 +10:00
parent b5c0deb6bc
commit 56ea52b743
74 changed files with 17778 additions and 236 deletions

View File

@@ -0,0 +1,591 @@
package orchestrator
import (
"context"
"fmt"
"time"
"github.com/chorus-services/whoosh/internal/composer"
"github.com/chorus-services/whoosh/internal/council"
"github.com/docker/docker/api/types/swarm"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rs/zerolog/log"
)
// AgentDeployer manages deployment of agent containers for teams
type AgentDeployer struct {
swarmManager *SwarmManager
db *pgxpool.Pool
registry string
ctx context.Context
cancel context.CancelFunc
}
// NewAgentDeployer creates a new agent deployer
func NewAgentDeployer(swarmManager *SwarmManager, db *pgxpool.Pool, registry string) *AgentDeployer {
ctx, cancel := context.WithCancel(context.Background())
if registry == "" {
registry = "registry.home.deepblack.cloud"
}
return &AgentDeployer{
swarmManager: swarmManager,
db: db,
registry: registry,
ctx: ctx,
cancel: cancel,
}
}
// Close shuts down the agent deployer
func (ad *AgentDeployer) Close() error {
ad.cancel()
return nil
}
// DeploymentRequest represents a request to deploy agents for a team
type DeploymentRequest struct {
TeamID uuid.UUID `json:"team_id"`
TaskID uuid.UUID `json:"task_id"`
TeamComposition *composer.TeamComposition `json:"team_composition"`
TaskContext *TaskContext `json:"task_context"`
DeploymentMode string `json:"deployment_mode"` // immediate, scheduled, manual
}
// DeploymentResult represents the result of a deployment operation
type DeploymentResult struct {
TeamID uuid.UUID `json:"team_id"`
TaskID uuid.UUID `json:"task_id"`
DeployedServices []DeployedService `json:"deployed_services"`
Status string `json:"status"` // success, partial, failed
Message string `json:"message"`
DeployedAt time.Time `json:"deployed_at"`
Errors []string `json:"errors,omitempty"`
}
// DeployedService represents a successfully deployed service
type DeployedService struct {
ServiceID string `json:"service_id"`
ServiceName string `json:"service_name"`
AgentRole string `json:"agent_role"`
AgentID string `json:"agent_id"`
Image string `json:"image"`
Status string `json:"status"`
}
// CouncilDeploymentRequest represents a request to deploy council agents
type CouncilDeploymentRequest struct {
CouncilID uuid.UUID `json:"council_id"`
ProjectName string `json:"project_name"`
CouncilComposition *council.CouncilComposition `json:"council_composition"`
ProjectContext *CouncilProjectContext `json:"project_context"`
DeploymentMode string `json:"deployment_mode"` // immediate, scheduled, manual
}
// CouncilProjectContext contains the project information for council agents
type CouncilProjectContext struct {
ProjectName string `json:"project_name"`
Repository string `json:"repository"`
ProjectBrief string `json:"project_brief"`
Constraints string `json:"constraints,omitempty"`
TechLimits string `json:"tech_limits,omitempty"`
ComplianceNotes string `json:"compliance_notes,omitempty"`
Targets string `json:"targets,omitempty"`
ExternalURL string `json:"external_url,omitempty"`
}
// DeployTeamAgents deploys all agents for a team
func (ad *AgentDeployer) DeployTeamAgents(request *DeploymentRequest) (*DeploymentResult, error) {
log.Info().
Str("team_id", request.TeamID.String()).
Str("task_id", request.TaskID.String()).
Int("agent_matches", len(request.TeamComposition.AgentMatches)).
Msg("🚀 Starting team agent deployment")
result := &DeploymentResult{
TeamID: request.TeamID,
TaskID: request.TaskID,
DeployedServices: []DeployedService{},
DeployedAt: time.Now(),
Errors: []string{},
}
// Deploy each agent in the team composition
for _, agentMatch := range request.TeamComposition.AgentMatches {
service, err := ad.deploySingleAgent(request, agentMatch)
if err != nil {
errorMsg := fmt.Sprintf("Failed to deploy agent %s for role %s: %v",
agentMatch.Agent.Name, agentMatch.Role.Name, err)
result.Errors = append(result.Errors, errorMsg)
log.Error().
Err(err).
Str("agent_id", agentMatch.Agent.ID.String()).
Str("role", agentMatch.Role.Name).
Msg("Failed to deploy agent")
continue
}
deployedService := DeployedService{
ServiceID: service.ID,
ServiceName: service.Spec.Name,
AgentRole: agentMatch.Role.Name,
AgentID: agentMatch.Agent.ID.String(),
Image: service.Spec.TaskTemplate.ContainerSpec.Image,
Status: "deploying",
}
result.DeployedServices = append(result.DeployedServices, deployedService)
// Update database with deployment info
err = ad.recordDeployment(request.TeamID, request.TaskID, agentMatch, service.ID)
if err != nil {
log.Error().
Err(err).
Str("service_id", service.ID).
Msg("Failed to record deployment in database")
}
}
// Determine overall deployment status
if len(result.Errors) == 0 {
result.Status = "success"
result.Message = fmt.Sprintf("Successfully deployed %d agents", len(result.DeployedServices))
} else if len(result.DeployedServices) > 0 {
result.Status = "partial"
result.Message = fmt.Sprintf("Deployed %d/%d agents with %d errors",
len(result.DeployedServices),
len(request.TeamComposition.AgentMatches),
len(result.Errors))
} else {
result.Status = "failed"
result.Message = "Failed to deploy any agents"
}
// Update team deployment status in database
err := ad.updateTeamDeploymentStatus(request.TeamID, result.Status, result.Message)
if err != nil {
log.Error().
Err(err).
Str("team_id", request.TeamID.String()).
Msg("Failed to update team deployment status")
}
log.Info().
Str("team_id", request.TeamID.String()).
Str("status", result.Status).
Int("deployed", len(result.DeployedServices)).
Int("errors", len(result.Errors)).
Msg("✅ Team agent deployment completed")
return result, nil
}
// selectAgentImage determines the appropriate CHORUS image for the agent role
func (ad *AgentDeployer) selectAgentImage(roleName string, agent *composer.Agent) string {
// All agents use the same CHORUS image, but with different configurations
// The image handles role specialization internally based on environment variables
return "docker.io/anthonyrawlins/chorus:backbeat-v2.0.1"
}
// buildAgentEnvironment creates environment variables for CHORUS agent configuration
func (ad *AgentDeployer) buildAgentEnvironment(request *DeploymentRequest, agentMatch *composer.AgentMatch) map[string]string {
env := map[string]string{
// Core CHORUS configuration - just pass the agent name from human-roles.yaml
// CHORUS will handle its own prompt composition and system behavior
"CHORUS_AGENT_NAME": agentMatch.Role.Name, // This maps to human-roles.yaml agent definition
"CHORUS_TEAM_ID": request.TeamID.String(),
"CHORUS_TASK_ID": request.TaskID.String(),
// Essential task context
"CHORUS_PROJECT": request.TaskContext.Repository,
"CHORUS_TASK_TITLE": request.TaskContext.IssueTitle,
"CHORUS_TASK_DESC": request.TaskContext.IssueDescription,
"CHORUS_PRIORITY": request.TaskContext.Priority,
"CHORUS_EXTERNAL_URL": request.TaskContext.ExternalURL,
// WHOOSH coordination
"WHOOSH_COORDINATOR": "true",
"WHOOSH_ENDPOINT": "http://whoosh:8080",
// Docker access for CHORUS sandbox management
"DOCKER_HOST": "unix:///var/run/docker.sock",
}
return env
}
// Note: CHORUS handles its own prompt composition from human-roles.yaml
// We just need to pass the agent name and essential task context
// determineAgentType maps role to agent type for resource allocation
func (ad *AgentDeployer) determineAgentType(agentMatch *composer.AgentMatch) string {
// Simple mapping for now - could be enhanced based on role complexity
return "standard"
}
// calculateResources determines resource requirements for the agent
func (ad *AgentDeployer) calculateResources(agentMatch *composer.AgentMatch) ResourceLimits {
// Standard resource allocation for CHORUS agents
// CHORUS handles its own resource management internally
return ResourceLimits{
CPULimit: 1000000000, // 1 CPU core
MemoryLimit: 1073741824, // 1GB RAM
CPURequest: 500000000, // 0.5 CPU core
MemoryRequest: 536870912, // 512MB RAM
}
}
// buildAgentVolumes creates volume mounts for CHORUS agents
func (ad *AgentDeployer) buildAgentVolumes(request *DeploymentRequest) []VolumeMount {
return []VolumeMount{
{
Type: "bind",
Source: "/var/run/docker.sock",
Target: "/var/run/docker.sock",
ReadOnly: false, // CHORUS needs Docker access for sandboxing
},
{
Type: "volume",
Source: fmt.Sprintf("whoosh-workspace-%s", request.TeamID.String()),
Target: "/workspace",
ReadOnly: false,
},
}
}
// buildAgentPlacement creates placement constraints for agents
func (ad *AgentDeployer) buildAgentPlacement(agentMatch *composer.AgentMatch) PlacementConfig {
return PlacementConfig{
Constraints: []string{
"node.role==worker", // Prefer worker nodes for agent containers
},
// Note: Placement preferences removed for compilation compatibility
}
}
// deploySingleAgent deploys a single agent for a specific role
func (ad *AgentDeployer) deploySingleAgent(request *DeploymentRequest, agentMatch *composer.AgentMatch) (*swarm.Service, error) {
// Determine agent image based on role
image := ad.selectAgentImage(agentMatch.Role.Name, agentMatch.Agent)
// Build deployment configuration
config := &AgentDeploymentConfig{
TeamID: request.TeamID.String(),
TaskID: request.TaskID.String(),
AgentRole: agentMatch.Role.Name,
AgentType: ad.determineAgentType(agentMatch),
Image: image,
Replicas: 1, // Start with single replica per agent
Resources: ad.calculateResources(agentMatch),
Environment: ad.buildAgentEnvironment(request, agentMatch),
TaskContext: *request.TaskContext,
Networks: []string{"chorus_default"},
Volumes: ad.buildAgentVolumes(request),
Placement: ad.buildAgentPlacement(agentMatch),
}
// Deploy the service
service, err := ad.swarmManager.DeployAgent(config)
if err != nil {
return nil, fmt.Errorf("failed to deploy agent service: %w", err)
}
return service, nil
}
// recordDeployment records agent deployment information in the database
func (ad *AgentDeployer) recordDeployment(teamID uuid.UUID, taskID uuid.UUID, agentMatch *composer.AgentMatch, serviceID string) error {
query := `
INSERT INTO agent_deployments (team_id, task_id, agent_id, role_id, service_id, status, deployed_at)
VALUES ($1, $2, $3, $4, $5, $6, NOW())
`
_, err := ad.db.Exec(ad.ctx, query, teamID, taskID, agentMatch.Agent.ID, agentMatch.Role.ID, serviceID, "deployed")
return err
}
// updateTeamDeploymentStatus updates the team deployment status in the database
func (ad *AgentDeployer) updateTeamDeploymentStatus(teamID uuid.UUID, status, message string) error {
query := `
UPDATE teams
SET deployment_status = $1, deployment_message = $2, updated_at = NOW()
WHERE id = $3
`
_, err := ad.db.Exec(ad.ctx, query, status, message, teamID)
return err
}
// DeployCouncilAgents deploys all agents for a project kickoff council
func (ad *AgentDeployer) DeployCouncilAgents(request *CouncilDeploymentRequest) (*council.CouncilDeploymentResult, error) {
log.Info().
Str("council_id", request.CouncilID.String()).
Str("project_name", request.ProjectName).
Int("core_agents", len(request.CouncilComposition.CoreAgents)).
Int("optional_agents", len(request.CouncilComposition.OptionalAgents)).
Msg("🎭 Starting council agent deployment")
result := &council.CouncilDeploymentResult{
CouncilID: request.CouncilID,
ProjectName: request.ProjectName,
DeployedAgents: []council.DeployedCouncilAgent{},
DeployedAt: time.Now(),
Errors: []string{},
}
// Deploy core agents (required)
for _, agent := range request.CouncilComposition.CoreAgents {
deployedAgent, err := ad.deploySingleCouncilAgent(request, agent)
if err != nil {
errorMsg := fmt.Sprintf("Failed to deploy core agent %s (%s): %v",
agent.AgentName, agent.RoleName, err)
result.Errors = append(result.Errors, errorMsg)
log.Error().
Err(err).
Str("agent_id", agent.AgentID).
Str("role", agent.RoleName).
Msg("Failed to deploy core council agent")
continue
}
result.DeployedAgents = append(result.DeployedAgents, *deployedAgent)
// Update database with deployment info
err = ad.recordCouncilAgentDeployment(request.CouncilID, agent, deployedAgent.ServiceID)
if err != nil {
log.Error().
Err(err).
Str("service_id", deployedAgent.ServiceID).
Msg("Failed to record council agent deployment in database")
}
}
// Deploy optional agents (best effort)
for _, agent := range request.CouncilComposition.OptionalAgents {
deployedAgent, err := ad.deploySingleCouncilAgent(request, agent)
if err != nil {
// Optional agents failing is not critical
log.Warn().
Err(err).
Str("agent_id", agent.AgentID).
Str("role", agent.RoleName).
Msg("Failed to deploy optional council agent (non-critical)")
continue
}
result.DeployedAgents = append(result.DeployedAgents, *deployedAgent)
// Update database with deployment info
err = ad.recordCouncilAgentDeployment(request.CouncilID, agent, deployedAgent.ServiceID)
if err != nil {
log.Error().
Err(err).
Str("service_id", deployedAgent.ServiceID).
Msg("Failed to record council agent deployment in database")
}
}
// Determine overall deployment status
coreAgentsCount := len(request.CouncilComposition.CoreAgents)
deployedCoreAgents := 0
for _, deployedAgent := range result.DeployedAgents {
// Check if this deployed agent is a core agent
for _, coreAgent := range request.CouncilComposition.CoreAgents {
if coreAgent.RoleName == deployedAgent.RoleName {
deployedCoreAgents++
break
}
}
}
if deployedCoreAgents == coreAgentsCount {
result.Status = "success"
result.Message = fmt.Sprintf("Successfully deployed %d agents (%d core, %d optional)",
len(result.DeployedAgents), deployedCoreAgents, len(result.DeployedAgents)-deployedCoreAgents)
} else if deployedCoreAgents > 0 {
result.Status = "partial"
result.Message = fmt.Sprintf("Deployed %d/%d core agents with %d errors",
deployedCoreAgents, coreAgentsCount, len(result.Errors))
} else {
result.Status = "failed"
result.Message = "Failed to deploy any core council agents"
}
// Update council deployment status in database
err := ad.updateCouncilDeploymentStatus(request.CouncilID, result.Status, result.Message)
if err != nil {
log.Error().
Err(err).
Str("council_id", request.CouncilID.String()).
Msg("Failed to update council deployment status")
}
log.Info().
Str("council_id", request.CouncilID.String()).
Str("status", result.Status).
Int("deployed", len(result.DeployedAgents)).
Int("errors", len(result.Errors)).
Msg("✅ Council agent deployment completed")
return result, nil
}
// deploySingleCouncilAgent deploys a single council agent
func (ad *AgentDeployer) deploySingleCouncilAgent(request *CouncilDeploymentRequest, agent council.CouncilAgent) (*council.DeployedCouncilAgent, error) {
// Use the CHORUS image for all council agents
image := "docker.io/anthonyrawlins/chorus:backbeat-v2.0.1"
// Build council-specific deployment configuration
config := &AgentDeploymentConfig{
TeamID: request.CouncilID.String(), // Use council ID as team ID
TaskID: request.CouncilID.String(), // Use council ID as task ID
AgentRole: agent.RoleName,
AgentType: "council",
Image: image,
Replicas: 1, // Single replica per council agent
Resources: ad.calculateCouncilResources(agent),
Environment: ad.buildCouncilAgentEnvironment(request, agent),
TaskContext: TaskContext{
Repository: request.ProjectContext.Repository,
IssueTitle: request.ProjectContext.ProjectName,
IssueDescription: request.ProjectContext.ProjectBrief,
Priority: "high", // Council formation is always high priority
ExternalURL: request.ProjectContext.ExternalURL,
},
Networks: []string{"chorus_default"}, // Connect to CHORUS network
Volumes: ad.buildCouncilAgentVolumes(request),
Placement: ad.buildCouncilAgentPlacement(agent),
}
// Deploy the service
service, err := ad.swarmManager.DeployAgent(config)
if err != nil {
return nil, fmt.Errorf("failed to deploy council agent service: %w", err)
}
// Create deployed agent result
deployedAgent := &council.DeployedCouncilAgent{
ServiceID: service.ID,
ServiceName: service.Spec.Name,
RoleName: agent.RoleName,
AgentID: agent.AgentID,
Image: image,
Status: "deploying",
DeployedAt: time.Now(),
}
return deployedAgent, nil
}
// buildCouncilAgentEnvironment creates environment variables for council agent configuration
func (ad *AgentDeployer) buildCouncilAgentEnvironment(request *CouncilDeploymentRequest, agent council.CouncilAgent) map[string]string {
env := map[string]string{
// Core CHORUS configuration for council mode
"CHORUS_AGENT_NAME": agent.RoleName, // Maps to human-roles.yaml agent definition
"CHORUS_COUNCIL_MODE": "true", // Enable council mode
"CHORUS_COUNCIL_ID": request.CouncilID.String(),
"CHORUS_PROJECT_NAME": request.ProjectContext.ProjectName,
// Council prompt and context
"CHORUS_COUNCIL_PROMPT": "/app/prompts/council.md",
"CHORUS_PROJECT_BRIEF": request.ProjectContext.ProjectBrief,
"CHORUS_CONSTRAINTS": request.ProjectContext.Constraints,
"CHORUS_TECH_LIMITS": request.ProjectContext.TechLimits,
"CHORUS_COMPLIANCE_NOTES": request.ProjectContext.ComplianceNotes,
"CHORUS_TARGETS": request.ProjectContext.Targets,
// Essential project context
"CHORUS_PROJECT": request.ProjectContext.Repository,
"CHORUS_EXTERNAL_URL": request.ProjectContext.ExternalURL,
"CHORUS_PRIORITY": "high",
// WHOOSH coordination
"WHOOSH_COORDINATOR": "true",
"WHOOSH_ENDPOINT": "http://whoosh:8080",
// Docker access for CHORUS sandbox management
"DOCKER_HOST": "unix:///var/run/docker.sock",
}
return env
}
// calculateCouncilResources determines resource requirements for council agents
func (ad *AgentDeployer) calculateCouncilResources(agent council.CouncilAgent) ResourceLimits {
// Council agents get slightly more resources since they handle complex analysis
return ResourceLimits{
CPULimit: 1500000000, // 1.5 CPU cores
MemoryLimit: 2147483648, // 2GB RAM
CPURequest: 750000000, // 0.75 CPU core
MemoryRequest: 1073741824, // 1GB RAM
}
}
// buildCouncilAgentVolumes creates volume mounts for council agents
func (ad *AgentDeployer) buildCouncilAgentVolumes(request *CouncilDeploymentRequest) []VolumeMount {
return []VolumeMount{
{
Type: "bind",
Source: "/var/run/docker.sock",
Target: "/var/run/docker.sock",
ReadOnly: false, // Council agents need Docker access for complex setup
},
{
Type: "volume",
Source: fmt.Sprintf("whoosh-council-%s", request.CouncilID.String()),
Target: "/workspace",
ReadOnly: false,
},
{
Type: "bind",
Source: "/rust/containers/WHOOSH/prompts",
Target: "/app/prompts",
ReadOnly: true, // Mount council prompts
},
}
}
// buildCouncilAgentPlacement creates placement constraints for council agents
func (ad *AgentDeployer) buildCouncilAgentPlacement(agent council.CouncilAgent) PlacementConfig {
return PlacementConfig{
Constraints: []string{
"node.role==worker", // Prefer worker nodes for council containers
},
}
}
// recordCouncilAgentDeployment records council agent deployment information in the database
func (ad *AgentDeployer) recordCouncilAgentDeployment(councilID uuid.UUID, agent council.CouncilAgent, serviceID string) error {
query := `
UPDATE council_agents
SET deployed = true, status = 'active', service_id = $1, deployed_at = NOW(), updated_at = NOW()
WHERE council_id = $2 AND agent_id = $3
`
_, err := ad.db.Exec(ad.ctx, query, serviceID, councilID, agent.AgentID)
return err
}
// updateCouncilDeploymentStatus updates the council deployment status in the database
func (ad *AgentDeployer) updateCouncilDeploymentStatus(councilID uuid.UUID, status, message string) error {
query := `
UPDATE councils
SET status = $1, updated_at = NOW()
WHERE id = $2
`
// Map deployment status to council status
councilStatus := "active"
if status == "failed" {
councilStatus = "failed"
} else if status == "partial" {
councilStatus = "active" // Partial deployment still allows council to function
}
_, err := ad.db.Exec(ad.ctx, query, councilStatus, councilID)
return err
}