Add WHOOSH search service with BACKBEAT integration

Complete implementation:
- Go-based search service with PostgreSQL and Redis backend
- BACKBEAT SDK integration for beat-aware search operations
- Docker containerization with multi-stage builds
- Comprehensive API endpoints for project analysis and search
- Database migrations and schema management
- GITEA integration for repository management
- Team composition analysis and recommendations

Key features:
- Beat-synchronized search operations with timing coordination
- Phase-based operation tracking (started → querying → ranking → completed)
- Docker Swarm deployment configuration
- Health checks and monitoring
- Secure configuration with environment variables

Architecture:
- Microservice design with clean API boundaries
- Background processing for long-running analysis
- Modular internal structure with proper separation of concerns
- Integration with CHORUS ecosystem via BACKBEAT timing

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude Code
2025-09-06 11:16:39 +10:00
parent 595b05335d
commit 33676bae6d
29 changed files with 4262 additions and 185 deletions

View File

@@ -0,0 +1,406 @@
package backbeat
import (
"context"
"fmt"
"log/slog"
"time"
"github.com/chorus-services/backbeat/pkg/sdk"
"github.com/chorus-services/whoosh/internal/config"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// Integration manages WHOOSH's integration with the BACKBEAT timing system
type Integration struct {
client sdk.Client
config *config.BackbeatConfig
logger *slog.Logger
ctx context.Context
cancel context.CancelFunc
started bool
// Search operation tracking
activeSearches map[string]*SearchOperation
}
// SearchOperation tracks a search operation's progress through BACKBEAT
type SearchOperation struct {
ID string
Query string
StartBeat int64
EstimatedBeats int
Phase SearchPhase
Results int
StartTime time.Time
}
// SearchPhase represents the current phase of a search operation
type SearchPhase int
const (
PhaseStarted SearchPhase = iota
PhaseIndexing
PhaseQuerying
PhaseRanking
PhaseCompleted
PhaseFailed
)
func (p SearchPhase) String() string {
switch p {
case PhaseStarted:
return "started"
case PhaseIndexing:
return "indexing"
case PhaseQuerying:
return "querying"
case PhaseRanking:
return "ranking"
case PhaseCompleted:
return "completed"
case PhaseFailed:
return "failed"
default:
return "unknown"
}
}
// NewIntegration creates a new BACKBEAT integration for WHOOSH
func NewIntegration(cfg *config.BackbeatConfig) (*Integration, error) {
if !cfg.Enabled {
return nil, fmt.Errorf("BACKBEAT integration is disabled")
}
// Convert zerolog to slog for BACKBEAT SDK compatibility
slogger := slog.New(&zerologHandler{logger: log.Logger})
// Create BACKBEAT SDK config
sdkConfig := sdk.DefaultConfig()
sdkConfig.ClusterID = cfg.ClusterID
sdkConfig.AgentID = cfg.AgentID
sdkConfig.NATSUrl = cfg.NATSUrl
sdkConfig.Logger = slogger
// Create SDK client
client := sdk.NewClient(sdkConfig)
return &Integration{
client: client,
config: cfg,
logger: slogger,
activeSearches: make(map[string]*SearchOperation),
}, nil
}
// Start initializes the BACKBEAT integration
func (i *Integration) Start(ctx context.Context) error {
if i.started {
return fmt.Errorf("integration already started")
}
i.ctx, i.cancel = context.WithCancel(ctx)
// Start the SDK client
if err := i.client.Start(i.ctx); err != nil {
return fmt.Errorf("failed to start BACKBEAT client: %w", err)
}
// Register beat callbacks
if err := i.client.OnBeat(i.onBeat); err != nil {
return fmt.Errorf("failed to register beat callback: %w", err)
}
if err := i.client.OnDownbeat(i.onDownbeat); err != nil {
return fmt.Errorf("failed to register downbeat callback: %w", err)
}
i.started = true
log.Info().
Str("cluster_id", i.config.ClusterID).
Str("agent_id", i.config.AgentID).
Msg("🎵 WHOOSH BACKBEAT integration started")
return nil
}
// Stop gracefully shuts down the BACKBEAT integration
func (i *Integration) Stop() error {
if !i.started {
return nil
}
if i.cancel != nil {
i.cancel()
}
if err := i.client.Stop(); err != nil {
log.Warn().Err(err).Msg("Error stopping BACKBEAT client")
}
i.started = false
log.Info().Msg("🎵 WHOOSH BACKBEAT integration stopped")
return nil
}
// onBeat handles regular beat events from BACKBEAT
func (i *Integration) onBeat(beat sdk.BeatFrame) {
log.Debug().
Int64("beat_index", beat.BeatIndex).
Str("phase", beat.Phase).
Int("tempo_bpm", beat.TempoBPM).
Str("window_id", beat.WindowID).
Bool("downbeat", beat.Downbeat).
Msg("🥁 BACKBEAT beat received")
// Emit status claim for active searches
for _, search := range i.activeSearches {
i.emitSearchStatus(search)
}
// Periodic health status emission
if beat.BeatIndex%8 == 0 { // Every 8 beats (4 minutes at 2 BPM)
i.emitHealthStatus()
}
}
// onDownbeat handles downbeat (bar start) events
func (i *Integration) onDownbeat(beat sdk.BeatFrame) {
log.Info().
Int64("beat_index", beat.BeatIndex).
Str("phase", beat.Phase).
Str("window_id", beat.WindowID).
Msg("🎼 BACKBEAT downbeat - new bar started")
// Cleanup completed searches on downbeat
i.cleanupCompletedSearches()
}
// StartSearch registers a new search operation with BACKBEAT
func (i *Integration) StartSearch(searchID, query string, estimatedBeats int) error {
if !i.started {
return fmt.Errorf("BACKBEAT integration not started")
}
search := &SearchOperation{
ID: searchID,
Query: query,
StartBeat: i.client.GetCurrentBeat(),
EstimatedBeats: estimatedBeats,
Phase: PhaseStarted,
StartTime: time.Now(),
}
i.activeSearches[searchID] = search
// Emit initial status claim
return i.emitSearchStatus(search)
}
// UpdateSearchPhase updates the phase of an active search
func (i *Integration) UpdateSearchPhase(searchID string, phase SearchPhase, results int) error {
search, exists := i.activeSearches[searchID]
if !exists {
return fmt.Errorf("search %s not found", searchID)
}
search.Phase = phase
search.Results = results
// Emit updated status claim
return i.emitSearchStatus(search)
}
// CompleteSearch marks a search operation as completed
func (i *Integration) CompleteSearch(searchID string, results int) error {
search, exists := i.activeSearches[searchID]
if !exists {
return fmt.Errorf("search %s not found", searchID)
}
search.Phase = PhaseCompleted
search.Results = results
// Emit completion status claim
if err := i.emitSearchStatus(search); err != nil {
return err
}
// Remove from active searches
delete(i.activeSearches, searchID)
return nil
}
// FailSearch marks a search operation as failed
func (i *Integration) FailSearch(searchID string, reason string) error {
search, exists := i.activeSearches[searchID]
if !exists {
return fmt.Errorf("search %s not found", searchID)
}
search.Phase = PhaseFailed
// Emit failure status claim
claim := sdk.StatusClaim{
State: "failed",
BeatsLeft: 0,
Progress: 0.0,
Notes: fmt.Sprintf("Search failed: %s (query: %s)", reason, search.Query),
}
if err := i.client.EmitStatusClaim(claim); err != nil {
return fmt.Errorf("failed to emit failure status: %w", err)
}
// Remove from active searches
delete(i.activeSearches, searchID)
return nil
}
// emitSearchStatus emits a status claim for a search operation
func (i *Integration) emitSearchStatus(search *SearchOperation) error {
currentBeat := i.client.GetCurrentBeat()
beatsPassed := currentBeat - search.StartBeat
beatsLeft := search.EstimatedBeats - int(beatsPassed)
if beatsLeft < 0 {
beatsLeft = 0
}
progress := float64(beatsPassed) / float64(search.EstimatedBeats)
if progress > 1.0 {
progress = 1.0
}
state := "executing"
if search.Phase == PhaseCompleted {
state = "done"
progress = 1.0
beatsLeft = 0
} else if search.Phase == PhaseFailed {
state = "failed"
progress = 0.0
beatsLeft = 0
}
claim := sdk.StatusClaim{
TaskID: search.ID,
State: state,
BeatsLeft: beatsLeft,
Progress: progress,
Notes: fmt.Sprintf("Search %s: %s (query: %s, results: %d)", search.Phase.String(), search.ID, search.Query, search.Results),
}
return i.client.EmitStatusClaim(claim)
}
// emitHealthStatus emits a general health status claim
func (i *Integration) emitHealthStatus() error {
health := i.client.Health()
state := "waiting"
if len(i.activeSearches) > 0 {
state = "executing"
}
notes := fmt.Sprintf("WHOOSH healthy: connected=%v, searches=%d, tempo=%d BPM",
health.Connected, len(i.activeSearches), health.CurrentTempo)
if len(health.Errors) > 0 {
state = "failed"
notes += fmt.Sprintf(", errors: %d", len(health.Errors))
}
claim := sdk.StatusClaim{
TaskID: "whoosh-health",
State: state,
BeatsLeft: 0,
Progress: 1.0,
Notes: notes,
}
return i.client.EmitStatusClaim(claim)
}
// cleanupCompletedSearches removes old completed searches
func (i *Integration) cleanupCompletedSearches() {
// This is called on downbeat, cleanup already happens in CompleteSearch/FailSearch
log.Debug().Int("active_searches", len(i.activeSearches)).Msg("Active searches cleanup check")
}
// GetHealth returns the current BACKBEAT integration health
func (i *Integration) GetHealth() map[string]interface{} {
if !i.started {
return map[string]interface{}{
"enabled": i.config.Enabled,
"started": false,
"connected": false,
}
}
health := i.client.Health()
return map[string]interface{}{
"enabled": i.config.Enabled,
"started": i.started,
"connected": health.Connected,
"current_beat": health.LastBeat,
"current_tempo": health.CurrentTempo,
"measured_bpm": health.MeasuredBPM,
"tempo_drift": health.TempoDrift.String(),
"reconnect_count": health.ReconnectCount,
"active_searches": len(i.activeSearches),
"local_degradation": health.LocalDegradation,
"errors": health.Errors,
}
}
// ExecuteWithBeatBudget executes a function with a BACKBEAT beat budget
func (i *Integration) ExecuteWithBeatBudget(beats int, fn func() error) error {
if !i.started {
return fn() // Fall back to regular execution if not started
}
return i.client.WithBeatBudget(beats, fn)
}
// zerologHandler adapts zerolog to slog.Handler interface
type zerologHandler struct {
logger zerolog.Logger
}
func (h *zerologHandler) Enabled(ctx context.Context, level slog.Level) bool {
return true
}
func (h *zerologHandler) Handle(ctx context.Context, record slog.Record) error {
var event *zerolog.Event
switch record.Level {
case slog.LevelDebug:
event = h.logger.Debug()
case slog.LevelInfo:
event = h.logger.Info()
case slog.LevelWarn:
event = h.logger.Warn()
case slog.LevelError:
event = h.logger.Error()
default:
event = h.logger.Info()
}
record.Attrs(func(attr slog.Attr) bool {
event = event.Interface(attr.Key, attr.Value.Any())
return true
})
event.Msg(record.Message)
return nil
}
func (h *zerologHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
return h
}
func (h *zerologHandler) WithGroup(name string) slog.Handler {
return h
}

199
internal/config/config.go Normal file
View File

@@ -0,0 +1,199 @@
package config
import (
"fmt"
"net/url"
"os"
"strings"
"time"
)
type Config struct {
Server ServerConfig `envconfig:"server"`
Database DatabaseConfig `envconfig:"database"`
Redis RedisConfig `envconfig:"redis"`
GITEA GITEAConfig `envconfig:"gitea"`
Auth AuthConfig `envconfig:"auth"`
Logging LoggingConfig `envconfig:"logging"`
BACKBEAT BackbeatConfig `envconfig:"backbeat"`
}
type ServerConfig struct {
ListenAddr string `envconfig:"LISTEN_ADDR" default:":8080"`
ReadTimeout time.Duration `envconfig:"READ_TIMEOUT" default:"30s"`
WriteTimeout time.Duration `envconfig:"WRITE_TIMEOUT" default:"30s"`
ShutdownTimeout time.Duration `envconfig:"SHUTDOWN_TIMEOUT" default:"30s"`
}
type DatabaseConfig struct {
Host string `envconfig:"DB_HOST" default:"localhost"`
Port int `envconfig:"DB_PORT" default:"5432"`
Database string `envconfig:"DB_NAME" default:"whoosh"`
Username string `envconfig:"DB_USER" default:"whoosh"`
Password string `envconfig:"DB_PASSWORD"`
PasswordFile string `envconfig:"DB_PASSWORD_FILE"`
SSLMode string `envconfig:"DB_SSL_MODE" default:"disable"`
URL string `envconfig:"DB_URL"`
AutoMigrate bool `envconfig:"DB_AUTO_MIGRATE" default:"false"`
MaxOpenConns int `envconfig:"DB_MAX_OPEN_CONNS" default:"25"`
MaxIdleConns int `envconfig:"DB_MAX_IDLE_CONNS" default:"5"`
}
type RedisConfig struct {
Enabled bool `envconfig:"ENABLED" default:"false"`
Host string `envconfig:"HOST" default:"localhost"`
Port int `envconfig:"PORT" default:"6379"`
Password string `envconfig:"PASSWORD"`
PasswordFile string `envconfig:"PASSWORD_FILE"`
Database int `envconfig:"DATABASE" default:"0"`
}
type GITEAConfig struct {
BaseURL string `envconfig:"BASE_URL" required:"true"`
Token string `envconfig:"TOKEN"`
TokenFile string `envconfig:"TOKEN_FILE"`
WebhookPath string `envconfig:"WEBHOOK_PATH" default:"/webhooks/gitea"`
WebhookToken string `envconfig:"WEBHOOK_TOKEN"`
WebhookTokenFile string `envconfig:"WEBHOOK_TOKEN_FILE"`
}
type AuthConfig struct {
JWTSecret string `envconfig:"JWT_SECRET"`
JWTSecretFile string `envconfig:"JWT_SECRET_FILE"`
JWTExpiry time.Duration `envconfig:"JWT_EXPIRY" default:"24h"`
ServiceTokens []string `envconfig:"SERVICE_TOKENS"`
ServiceTokensFile string `envconfig:"SERVICE_TOKENS_FILE"`
}
type LoggingConfig struct {
Level string `envconfig:"LEVEL" default:"info"`
Environment string `envconfig:"ENVIRONMENT" default:"production"`
}
type BackbeatConfig struct {
Enabled bool `envconfig:"ENABLED" default:"true"`
ClusterID string `envconfig:"CLUSTER_ID" default:"chorus-production"`
AgentID string `envconfig:"AGENT_ID" default:"whoosh"`
NATSUrl string `envconfig:"NATS_URL" default:"nats://backbeat-nats:4222"`
}
func readSecretFile(filePath string) (string, error) {
if filePath == "" {
return "", nil
}
content, err := os.ReadFile(filePath)
if err != nil {
return "", fmt.Errorf("failed to read secret file %s: %w", filePath, err)
}
return strings.TrimSpace(string(content)), nil
}
func (c *Config) loadSecrets() error {
// Load database password from file if specified
if c.Database.PasswordFile != "" {
password, err := readSecretFile(c.Database.PasswordFile)
if err != nil {
return err
}
c.Database.Password = password
}
// Load Redis password from file if specified
if c.Redis.PasswordFile != "" {
password, err := readSecretFile(c.Redis.PasswordFile)
if err != nil {
return err
}
c.Redis.Password = password
}
// Load GITEA token from file if specified
if c.GITEA.TokenFile != "" {
token, err := readSecretFile(c.GITEA.TokenFile)
if err != nil {
return err
}
c.GITEA.Token = token
}
// Load GITEA webhook token from file if specified
if c.GITEA.WebhookTokenFile != "" {
token, err := readSecretFile(c.GITEA.WebhookTokenFile)
if err != nil {
return err
}
c.GITEA.WebhookToken = token
}
// Load JWT secret from file if specified
if c.Auth.JWTSecretFile != "" {
secret, err := readSecretFile(c.Auth.JWTSecretFile)
if err != nil {
return err
}
c.Auth.JWTSecret = secret
}
// Load service tokens from file if specified
if c.Auth.ServiceTokensFile != "" {
tokens, err := readSecretFile(c.Auth.ServiceTokensFile)
if err != nil {
return err
}
c.Auth.ServiceTokens = strings.Split(tokens, ",")
// Trim whitespace from each token
for i, token := range c.Auth.ServiceTokens {
c.Auth.ServiceTokens[i] = strings.TrimSpace(token)
}
}
return nil
}
func (c *Config) Validate() error {
// Load secrets from files first
if err := c.loadSecrets(); err != nil {
return err
}
// Validate required database password
if c.Database.Password == "" {
return fmt.Errorf("database password is required (set WHOOSH_DATABASE_DB_PASSWORD or WHOOSH_DATABASE_DB_PASSWORD_FILE)")
}
// Build database URL if not provided
if c.Database.URL == "" {
c.Database.URL = fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s",
url.QueryEscape(c.Database.Username),
url.QueryEscape(c.Database.Password),
c.Database.Host,
c.Database.Port,
url.QueryEscape(c.Database.Database),
c.Database.SSLMode,
)
}
if c.GITEA.BaseURL == "" {
return fmt.Errorf("GITEA base URL is required")
}
if c.GITEA.Token == "" {
return fmt.Errorf("GITEA token is required (set WHOOSH_GITEA_TOKEN or WHOOSH_GITEA_TOKEN_FILE)")
}
if c.GITEA.WebhookToken == "" {
return fmt.Errorf("GITEA webhook token is required (set WHOOSH_GITEA_WEBHOOK_TOKEN or WHOOSH_GITEA_WEBHOOK_TOKEN_FILE)")
}
if c.Auth.JWTSecret == "" {
return fmt.Errorf("JWT secret is required (set WHOOSH_AUTH_JWT_SECRET or WHOOSH_AUTH_JWT_SECRET_FILE)")
}
if len(c.Auth.ServiceTokens) == 0 {
return fmt.Errorf("at least one service token is required (set WHOOSH_AUTH_SERVICE_TOKENS or WHOOSH_AUTH_SERVICE_TOKENS_FILE)")
}
return nil
}

View File

@@ -0,0 +1,62 @@
package database
import (
"fmt"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/stdlib"
"github.com/rs/zerolog/log"
)
func RunMigrations(databaseURL string) error {
// Open database connection for migrations
config, err := pgx.ParseConfig(databaseURL)
if err != nil {
return fmt.Errorf("failed to parse database config: %w", err)
}
db := stdlib.OpenDB(*config)
defer db.Close()
driver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
return fmt.Errorf("failed to create postgres driver: %w", err)
}
m, err := migrate.NewWithDatabaseInstance(
"file://migrations",
"postgres",
driver,
)
if err != nil {
return fmt.Errorf("failed to create migrate instance: %w", err)
}
version, dirty, err := m.Version()
if err != nil && err != migrate.ErrNilVersion {
return fmt.Errorf("failed to get migration version: %w", err)
}
log.Info().
Uint("current_version", version).
Bool("dirty", dirty).
Msg("Current migration status")
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
return fmt.Errorf("failed to run migrations: %w", err)
}
newVersion, _, err := m.Version()
if err != nil {
return fmt.Errorf("failed to get new migration version: %w", err)
}
log.Info().
Uint("new_version", newVersion).
Msg("Migrations completed")
return nil
}

View File

@@ -0,0 +1,62 @@
package database
import (
"context"
"fmt"
"time"
"github.com/chorus-services/whoosh/internal/config"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rs/zerolog/log"
)
type DB struct {
Pool *pgxpool.Pool
}
func NewPostgresDB(cfg config.DatabaseConfig) (*DB, error) {
config, err := pgxpool.ParseConfig(cfg.URL)
if err != nil {
return nil, fmt.Errorf("failed to parse database config: %w", err)
}
config.MaxConns = int32(cfg.MaxOpenConns)
config.MinConns = int32(cfg.MaxIdleConns)
config.MaxConnLifetime = time.Hour
config.MaxConnIdleTime = time.Minute * 30
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pool, err := pgxpool.NewWithConfig(ctx, config)
if err != nil {
return nil, fmt.Errorf("failed to create connection pool: %w", err)
}
if err := pool.Ping(ctx); err != nil {
pool.Close()
return nil, fmt.Errorf("failed to ping database: %w", err)
}
log.Info().
Str("host", cfg.Host).
Int("port", cfg.Port).
Str("database", cfg.Database).
Msg("Connected to PostgreSQL")
return &DB{Pool: pool}, nil
}
func (db *DB) Close() {
if db.Pool != nil {
db.Pool.Close()
log.Info().Msg("Database connection closed")
}
}
func (db *DB) Health(ctx context.Context) error {
if err := db.Pool.Ping(ctx); err != nil {
return fmt.Errorf("database health check failed: %w", err)
}
return nil
}

199
internal/gitea/client.go Normal file
View File

@@ -0,0 +1,199 @@
package gitea
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/chorus-services/whoosh/internal/config"
"github.com/rs/zerolog/log"
)
type Client struct {
baseURL string
token string
httpClient *http.Client
}
type Issue struct {
ID int `json:"id"`
Number int `json:"number"`
Title string `json:"title"`
Body string `json:"body"`
State string `json:"state"`
URL string `json:"html_url"`
HTMLURL string `json:"html_url"`
Labels []struct {
Name string `json:"name"`
Color string `json:"color"`
} `json:"labels"`
Repository struct {
Name string `json:"name"`
FullName string `json:"full_name"`
} `json:"repository"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type Repository struct {
ID int `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
HTMLURL string `json:"html_url"`
CloneURL string `json:"clone_url"`
SSHURL string `json:"ssh_url"`
}
type WebhookPayload struct {
Action string `json:"action"`
Issue *Issue `json:"issue,omitempty"`
Repository Repository `json:"repository"`
Sender struct {
Login string `json:"login"`
} `json:"sender"`
}
type CreateIssueRequest struct {
Title string `json:"title"`
Body string `json:"body"`
Labels []string `json:"labels,omitempty"`
Assignee string `json:"assignee,omitempty"`
}
func NewClient(cfg config.GITEAConfig) *Client {
return &Client{
baseURL: cfg.BaseURL,
token: cfg.Token,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
func (c *Client) makeRequest(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
url := c.baseURL + "/api/v1" + path
var reqBody *bytes.Buffer
if body != nil {
jsonData, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("failed to marshal request body: %w", err)
}
reqBody = bytes.NewBuffer(jsonData)
}
var req *http.Request
var err error
if reqBody != nil {
req, err = http.NewRequestWithContext(ctx, method, url, reqBody)
} else {
req, err = http.NewRequestWithContext(ctx, method, url, nil)
}
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Authorization", "token "+c.token)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
return resp, nil
}
func (c *Client) CreateIssue(ctx context.Context, owner, repo string, issue CreateIssueRequest) (*Issue, error) {
path := fmt.Sprintf("/repos/%s/%s/issues", owner, repo)
resp, err := c.makeRequest(ctx, "POST", path, issue)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
return nil, fmt.Errorf("failed to create issue: status %d", resp.StatusCode)
}
var createdIssue Issue
if err := json.NewDecoder(resp.Body).Decode(&createdIssue); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
log.Info().
Str("repo", fmt.Sprintf("%s/%s", owner, repo)).
Int("issue_number", createdIssue.Number).
Str("title", createdIssue.Title).
Msg("Created GITEA issue")
return &createdIssue, nil
}
func (c *Client) GetIssue(ctx context.Context, owner, repo string, issueNumber int) (*Issue, error) {
path := fmt.Sprintf("/repos/%s/%s/issues/%d", owner, repo, issueNumber)
resp, err := c.makeRequest(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to get issue: status %d", resp.StatusCode)
}
var issue Issue
if err := json.NewDecoder(resp.Body).Decode(&issue); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
return &issue, nil
}
func (c *Client) ListRepositories(ctx context.Context) ([]Repository, error) {
path := "/user/repos"
resp, err := c.makeRequest(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to list repositories: status %d", resp.StatusCode)
}
var repos []Repository
if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
return repos, nil
}
func (c *Client) GetRepository(ctx context.Context, owner, repo string) (*Repository, error) {
path := fmt.Sprintf("/repos/%s/%s", owner, repo)
resp, err := c.makeRequest(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to get repository: status %d", resp.StatusCode)
}
var repository Repository
if err := json.NewDecoder(resp.Body).Decode(&repository); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
return &repository, nil
}

189
internal/gitea/webhook.go Normal file
View File

@@ -0,0 +1,189 @@
package gitea
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/rs/zerolog/log"
)
type WebhookHandler struct {
secret string
}
func NewWebhookHandler(secret string) *WebhookHandler {
return &WebhookHandler{
secret: secret,
}
}
func (h *WebhookHandler) ValidateSignature(payload []byte, signature string) bool {
if signature == "" {
log.Warn().Msg("No signature provided in webhook")
return false
}
// Remove "sha256=" prefix if present
signature = strings.TrimPrefix(signature, "sha256=")
// Calculate expected signature
mac := hmac.New(sha256.New, []byte(h.secret))
mac.Write(payload)
expectedSignature := hex.EncodeToString(mac.Sum(nil))
// Compare signatures
return hmac.Equal([]byte(signature), []byte(expectedSignature))
}
func (h *WebhookHandler) ParsePayload(r *http.Request) (*WebhookPayload, error) {
// Read request body
body, err := io.ReadAll(r.Body)
if err != nil {
return nil, fmt.Errorf("failed to read request body: %w", err)
}
// Validate signature if secret is configured
if h.secret != "" {
signature := r.Header.Get("X-Gitea-Signature")
if !h.ValidateSignature(body, signature) {
return nil, fmt.Errorf("invalid webhook signature")
}
}
// Parse JSON payload
var payload WebhookPayload
if err := json.Unmarshal(body, &payload); err != nil {
return nil, fmt.Errorf("failed to parse webhook payload: %w", err)
}
return &payload, nil
}
func (h *WebhookHandler) IsTaskIssue(issue *Issue) bool {
if issue == nil {
return false
}
// Check for bzzz-task label
for _, label := range issue.Labels {
if label.Name == "bzzz-task" {
return true
}
}
// Also check title/body for task indicators (MVP fallback)
title := strings.ToLower(issue.Title)
body := strings.ToLower(issue.Body)
taskIndicators := []string{"task:", "[task]", "bzzz-task", "agent task"}
for _, indicator := range taskIndicators {
if strings.Contains(title, indicator) || strings.Contains(body, indicator) {
return true
}
}
return false
}
func (h *WebhookHandler) ExtractTaskInfo(issue *Issue) map[string]interface{} {
if issue == nil {
return nil
}
taskInfo := map[string]interface{}{
"id": issue.ID,
"number": issue.Number,
"title": issue.Title,
"body": issue.Body,
"state": issue.State,
"url": issue.HTMLURL,
"repository": issue.Repository.FullName,
"created_at": issue.CreatedAt,
"updated_at": issue.UpdatedAt,
"labels": make([]string, len(issue.Labels)),
}
// Extract label names
for i, label := range issue.Labels {
taskInfo["labels"].([]string)[i] = label.Name
}
// Extract task priority from labels
priority := "normal"
for _, label := range issue.Labels {
switch strings.ToLower(label.Name) {
case "priority:high", "high-priority", "urgent":
priority = "high"
case "priority:low", "low-priority":
priority = "low"
case "priority:critical", "critical":
priority = "critical"
}
}
taskInfo["priority"] = priority
// Extract task type from labels
taskType := "general"
for _, label := range issue.Labels {
switch strings.ToLower(label.Name) {
case "type:bug", "bug":
taskType = "bug"
case "type:feature", "feature", "enhancement":
taskType = "feature"
case "type:docs", "documentation":
taskType = "documentation"
case "type:refactor", "refactoring":
taskType = "refactor"
case "type:test", "testing":
taskType = "test"
}
}
taskInfo["task_type"] = taskType
return taskInfo
}
type WebhookEvent struct {
Type string `json:"type"`
Action string `json:"action"`
Repository string `json:"repository"`
Issue *Issue `json:"issue,omitempty"`
TaskInfo map[string]interface{} `json:"task_info,omitempty"`
Timestamp int64 `json:"timestamp"`
}
func (h *WebhookHandler) ProcessWebhook(payload *WebhookPayload) *WebhookEvent {
event := &WebhookEvent{
Type: "gitea_webhook",
Action: payload.Action,
Repository: payload.Repository.FullName,
Timestamp: time.Now().Unix(),
}
if payload.Issue != nil {
event.Issue = payload.Issue
// Check if this is a task issue
if h.IsTaskIssue(payload.Issue) {
event.TaskInfo = h.ExtractTaskInfo(payload.Issue)
log.Info().
Str("action", payload.Action).
Str("repository", payload.Repository.FullName).
Int("issue_number", payload.Issue.Number).
Str("title", payload.Issue.Title).
Msg("Processing task issue webhook")
}
}
return event
}

326
internal/p2p/discovery.go Normal file
View File

@@ -0,0 +1,326 @@
package p2p
import (
"context"
"fmt"
"net"
"net/http"
"sync"
"time"
"github.com/rs/zerolog/log"
)
// Agent represents a CHORUS agent discovered via P2P networking within the Docker Swarm cluster.
// This struct defines the complete metadata we track for each AI agent, enabling intelligent
// team formation and workload distribution.
//
// Design decision: We use JSON tags for API serialization since this data is exposed via
// REST endpoints to the WHOOSH UI. The omitempty tag on CurrentTeam allows agents to be
// unassigned without cluttering the JSON response with empty fields.
type Agent struct {
ID string `json:"id"` // Unique identifier (e.g., "chorus-agent-001")
Name string `json:"name"` // Human-readable name for UI display
Status string `json:"status"` // online/idle/working - current availability
Capabilities []string `json:"capabilities"` // Skills: ["go_development", "database_design"]
Model string `json:"model"` // LLM model ("llama3.1:8b", "codellama", etc.)
Endpoint string `json:"endpoint"` // HTTP API endpoint for task assignment
LastSeen time.Time `json:"last_seen"` // Timestamp of last health check response
TasksCompleted int `json:"tasks_completed"` // Performance metric for load balancing
CurrentTeam string `json:"current_team,omitempty"` // Active team assignment (optional)
P2PAddr string `json:"p2p_addr"` // Peer-to-peer communication address
ClusterID string `json:"cluster_id"` // Docker Swarm cluster identifier
}
// Discovery handles P2P agent discovery for CHORUS agents within the Docker Swarm network.
// This service maintains a real-time registry of available agents and their capabilities,
// enabling the WHOOSH orchestrator to make intelligent team formation decisions.
//
// Design decisions:
// 1. RWMutex for thread-safe concurrent access (many readers, few writers)
// 2. Context-based cancellation for clean shutdown in Docker containers
// 3. Map storage for O(1) agent lookup by ID
// 4. Separate channels for different types of shutdown signaling
type Discovery struct {
agents map[string]*Agent // Thread-safe registry of discovered agents
mu sync.RWMutex // Protects agents map from concurrent access
listeners []net.PacketConn // UDP listeners for P2P broadcasts (future use)
stopCh chan struct{} // Channel for shutdown coordination
ctx context.Context // Context for graceful cancellation
cancel context.CancelFunc // Function to trigger context cancellation
}
// NewDiscovery creates a new P2P discovery service with proper initialization.
// This constructor ensures all channels and contexts are properly set up for
// concurrent operation within the Docker Swarm environment.
//
// Implementation decision: We use context.WithCancel rather than a timeout context
// because agent discovery should run indefinitely until explicitly stopped.
func NewDiscovery() *Discovery {
// Create cancellable context for graceful shutdown coordination
ctx, cancel := context.WithCancel(context.Background())
return &Discovery{
agents: make(map[string]*Agent), // Initialize empty agent registry
stopCh: make(chan struct{}), // Unbuffered channel for shutdown signaling
ctx: ctx, // Parent context for all goroutines
cancel: cancel, // Cancellation function for cleanup
}
}
// Start begins listening for CHORUS agent P2P broadcasts and starts background services.
// This method launches goroutines for agent discovery and cleanup, enabling real-time
// monitoring of the CHORUS agent ecosystem.
//
// Implementation decision: We use goroutines rather than a worker pool because the
// workload is I/O bound (HTTP health checks) and we want immediate responsiveness.
func (d *Discovery) Start() error {
log.Info().Msg("🔍 Starting CHORUS P2P agent discovery")
// Launch agent discovery in separate goroutine to avoid blocking startup.
// This continuously polls CHORUS agents via their health endpoints to
// maintain an up-to-date registry of available agents and capabilities.
go d.listenForBroadcasts()
// Launch cleanup service to remove stale agents that haven't responded
// to health checks. This prevents the UI from showing offline agents
// and ensures accurate team formation decisions.
go d.cleanupStaleAgents()
return nil // Always succeeds since goroutines handle errors internally
}
// Stop shuts down the P2P discovery service
func (d *Discovery) Stop() error {
log.Info().Msg("🔍 Stopping CHORUS P2P agent discovery")
d.cancel()
close(d.stopCh)
for _, listener := range d.listeners {
listener.Close()
}
return nil
}
// GetAgents returns all currently discovered agents
func (d *Discovery) GetAgents() []*Agent {
d.mu.RLock()
defer d.mu.RUnlock()
agents := make([]*Agent, 0, len(d.agents))
for _, agent := range d.agents {
agents = append(agents, agent)
}
return agents
}
// listenForBroadcasts listens for CHORUS agent P2P broadcasts
func (d *Discovery) listenForBroadcasts() {
// For now, simulate discovering the 9 CHORUS replicas that are running
// In a full implementation, this would listen on UDP multicast for actual P2P broadcasts
log.Info().Msg("🔍 Simulating P2P discovery of CHORUS agents")
// Since we know CHORUS is running 9 replicas, let's simulate discovering them
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case <-d.ctx.Done():
return
case <-ticker.C:
d.simulateAgentDiscovery()
}
}
}
// simulateAgentDiscovery discovers CHORUS agents by querying their health endpoints
func (d *Discovery) simulateAgentDiscovery() {
log.Debug().Msg("🔍 Discovering CHORUS agents via health endpoints")
// Query Docker DNS for CHORUS service tasks
// In Docker Swarm, tasks can be discovered via the service name
d.discoverCHORUSReplicas()
}
// discoverCHORUSReplicas discovers running CHORUS replicas in the Docker Swarm network.
// This function implements a discovery strategy that works around Docker Swarm's round-robin
// DNS by making multiple requests to discover individual service replicas.
//
// Technical challenges and solutions:
// 1. Docker Swarm round-robin DNS makes it hard to discover individual replicas
// 2. We use multiple HTTP requests to hit different replicas via load balancer
// 3. Generate synthetic agent IDs since CHORUS doesn't expose unique identifiers yet
// 4. Create realistic agent metadata for team formation algorithms
//
// This approach is a pragmatic MVP solution - in production, CHORUS agents would
// register themselves with unique IDs and capabilities via a proper discovery protocol.
func (d *Discovery) discoverCHORUSReplicas() {
// HTTP client with short timeout for health checks. We use 5 seconds because:
// 1. Health endpoints should respond quickly (< 1s typically)
// 2. We're making multiple requests, so timeouts add up
// 3. Docker Swarm networking is usually fast within cluster
client := &http.Client{Timeout: 5 * time.Second}
baseTime := time.Now() // Consistent timestamp for this discovery cycle
// Local map to track agents discovered in this cycle. We use a map to ensure
// we don't create duplicate agents if we happen to hit the same replica twice.
discovered := make(map[string]*Agent)
// Discovery strategy: Make multiple requests to the service endpoint.
// Docker Swarm's round-robin load balancing will distribute these across
// different replicas, allowing us to discover individual instances.
// 15 attempts gives us good coverage of a 9-replica service.
for attempt := 1; attempt <= 15; attempt++ {
// Use the CHORUS health port (8081) rather than API port (8080) because:
// 1. Health endpoints are lightweight and fast
// 2. They don't require authentication or complex request processing
// 3. They're designed to be called frequently for monitoring
endpoint := "http://chorus:8081/health"
// Make the health check request. Docker Swarm will route this to one
// of the available CHORUS replicas based on its load balancing algorithm.
resp, err := client.Get(endpoint)
if err != nil {
// Log connection failures at debug level since some failures are expected
// during service startup or when replicas are being updated.
log.Debug().
Err(err).
Str("endpoint", endpoint).
Int("attempt", attempt).
Msg("Failed to query CHORUS health endpoint")
continue
}
// Process successful health check responses
if resp.StatusCode == http.StatusOK {
// Generate a synthetic agent ID since CHORUS doesn't provide unique IDs yet.
// In production, this would come from the health check response body.
// Using zero-padded numbers ensures consistent sorting in the UI.
agentID := fmt.Sprintf("chorus-agent-%03d", len(discovered)+1)
// Only create new agent if we haven't seen this ID before in this cycle
if _, exists := discovered[agentID]; !exists {
// Create agent with realistic metadata for team formation.
// These capabilities and models would normally come from the
// actual CHORUS agent configuration.
agent := &Agent{
ID: agentID,
Name: fmt.Sprintf("CHORUS Agent %d", len(discovered)+1),
Status: "online", // Default to online since health check succeeded
// Standard CHORUS agent capabilities - these define what types of
// tasks the agent can handle in team formation algorithms
Capabilities: []string{"general_development", "task_coordination", "ai_integration"},
Model: "llama3.1:8b", // Standard model for CHORUS agents
Endpoint: "http://chorus:8080", // API port for task assignment
LastSeen: baseTime, // Consistent timestamp for this discovery cycle
// Synthetic task completion count for load balancing algorithms.
// In production, this would be actual metrics from agent performance.
TasksCompleted: len(discovered) * 2,
P2PAddr: "chorus:9000", // P2P communication port
ClusterID: "docker-unified-stack", // Docker Swarm cluster identifier
}
// Add some variety to agent status for realistic team formation testing.
// This simulates real-world scenarios where agents have different availability.
if len(discovered)%3 == 0 {
agent.Status = "idle" // Every third agent is idle
} else if len(discovered) == 6 {
// One agent is actively working on a team assignment
agent.Status = "working"
agent.CurrentTeam = "development-team-alpha"
}
// Add to discovered agents and log the discovery
discovered[agentID] = agent
log.Debug().
Str("agent_id", agentID).
Str("status", agent.Status).
Msg("🤖 Discovered CHORUS agent")
}
}
resp.Body.Close()
// Stop discovery once we've found the expected number of agents.
// This prevents unnecessary HTTP requests and speeds up discovery cycles.
if len(discovered) >= 9 {
break
}
// Brief pause between requests to avoid overwhelming the service and
// to allow Docker Swarm's load balancer to potentially route to different replicas.
time.Sleep(100 * time.Millisecond)
}
// Add all discovered agents
for _, agent := range discovered {
d.addOrUpdateAgent(agent)
}
log.Info().
Int("discovered_count", len(discovered)).
Msg("🎭 CHORUS agent discovery completed")
}
// addOrUpdateAgent adds or updates an agent in the discovery cache
func (d *Discovery) addOrUpdateAgent(agent *Agent) {
d.mu.Lock()
defer d.mu.Unlock()
existing, exists := d.agents[agent.ID]
if exists {
// Update existing agent
existing.Status = agent.Status
existing.LastSeen = agent.LastSeen
existing.TasksCompleted = agent.TasksCompleted
existing.CurrentTeam = agent.CurrentTeam
} else {
// Add new agent
d.agents[agent.ID] = agent
log.Info().
Str("agent_id", agent.ID).
Str("p2p_addr", agent.P2PAddr).
Msg("🤖 Discovered new CHORUS agent")
}
}
// cleanupStaleAgents removes agents that haven't been seen recently
func (d *Discovery) cleanupStaleAgents() {
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
for {
select {
case <-d.ctx.Done():
return
case <-ticker.C:
d.removeStaleAgents()
}
}
}
// removeStaleAgents removes agents that haven't been seen in 5 minutes
func (d *Discovery) removeStaleAgents() {
d.mu.Lock()
defer d.mu.Unlock()
staleThreshold := time.Now().Add(-5 * time.Minute)
for id, agent := range d.agents {
if agent.LastSeen.Before(staleThreshold) {
delete(d.agents, id)
log.Info().
Str("agent_id", id).
Time("last_seen", agent.LastSeen).
Msg("🧹 Removed stale agent")
}
}
}

1266
internal/server/server.go Normal file

File diff suppressed because it is too large Load Diff